web_logs: Refactor and store daily logs

roles/web_logs/files/parse-access-logs.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -eu
+umask 027
+
+get_vhosts() {
+    {
+        for hostdir in /srv/weblog/* ; do
+            [ -d "$hostdir" ] || continue
+            for log in "${hostdir}/"*.access.log ; do
+                [ -f "$log" ] || continue
+                basename "$log" ".access.log"
+            done
+        done
+    } | sort | uniq
+}
+
+print_date() {
+    date -r "$(($(date +%s) - $1 * 86400))" "+%Y-%m-%d"
+}
+
+get_vhosts | while read -r vhost ; do
+    destdir="/srv/weblog/parsed/${vhost}"
+    [ -d "$destdir" ] || mkdir "$destdir"
+    for i in $(seq 0 7); do
+        isodate="$(print_date $i)"
+	outfile="${destdir}/access.log.${isodate}"
+        combine-logs -d "$isodate" \
+            /srv/weblog/*/"${vhost}".access.log* > "${outfile}.tmp"
+        if [ -s "${outfile}.tmp" ]; then
+            mv "${outfile}.tmp" "$outfile"
+        else
+            rm -f "${outfile}.tmp"
+            break
+        fi
+    done
+done

roles/web_logs/tasks/main.yml

@@ -1,44 +1,44 @@
 ---
-- name: Create logsync group
+- name: Create weblog group
   ansible.builtin.group:
-    name: logsync
+    name: weblog
     gid: 312
     system: true
 
-- name: Create logsync user
+- name: Create weblog user
   ansible.builtin.user:
-    name: logsync
+    name: weblog
-    comment: Service logsync
+    comment: Service weblog
     createhome: false
-    group: logsync
+    group: weblog
     home: /var/empty
     shell: /bin/sh
     system: true
     uid: 312
 
+- name: Create data directory
+  ansible.builtin.file:
+    path: /export/weblog
+    state: directory
+    mode: "0770"
+    owner: root
+    group: weblog
+
+- name: Link data directory
+  ansible.builtin.file:
+    path: /srv/weblog
+    src: /export/weblog
+    state: link
+    owner: root
+    group: "{{ ansible_wheel }}"
+    follow: false
+
 - name: Include rclone role
   ansible.builtin.include_role:
     name: rclone
   vars:
     rclone_hostgroup: proxy
-    rclone_service: logsync
+    rclone_service: weblog
-
-- name: Create data directory
-  ansible.builtin.file:
-    path: /export/web-log
-    state: directory
-    mode: "0750"
-    owner: root
-    group: "{{ ansible_wheel }}"
-
-- name: Link data directory
-  ansible.builtin.file:
-    path: /srv/web-log
-    src: /export/web-log
-    state: link
-    owner: root
-    group: "{{ ansible_wheel }}"
-    follow: false
 
 - name: Copy log combiner
   ansible.builtin.copy:
@@ -47,3 +47,19 @@
     mode: "0755"
     owner: root
     group: "{{ ansible_wheel }}"
+
+- name: Copy log parser
+  ansible.builtin.copy:
+    dest: /usr/local/bin/parse-access-logs
+    src: parse-access-logs.sh
+    mode: "0755"
+    owner: root
+    group: "{{ ansible_wheel }}"
+
+- name: Install log parser cron job
+  ansible.builtin.cron:
+    name: parse-access-logs
+    job: /usr/local/bin/parse-access-logs
+    user: weblog
+    hour: "04"
+    minute: "00"