web-logs: Initial version of role

roles/web-logs/files/sync-http-logs.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -u
+umask 022
+
+TARGET="/var/cache/sync-http-logs"
+CONFIG="/etc/rclone/rclone.conf"
+LOGDIR="/var/log/rclone"
+
+timestamp="$(date +%Y%m%d%H%M%S)"
+
+if [ ! -d "$TARGET" ]; then
+    echo "ERR: Destination directory '${TARGET}' does not exist" 1>&2
+    exit 1
+fi
+
+for host in $(rclone --config "$CONFIG" listremotes | tr -d ":") ; do
+    fqdn="$(rclone --config "$CONFIG" config show "$host" | \
+        awk '{ if ($1 == "host") print $3 }')"
+    if [ ! -d "${TARGET}/${fqdn}" ]; then
+        mkdir "${TARGET}/${fqdn}"
+    fi
+    log="${LOGDIR}/${fqdn}.${timestamp}.log"
+    if ! rclone --config "$CONFIG" --log-file "$log" --log-level INFO \
+        sync "${host}:/" "${TARGET}/${fqdn}/"; then
+        cat "$log"
+    fi
+done

roles/web-logs/tasks/main.yml

@@ -0,0 +1,65 @@
+---
+- name: install packages
+  package:
+    name: "{{ item }}"
+    state: installed
+  with_items:
+    - rclone
+
+- name: create ssh known_hosts
+  template:
+    dest: /etc/ssh/ssh_known_hosts
+    src: ssh_known_hosts.j2
+    mode: 0644
+    owner: root
+    group: "{{ ansible_wheel }}"
+
+- name: create data directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+    owner: root
+    group: "{{ ansible_wheel }}"
+  with_items:
+    - /var/cache/sync-http-logs
+
+- name: create log directory
+  file:
+    path: /var/log/rclone
+    state: directory
+    mode: 0755
+    owner: root
+    group: "{{ ansible_wheel }}"
+
+- name: copy logsync script
+  copy:
+    dest: /usr/local/bin/sync-http-logs
+    src: sync-http-logs.sh
+    mode: 0755
+    owner: root
+    group: "{{ ansible_wheel }}"
+
+- name: create config directory
+  file:
+    path: /etc/rclone
+    state: directory
+    mode: 0755
+    owner: root
+    group: "{{ ansible_wheel }}"
+
+- name: create host config
+  template:
+    dest: /etc/rclone/rclone.conf
+    src: rclone.conf.j2
+    mode: 0644
+    owner: root
+    group: "{{ ansible_wheel }}"
+
+- name: add log sync cron job
+  cron:
+    name: sync-http-logs
+    user: root
+    hour: "3"
+    minute: "0"
+    job: /usr/local/bin/sync-http-logs

roles/web-logs/templates/rclone.conf.j2

@@ -0,0 +1,10 @@
+# {{ ansible_managed }}
+{% for host in groups['webservers'] %}
+
+[{{ host.split('.')[0] }}]
+type = sftp
+host = {{ host }}
+user = logsync
+key_file = ~/.ssh/id_ed25519
+known_hosts_file = /etc/ssh/ssh_known_hosts
+{% endfor %}

roles/web-logs/templates/ssh_known_hosts.j2

@@ -0,0 +1,5 @@
+{% for host, vars in hostvars|dictsort %}
+{%   if vars["ansible_ssh_host_key_ed25519_public"] is defined %}
+{{ host }} ssh-ed25519 {{ vars["ansible_ssh_host_key_ed25519_public"] }}
+{%   endif %}
+{% endfor %}