diff --git a/roles/web-logs/files/sync-http-logs.sh b/roles/web-logs/files/sync-http-logs.sh new file mode 100755 index 0000000..45d1e0f --- /dev/null +++ b/roles/web-logs/files/sync-http-logs.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -u +umask 022 + +TARGET="/var/cache/sync-http-logs" +CONFIG="/etc/rclone/rclone.conf" +LOGDIR="/var/log/rclone" + +timestamp="$(date +%Y%m%d%H%M%S)" + +if [ ! -d "$TARGET" ]; then + echo "ERR: Destination directory '${TARGET}' does not exist" 1>&2 + exit 1 +fi + +for host in $(rclone --config "$CONFIG" listremotes | tr -d ":") ; do + fqdn="$(rclone --config "$CONFIG" config show "$host" | \ + awk '{ if ($1 == "host") print $3 }')" + if [ ! -d "${TARGET}/${fqdn}" ]; then + mkdir "${TARGET}/${fqdn}" + fi + log="${LOGDIR}/${fqdn}.${timestamp}.log" + if ! rclone --config "$CONFIG" --log-file "$log" --log-level INFO \ + sync "${host}:/" "${TARGET}/${fqdn}/"; then + cat "$log" + fi +done diff --git a/roles/web-logs/tasks/main.yml b/roles/web-logs/tasks/main.yml new file mode 100644 index 0000000..b377af9 --- /dev/null +++ b/roles/web-logs/tasks/main.yml @@ -0,0 +1,65 @@ +--- +- name: install packages + package: + name: "{{ item }}" + state: installed + with_items: + - rclone + +- name: create ssh known_hosts + template: + dest: /etc/ssh/ssh_known_hosts + src: ssh_known_hosts.j2 + mode: 0644 + owner: root + group: "{{ ansible_wheel }}" + +- name: create data directories + file: + path: "{{ item }}" + state: directory + mode: 0755 + owner: root + group: "{{ ansible_wheel }}" + with_items: + - /var/cache/sync-http-logs + +- name: create log directory + file: + path: /var/log/rclone + state: directory + mode: 0755 + owner: root + group: "{{ ansible_wheel }}" + +- name: copy logsync script + copy: + dest: /usr/local/bin/sync-http-logs + src: sync-http-logs.sh + mode: 0755 + owner: root + group: "{{ ansible_wheel }}" + +- name: create config directory + file: + path: /etc/rclone + state: directory + mode: 0755 + owner: root + group: "{{ ansible_wheel }}" + +- name: create host config + template: + dest: /etc/rclone/rclone.conf + src: rclone.conf.j2 + mode: 0644 + owner: root + group: "{{ ansible_wheel }}" + +- name: add log sync cron job + cron: + name: sync-http-logs + user: root + hour: "3" + minute: "0" + job: /usr/local/bin/sync-http-logs diff --git a/roles/web-logs/templates/rclone.conf.j2 b/roles/web-logs/templates/rclone.conf.j2 new file mode 100644 index 0000000..34524ec --- /dev/null +++ b/roles/web-logs/templates/rclone.conf.j2 @@ -0,0 +1,10 @@ +# {{ ansible_managed }} +{% for host in groups['webservers'] %} + +[{{ host.split('.')[0] }}] +type = sftp +host = {{ host }} +user = logsync +key_file = ~/.ssh/id_ed25519 +known_hosts_file = /etc/ssh/ssh_known_hosts +{% endfor %} diff --git a/roles/web-logs/templates/ssh_known_hosts.j2 b/roles/web-logs/templates/ssh_known_hosts.j2 new file mode 100644 index 0000000..d6fc971 --- /dev/null +++ b/roles/web-logs/templates/ssh_known_hosts.j2 @@ -0,0 +1,5 @@ +{% for host, vars in hostvars|dictsort %} +{% if vars["ansible_ssh_host_key_ed25519_public"] is defined %} +{{ host }} ssh-ed25519 {{ vars["ansible_ssh_host_key_ed25519_public"] }} +{% endif %} +{% endfor %}