From 278f2deb928e9334aaa03107ceae1976d8b8c3bb Mon Sep 17 00:00:00 2001 From: Timo Makinen Date: Sat, 11 May 2019 01:57:50 +0300 Subject: [PATCH] add mirror role and enable syncs on mirror group --- playbooks/mirror.yml | 36 ++++++ roles/mirror/base/files/sync-mirrors | 128 +++++++++++++++++++++ roles/mirror/base/tasks/main.yml | 87 ++++++++++++++ roles/mirror/sync/defaults/main.yml | 3 + roles/mirror/sync/meta/main.yml | 3 + roles/mirror/sync/tasks/main.yml | 18 +++ roles/mirror/sync/templates/mirror.conf.j2 | 2 + 7 files changed, 277 insertions(+) create mode 100755 roles/mirror/base/files/sync-mirrors create mode 100644 roles/mirror/base/tasks/main.yml create mode 100644 roles/mirror/sync/defaults/main.yml create mode 100644 roles/mirror/sync/meta/main.yml create mode 100644 roles/mirror/sync/tasks/main.yml create mode 100644 roles/mirror/sync/templates/mirror.conf.j2 diff --git a/playbooks/mirror.yml b/playbooks/mirror.yml index 4f2d68a..fcbf583 100644 --- a/playbooks/mirror.yml +++ b/playbooks/mirror.yml @@ -19,3 +19,39 @@ roles: - base + - mirror/base + - role: mirror/sync + label: fedora-epel + source: "rsync://rsync.nic.funet.fi/ftp/pub/mirrors/fedora.redhat.com/pub/epel" + rsyncoptions: + - "--exclude=SRPMS" + - "--exclude=debug" + - "--delete-excluded" + - role: mirror/sync + label: fedora + source: "rsync://rsync.nic.funet.fi/ftp/pub/mirrors/fedora.redhat.com/pub/fedora/linux/" + rsyncoptions: + - "--exclude=/atomic" + - "--exclude=/development" + - "--exclude=/releases/test" + - "--exclude=/updates/testing" + - "--exclude=Live" + - "--exclude=SRPMS" + - "--exclude=source" + - "--exclude=armhfp" + - "--exclude=debug" + - "--delete-excluded" + - role: mirror/sync + label: openbsd + source: "rsync://ftp.aso.ee/OpenBSD/" + rsyncoptions: + - "--include=/?.?/" + - "--include=/?.?/packages/" + - "--include=/?.?/packages/amd64/" + - "--include=/?.?/packages/amd64/*" + - "--include=/syspatch/" + - "--include=/syspatch/?.?/" + - "--include=/syspatch/?.?/amd64/" + - "--include=/syspatch/?.?/amd64/*" + - "--exclude=*" + - "--delete-excluded" diff --git a/roles/mirror/base/files/sync-mirrors b/roles/mirror/base/files/sync-mirrors new file mode 100755 index 0000000..7d8d330 --- /dev/null +++ b/roles/mirror/base/files/sync-mirrors @@ -0,0 +1,128 @@ +#!/bin/bash + +LOCKFILE="/var/run/sync-mirrors/lockfile" +LOGFILE="/var/log/sync-mirrors/sync-mirrors-$(date +%Y%m%d%H%M%S).log" +CONFDIR="/etc/sync-mirrors" + +usage() { + echo "Usage: $(basename "$0") [-v] [mirror]" 1>&2 + echo " $(basename "$0") -l" 1>&2 +} + +logmsg() { + [ "${VERBOSE}" -eq 1 ] && echo "$1" + echo "$(date '+%Y/%m/%d %H:%M:%S') [$$] $1" >> "${LOGFILE}" +} + +if [ -d ${CONFDIR} ]; then + MIRRORLIST="$(find ${CONFDIR}/ -name \*.conf | while read f ; \ + do basename "${f}" | sed -e 's/\.conf$//' ; done)" + if [ "${MIRRORLIST}" = "" ]; then + echo "ERR: No configured mirrors found" 1>&2 + exit 1 + fi +else + echo "ERR: Config directory [${CONFDIR}] missing" 1>&2 + exit 1 +fi + +VERBOSE=0 +NOOP="" +EXTRA_OPTS="" +while getopts "vhln" c ; do + case $c in + v) + VERBOSE=1 + EXTRA_OPTS="${EXTRA_OPTS} -v --progress" + ;; + h) + usage + exit 1 + ;; + l) + echo "Available mirrors:" + for name in ${MIRRORLIST} ; do + echo " ${name}" + done + exit 0 + ;; + n) + NOOP=" (DRY RUN)" + EXTRA_OPTS="${EXTRA_OPTS} -n" + ;; + *) + usage + exit 1 + ;; + esac +done + +shift "$((OPTIND - 1))" + +if [ $# -gt 0 ]; then + for mirror in "$@" ; do + if [ ! -f "${CONFDIR}/$1.conf" ]; then + echo "ERR: No mirror named [$1]" 1>&2 + exit 1 + fi + SYNC="${MIRRORS} $1" + shift + done +else + SYNC="${MIRRORLIST}" +fi + +if [ "$(whoami)" != "mirror" ]; then + echo "ERR: Script needs to be run as mirror user" 1>&2 + exit 1 +fi + +umask 022 + +if [ -f "${LOCKFILE}" ]; then + kill -0 "$(cat ${LOCKFILE})" + if [ $? -ne 1 ]; then + which stat > /dev/null 2>&1 + if [ $? -eq 0 ]; then + STARTED=" ($(stat --format='%y' ${LOCKFILE}))" + else + STARTED="" + fi + echo "ERR: Lockfile exists${STARTED}, exiting" 1>&2 + exit 1 + else + echo "WARN: Removing stale lock file..." 1>&2 + rm -f "${LOCKFILE}" + fi +fi +trap 'rm -f ${LOCKFILE}' INT TERM EXIT +echo "$$" > "${LOCKFILE}" + +for mirror in ${SYNC} ; do + POSTCMD="" + SRC="" + RSYNCOPTS="" + . "${CONFDIR}/${mirror}.conf" + if [ "${SRC}" = "" ]; then + echo "ERR: No SRC set for mirror ${mirror} ..." 1>&2 + exit 1 + fi + logmsg "Starting ${mirror} sync${NOOP}..." + rsync -aH -4 ${EXTRA_OPTS} --numeric-ids --delete --delete-delay \ + --delay-updates --no-motd ${RSYNCOPTS} --log-file="${LOGFILE}" \ + --exclude=.~tmp~/ "${SRC}" "/srv/mirrors/${mirror}/" + STATUS=$? + if [ ${STATUS} -ne 0 ]; then + echo "WARN: Encountered errors on ${mirror} sync, see ${LOGFILE} for details" 1>&2 + fi + logmsg "Finished ${mirror} sync with exit status ${STATUS}${NOOP} ..." + if [ "${POSTCMD}" != "" ]; then + logmsg "Running post for ${mirror} ..." + ${POSTCMD} 2>&1 | awk \ + "{ print strftime(\"%Y/%m/%d %H:%M:%S\") \" [$$] \" \$0 }" \ + >> "${LOGFILE}" + logmsg "Finished post for ${mirror} ..." + fi +done + +rm -f "${LOCKFILE}" diff --git a/roles/mirror/base/tasks/main.yml b/roles/mirror/base/tasks/main.yml new file mode 100644 index 0000000..198abff --- /dev/null +++ b/roles/mirror/base/tasks/main.yml @@ -0,0 +1,87 @@ +--- +- name: install rsync + package: + name: rsync + state: installed + +- name: create mirror group + group: + name: mirror + gid: 1001 + +- name: create mirror user + user: + name: mirror + comment: Service Mirror + createhome: false + group: mirror + home: /var/empty + shell: /sbin/nologin + uid: 1001 + +- name: create data directory + file: + path: /export/mirrors + state: directory + mode: 0755 + owner: root + group: root + +- name: create data directory link + file: + path: /srv/mirrors + state: link + src: /export/mirrors + owner: root + group: root + +- name: create config directory + file: + path: /etc/sync-mirrors + state: directory + mode: 0755 + owner: root + group: root + +- name: create runtime and log directories + file: + path: "{{ item }}" + state: directory + mode: 0755 + owner: mirror + group: mirror + with_items: + - /var/run/sync-mirrors + - /var/log/sync-mirrors + +- name: configure tmpfiles to create runtime directory on boot + copy: + dest: /usr/lib/tmpfiles.d/sync-mirrors.conf + content: "d /run/sync-mirrors 0755 mirror mirror\n" + mode: 0644 + owner: root + group: root + +- name: copy mirroring script + copy: + dest: /usr/local/bin/sync-mirrors + src: sync-mirrors + mode: 0755 + owner: root + group: root + +- name: create mirror cron job + cron: + name: sync-mirrors + hour: "0,6,12,18" + minute: 0 + job: /usr/local/bin/sync-mirrors + user: mirror + +- name: create log rotate cron job for sync-mirrors + cron: + name: sync-mirrors-logs + hour: 5 + minute: 10 + job: "find /var/log/sync-mirrors/ -xdev -type f -mtime +30 -name '*.log' -execdir rm {} \\+" + user: mirror diff --git a/roles/mirror/sync/defaults/main.yml b/roles/mirror/sync/defaults/main.yml new file mode 100644 index 0000000..264336b --- /dev/null +++ b/roles/mirror/sync/defaults/main.yml @@ -0,0 +1,3 @@ +--- +rsyncoptions: [] +postcmd: "" diff --git a/roles/mirror/sync/meta/main.yml b/roles/mirror/sync/meta/main.yml new file mode 100644 index 0000000..7e5648a --- /dev/null +++ b/roles/mirror/sync/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - {role: mirror/base} diff --git a/roles/mirror/sync/tasks/main.yml b/roles/mirror/sync/tasks/main.yml new file mode 100644 index 0000000..ee11cd5 --- /dev/null +++ b/roles/mirror/sync/tasks/main.yml @@ -0,0 +1,18 @@ + +--- + +- name: create config for {{ label }} + template: + dest: "/etc/sync-mirrors/{{ label }}.conf" + src: mirror.conf.j2 + mode: 0644 + owner: root + group: root + +- name: create target directory + file: + path: "/srv/mirrors/{{ label }}" + state: directory + mode: 0755 + owner: mirror + group: mirror diff --git a/roles/mirror/sync/templates/mirror.conf.j2 b/roles/mirror/sync/templates/mirror.conf.j2 new file mode 100644 index 0000000..bac8f83 --- /dev/null +++ b/roles/mirror/sync/templates/mirror.conf.j2 @@ -0,0 +1,2 @@ +SRC="{{ source }}" +RSYNCOPTS="{{ rsyncoptions | join(' ') }}"