add mirror role and enable syncs on mirror group

playbooks/mirror.yml

@@ -19,3 +19,39 @@
 
   roles:
     - base
+    - mirror/base
+    - role: mirror/sync
+      label: fedora-epel
+      source: "rsync://rsync.nic.funet.fi/ftp/pub/mirrors/fedora.redhat.com/pub/epel"
+      rsyncoptions:
+        - "--exclude=SRPMS"
+        - "--exclude=debug"
+        - "--delete-excluded"
+    - role: mirror/sync
+      label: fedora
+      source: "rsync://rsync.nic.funet.fi/ftp/pub/mirrors/fedora.redhat.com/pub/fedora/linux/"
+      rsyncoptions:
+        - "--exclude=/atomic"
+        - "--exclude=/development"
+        - "--exclude=/releases/test"
+        - "--exclude=/updates/testing"
+        - "--exclude=Live"
+        - "--exclude=SRPMS"
+        - "--exclude=source"
+        - "--exclude=armhfp"
+        - "--exclude=debug"
+        - "--delete-excluded"
+    - role: mirror/sync
+      label: openbsd
+      source: "rsync://ftp.aso.ee/OpenBSD/"
+      rsyncoptions:
+        - "--include=/?.?/"
+        - "--include=/?.?/packages/"
+        - "--include=/?.?/packages/amd64/"
+        - "--include=/?.?/packages/amd64/*"
+        - "--include=/syspatch/"
+        - "--include=/syspatch/?.?/"
+        - "--include=/syspatch/?.?/amd64/"
+        - "--include=/syspatch/?.?/amd64/*"
+        - "--exclude=*"
+        - "--delete-excluded"

roles/mirror/base/files/sync-mirrors

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+LOCKFILE="/var/run/sync-mirrors/lockfile"
+LOGFILE="/var/log/sync-mirrors/sync-mirrors-$(date +%Y%m%d%H%M%S).log"
+CONFDIR="/etc/sync-mirrors"
+
+usage() {
+    echo "Usage: $(basename "$0") [-v] [mirror]" 1>&2
+    echo "       $(basename "$0") -l" 1>&2
+}
+
+logmsg() {
+    [ "${VERBOSE}" -eq 1 ] && echo "$1"
+    echo "$(date '+%Y/%m/%d %H:%M:%S') [$$] $1" >> "${LOGFILE}"
+}
+
+if [ -d ${CONFDIR} ]; then
+    MIRRORLIST="$(find ${CONFDIR}/ -name \*.conf | while read f ; \
+        do basename "${f}" | sed -e 's/\.conf$//' ; done)"
+    if [ "${MIRRORLIST}" = "" ]; then
+        echo "ERR: No configured mirrors found" 1>&2
+        exit 1
+    fi
+else
+    echo "ERR: Config directory [${CONFDIR}] missing" 1>&2
+    exit 1
+fi
+
+VERBOSE=0
+NOOP=""
+EXTRA_OPTS=""
+while getopts "vhln" c ; do
+    case $c in
+	v)
+	    VERBOSE=1
+	    EXTRA_OPTS="${EXTRA_OPTS} -v --progress"
+	    ;;
+	h)
+	    usage
+	    exit 1
+	    ;;
+	l)
+	    echo "Available mirrors:"
+            for name in ${MIRRORLIST} ; do
+		echo "  ${name}"
+	    done
+	    exit 0
+	    ;;
+        n)
+            NOOP=" (DRY RUN)"
+            EXTRA_OPTS="${EXTRA_OPTS} -n"
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+shift "$((OPTIND - 1))"
+
+if [ $# -gt 0 ]; then
+    for mirror in "$@" ; do
+        if [ ! -f "${CONFDIR}/$1.conf" ]; then
+            echo "ERR: No mirror named [$1]" 1>&2
+            exit 1
+        fi
+        SYNC="${MIRRORS} $1"
+        shift
+    done
+else
+    SYNC="${MIRRORLIST}"
+fi
+
+if [ "$(whoami)" != "mirror" ]; then
+    echo "ERR: Script needs to be run as mirror user" 1>&2
+    exit 1
+fi
+
+umask 022
+
+if [ -f "${LOCKFILE}" ]; then
+    kill -0 "$(cat ${LOCKFILE})"
+    if [ $? -ne 1 ]; then
+	which stat > /dev/null 2>&1
+	if [ $? -eq 0 ]; then
+	    STARTED=" ($(stat --format='%y' ${LOCKFILE}))"
+	else
+	    STARTED=""
+	fi
+        echo "ERR: Lockfile exists${STARTED}, exiting" 1>&2
+        exit 1
+    else
+        echo "WARN: Removing stale lock file..." 1>&2
+        rm -f "${LOCKFILE}"
+    fi
+fi
+trap 'rm -f ${LOCKFILE}' INT TERM EXIT
+echo "$$" > "${LOCKFILE}"
+
+for mirror in ${SYNC} ; do
+    POSTCMD=""
+    SRC=""
+    RSYNCOPTS=""
+    . "${CONFDIR}/${mirror}.conf"
+    if [ "${SRC}" = "" ]; then
+        echo "ERR: No SRC set for mirror ${mirror} ..." 1>&2
+        exit 1
+    fi
+    logmsg "Starting ${mirror} sync${NOOP}..."
+    rsync -aH -4 ${EXTRA_OPTS} --numeric-ids --delete --delete-delay \
+        --delay-updates --no-motd ${RSYNCOPTS} --log-file="${LOGFILE}" \
+        --exclude=.~tmp~/ "${SRC}" "/srv/mirrors/${mirror}/"
+    STATUS=$?
+    if [ ${STATUS} -ne 0 ]; then
+        echo "WARN: Encountered errors on ${mirror} sync, see ${LOGFILE} for details" 1>&2
+    fi
+    logmsg "Finished ${mirror} sync with exit status ${STATUS}${NOOP} ..."
+    if [ "${POSTCMD}" != "" ]; then
+        logmsg "Running post for ${mirror} ..."
+        ${POSTCMD} 2>&1 | awk \
+           "{ print strftime(\"%Y/%m/%d %H:%M:%S\") \" [$$] \" \$0 }" \
+           >> "${LOGFILE}"
+        logmsg "Finished post for ${mirror} ..."
+    fi
+done
+
+rm -f "${LOCKFILE}"

roles/mirror/base/tasks/main.yml

@@ -0,0 +1,87 @@
+---
+- name: install rsync
+  package:
+    name: rsync
+    state: installed
+
+- name: create mirror group
+  group:
+    name: mirror
+    gid: 1001
+
+- name: create mirror user
+  user:
+    name: mirror
+    comment: Service Mirror
+    createhome: false
+    group: mirror
+    home: /var/empty
+    shell: /sbin/nologin
+    uid: 1001
+
+- name: create data directory
+  file:
+    path: /export/mirrors
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+
+- name: create data directory link
+  file:
+    path: /srv/mirrors
+    state: link
+    src: /export/mirrors
+    owner: root
+    group: root
+
+- name: create config directory
+  file:
+    path: /etc/sync-mirrors
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+
+- name: create runtime and log directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+    owner: mirror
+    group: mirror
+  with_items:
+    - /var/run/sync-mirrors
+    - /var/log/sync-mirrors
+
+- name: configure tmpfiles to create runtime directory on boot
+  copy:
+    dest: /usr/lib/tmpfiles.d/sync-mirrors.conf
+    content: "d /run/sync-mirrors 0755 mirror mirror\n"
+    mode: 0644
+    owner: root
+    group: root
+
+- name: copy mirroring script
+  copy:
+    dest: /usr/local/bin/sync-mirrors
+    src: sync-mirrors
+    mode: 0755
+    owner: root
+    group: root
+
+- name: create mirror cron job
+  cron:
+    name: sync-mirrors
+    hour: "0,6,12,18"
+    minute: 0
+    job: /usr/local/bin/sync-mirrors
+    user: mirror
+
+- name: create log rotate cron job for sync-mirrors
+  cron:
+    name: sync-mirrors-logs
+    hour: 5
+    minute: 10
+    job: "find /var/log/sync-mirrors/ -xdev -type f -mtime +30 -name '*.log' -execdir rm {} \\+"
+    user: mirror

roles/mirror/sync/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+rsyncoptions: []
+postcmd: ""

roles/mirror/sync/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - {role: mirror/base}

roles/mirror/sync/tasks/main.yml

@@ -0,0 +1,18 @@
+
+---
+
+- name: create config for {{ label }}
+  template:
+    dest: "/etc/sync-mirrors/{{ label }}.conf"
+    src: mirror.conf.j2
+    mode: 0644
+    owner: root
+    group: root
+
+- name: create target directory
+  file:
+    path: "/srv/mirrors/{{ label }}"
+    state: directory
+    mode: 0755
+    owner: mirror
+    group: mirror

roles/mirror/sync/templates/mirror.conf.j2

@@ -0,0 +1,2 @@
+SRC="{{ source }}"
+RSYNCOPTS="{{ rsyncoptions | join(' ') }}"