From 78b333cf6cbc95829d3982ed346d85be25bbba20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20M=E4kinen?= Date: Fri, 22 Mar 2013 00:27:43 +0200 Subject: [PATCH] Initial version of mirror module. --- mirror/files/sync-mirrors | 103 +++++++++++++ mirror/manifests/init.pp | 186 ++++++++++++++++++++++++ mirror/templates/mirror.conf.erb | 2 + mirror/templates/report_mirror.conf.erb | 96 ++++++++++++ user/manifests/init.pp | 15 ++ 5 files changed, 402 insertions(+) create mode 100755 mirror/files/sync-mirrors create mode 100644 mirror/manifests/init.pp create mode 100644 mirror/templates/mirror.conf.erb create mode 100644 mirror/templates/report_mirror.conf.erb diff --git a/mirror/files/sync-mirrors b/mirror/files/sync-mirrors new file mode 100755 index 0000000..52f6fbf --- /dev/null +++ b/mirror/files/sync-mirrors @@ -0,0 +1,103 @@ +#!/bin/bash + +LOCKFILE=/var/run/sync-mirrors/lockfile +LOGFILE=/var/log/sync-mirrors/sync-mirrors-`date +%Y%m%d%H%M%S`.log +CONFDIR=/etc/sync-mirrors + +usage() { + echo "Usage: `basename $0` [-v] [mirror]" 1>&2 + echo " `basename $0` -l" 1>&2 +} + +if [ -d ${CONFDIR} ]; then + MIRRORLIST=`ls ${CONFDIR}/*.conf 2> /dev/null | while read f ; \ + do basename $f | sed -e 's/\.conf$//' ; done` + if [ "${MIRRORLIST}" = "" ]; then + echo "ERR: No configured mirrors found" 1>&2 + exit 1 + fi +else + echo "ERR: Config directory [${CONFDIR}] missing" 1>&2 + exit 1 +fi + +VERBOSE=0 +EXTRA_OPTS="" +while getopts "vhl" c ; do + case $c in + v) + VERBOSE=1 + EXTRA_OPTS="-v --progress" + ;; + h) + usage + exit 1 + ;; + l) + echo "Available mirrors:" + for name in ${MIRRORLIST} ; do + echo " ${name}" + done + exit 0 + ;; + esac +done + +shift `expr $OPTIND - 1` + +if [ $# -gt 0 ]; then + while true ; do + if [ ! -f ${CONFDIR}/$1.conf ]; then + echo "ERR: No mirror named [$1]" 1>&2 + exit 1 + fi + SYNC="${MIRRORS} $1" + shift + done +else + SYNC=${MIRRORLIST} +fi + +if [ `whoami` != mirror ]; then + echo "ERR: Script needs to be run as mirror user" 1>&2 + exit 1 +fi + +umask 022 + +if [ -f ${LOCKFILE} ]; then + kill -0 `cat ${LOCKFILE}` + if [ $? -ne 1 ]; then + which stat > /dev/null 2>&1 + if [ $? -eq 0 ]; then + STARTED=" (`stat ${LOCKFILE} | sed -n 's/^Modify: \(.*\)/\1/p'`)" + else + STARTED="" + fi + echo "ERR: Lockfile exists${STARTED}, exiting" 1>&2 + exit 1 + else + echo "WARN: Removing stale lock file..." 1>&2 + rm -f ${LOCKFILE} + fi +fi +trap "rm -f ${LOCKFILE}" INT TERM EXIT +echo $$ > ${LOCKFILE} + +for mirror in ${SYNC} ; do + SRC="" + RSYNCOPTS="" + . ${CONFDIR}/${mirror}.conf + if [ "${SRC}" = "" ]; then + echo "ERR: No SRC set for mirror ${mirror} ..." 1>&2 + exit 1 + fi + [ ${VERBOSE} -eq 1 ] && echo "Starting ${mirror} sync ..." + echo "`date '+%Y/%m/%d %H:%M:%S'` [$$] Starting ${mirror} sync ..." \ + >> ${LOGFILE} + rsync -aH -4 ${EXTRA_OPTS} --numeric-ids --delete --delete-after \ + --delay-updates --no-motd ${RSYNCOPTS} --log-file=${LOGFILE} \ + ${SRC} /srv/mirrors/${mirror}/ +done + +rm -f ${LOCKFILE} diff --git a/mirror/manifests/init.pp b/mirror/manifests/init.pp new file mode 100644 index 0000000..9001254 --- /dev/null +++ b/mirror/manifests/init.pp @@ -0,0 +1,186 @@ + +# Install mirroring scripts +# +# === Parameters +# +# $datadir: +# Directory for mirrored data. Defaults to /srv/mirrors +# +class mirror($datadir = "/srv/mirrors") { + + include user::system + realize(User["mirror"], Group["mirror"]) + + if $datadir != "/srv/mirrors" { + file { "/srv/mirrors": + ensure => link, + target => $datadir, + owner => "root", + group => "root", + seltype => "httpd_sys_content_t", + before => File[$datadir], + } + selinux::manage_fcontext { "/srv/mirrors(/.*)?": + type => "httpd_sys_content_t", + before => File["/srv/mirrors"], + } + } + file { $datadir: + ensure => directory, + mode => "0755", + owner => "root", + group => "root", + seltype => "httpd_sys_content_t", + } + selinux::manage_fcontext { "${datadir}(/.*)?": + type => "httpd_sys_content_t", + before => File[$datadir], + } + + file { "/etc/sync-mirrors": + ensure => directory, + mode => "0755", + owner => "root", + group => "root", + recurse => true, + purge => true, + } + + file { [ "/var/run/sync-mirrors", "/var/log/sync-mirrors", ]: + ensure => directory, + mode => "0755", + owner => "mirror", + group => "mirror", + before => Cron["sync-mirrors"], + require => User["mirror"], + } + + file { "/usr/local/bin/sync-mirrors": + ensure => present, + source => "puppet:///modules/mirror/sync-mirrors", + mode => "0755", + owner => "root", + group => "root", + } + cron { "sync-mirrors": + command => "/usr/local/bin/sync-mirrors", + user => "mirror", + hour => [ 0, 6, 12, 18, ], + minute => 0, + require => [ File["/usr/local/bin/sync-mirrors"], User["mirror"], ], + } + +} + + +# Create new mirror +# +# === Parameters: +# +# $name: +# Mirror name +# +# $source: +# Rsync path from where to sync mirror +# +# $rsync_options: +# Extra options for rsync +# +# === Sample usage: +# +# mirrors::mirror { "centos": +# source => "rsync://rsync.nic.funet.fi/ftp/pub/mirrors/centos.org/", +# rsync_options => [ +# "--exclude=SRPMS", +# "--exclude=debug", +# "--exclude=isos", +# ], +# } +# +define mirror::mirror($source, $rsync_options=[]) { + + require mirror + + file { "/etc/sync-mirrors/${name}.conf": + ensure => present, + content => template("mirror/mirror.conf.erb"), + mode => "0644", + owner => "root", + group => "root", + require => File["/srv/mirrors/${name}"], + } + + file { "/srv/mirrors/${name}": + ensure => directory, + mode => "0755", + owner => "mirror", + group => "mirror", + seltype => "httpd_sys_content_t", + } + +} + + +# Run weekly hardlinking for mirrored data +# +class mirror::hardlink { + + require mirror + + package { "hardlink": + ensure => installed, + } + cron { "hardlink-mirrors-weekly": + command => "/usr/sbin/hardlink /srv/mirrors/", + user => "mirror", + hour => 4, + minute => 0, + weekday => 0, + } + +} + + +# Install Fedora mirror reporting tool +# +# https://fedoraproject.org/wiki/Infrastructure/Mirroring +# +# === Parameters: +# +# $sitename: +# Site name configured to mirrormanager +# +# $password: +# Site password configured to mirrormanager +# +# $hostname: +# Host name configured to mirrormanager +# +# $mirrors: +# Components mirrored to this hosts +# +# === Sample usage: +# +# mirrors::reportmirror { +# sitename => "foo.sh" +# password => "secret", +# hostname => "mirrors.foo.sh", +# mirrors => [ "fedora", "epel", ], +# } +# +class mirror::reportmirror($sitename, $password, $hostname, $mirrors=[]) { + + package { "mirrormanager-client": + ensure => installed, + } + + file { "/etc/mirrormanager-client/report_mirror.conf": + ensure => present, + content => template("mirror/report_mirror.conf.erb"), + mode => "0640", + owner => "root", + group => "mirror", + require => [ Package["mirrormanager-client"], Group["mirror"], ], + } + +} diff --git a/mirror/templates/mirror.conf.erb b/mirror/templates/mirror.conf.erb new file mode 100644 index 0000000..cd001a6 --- /dev/null +++ b/mirror/templates/mirror.conf.erb @@ -0,0 +1,2 @@ +SRC="<%= source %>" +RSYNCOPTS="<%= rsync_options.join(' ') %>" diff --git a/mirror/templates/report_mirror.conf.erb b/mirror/templates/report_mirror.conf.erb new file mode 100644 index 0000000..db9682f --- /dev/null +++ b/mirror/templates/report_mirror.conf.erb @@ -0,0 +1,96 @@ +[global] +# if enabled=0, no data is sent to the database +enabled=1 +# server= is the URL to the MirrorManager XML-RPC interface +server=https://admin.fedoraproject.org/mirrormanager/xmlrpc + + +[site] +# if enabled=0, no data about this site is sent to the database +enabled=1 +# Name and Password fields need to match the Site name and password +# fields you entered for your Site in the MirrorManager database at +# https://admin.fedoraproject.org/mirrormanager +name=<%= sitename %> +password=<%= password %> + +[host] +# if enabled=0, no data about this host is sent to the database +enabled=1 +# Name field need to match the Host name field you entered for your +# Host in the MirrorManager database at +# https://admin.fedoraproject.org/mirrormanager +name=<%= hostname %> +# if user_active=0, no data about this category is given to the public +# This can be used to toggle between serving and not serving data, +# such enabled during the nighttime (when you have more idle bandwidth +# available) and disabled during the daytime. +# By not specifying user_active, the database will not be updated. +# user_active=1 + +[stats] +# Stats are only sent when run with the -s option +# and when this section is enabled. +# This feature is not presently implemented +enabled=0 +apache=/var/log/httpd/access_log +vsftpd=/var/log/vsftpd.log +# remember to enable log file and transfer logging in rsyncd.conf +rsyncd=/var/log/rsyncd.log + + +# Content Categories +# These sections match the Categories for content tracked by MirrorManager. +# +# enabled=1 means information about this category will be sent to the database. +# enabled=0, no data about this host is sent to the database. If the +# database already has information for you for this Category, it will +# remain unchanged. This can be used to update the database after you +# have manually synced some infrequently-updated content, such as +# historical releases. +# +# path= is the path on your local disk to the top-level directory for this Category + +[Fedora Linux] +<% if mirrors.include?('fedora') -%> +enabled=1 +<% else -%> +enabled=0 +<% end -%> +path=/srv/mirrors/fedora + +[Fedora EPEL] +<% if mirrors.include?('epel') -%> +enabled=1 +<% else -%> +enabled=0 +<% end -%> +path=/srv/mirrors/fedora-epel + +# lesser used categories below + +[Fedora Web] +enabled=0 +path=/var/www/html/pub/fedora/web + +[Fedora Secondary Arches] +enabled=0 +path=/var/www/html/pub/fedora-secondary + +[Fedora Other] +enabled=0 +path=/var/www/html/pub/alt + +# historical content + +[Fedora Core] +enabled=0 +path=/var/www/html/pub/fedora/linux/core + +[Fedora Extras] +enabled=0 +path=/var/www/html/pub/fedora/linux/extras + +[Fedora Archive] +enabled=0 +path=/var/www/html/pub/fedora-archive diff --git a/user/manifests/init.pp b/user/manifests/init.pp index aefda07..9533dd9 100644 --- a/user/manifests/init.pp +++ b/user/manifests/init.pp @@ -290,6 +290,21 @@ class user::system { require => Group["murmur"], } + # Mirrors + @group { "mirror": + ensure => present, + gid => 820, + } + @user { "mirror": + ensure => present, + uid => 820, + gid => 820, + comment => "Service Mirror", + home => "/var/empty", + shell => "/sbin/nologin", + require => Group["mirror"], + } + }