From 30d8c44c6f31d1d59af934a45b9a31f257f70287 Mon Sep 17 00:00:00 2001 From: Ossi Herrala Date: Wed, 14 Aug 2013 11:38:57 +0000 Subject: [PATCH] rsync: First version of rsync backup jobs --- rsync/files/rsync-backup-summary.sh | 11 ++ rsync/manifests/init.pp | 174 ++++++++++++++++++++++++++++ rsync/templates/rsync-backup.sh.erb | 35 ++++++ 3 files changed, 220 insertions(+) create mode 100644 rsync/files/rsync-backup-summary.sh create mode 100644 rsync/templates/rsync-backup.sh.erb diff --git a/rsync/files/rsync-backup-summary.sh b/rsync/files/rsync-backup-summary.sh new file mode 100644 index 0000000..4772a23 --- /dev/null +++ b/rsync/files/rsync-backup-summary.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +RSYNC_LOGDIR="/srv/rsync-log" + +cd $RSYNC_LOGDIR || exit 1 +for host in `ls *.log | cut -d"-" -f1 | sort -u`; do + LOG=`ls $RSYNC_LOGDIR/$host-*.log | sort -nr | head -n1` + ls -l --time-style=full-iso $LOG + grep "^\*\*\*\*\*" $LOG + echo "" +done diff --git a/rsync/manifests/init.pp b/rsync/manifests/init.pp index aab2626..0ee5090 100644 --- a/rsync/manifests/init.pp +++ b/rsync/manifests/init.pp @@ -24,3 +24,177 @@ class rsync::run-rsync { } } } + + +# Set up environment to execute backup jobs +# +# $rsync_logdir: +# (optional) Rsync job logs goes to this directory. Defaults to "/srv/rsync-log" +# +class rsync::backup { + + package { "util-linux-ng": ensure => installed, } # Needed for /usr/bin/flock + + if $rsync_logdir { + file { $rsync_logdir: + ensure => directory, + owner => "root", + group => "root", + mode => 0700, + } + file { "/srv/rsync-log": + ensure => link, + target => $rsync_logdir, + require => File[$rsync_logdir], + } + } else { + file { "/srv/rsync-log": + ensure => directory, + owner => "root", + group => "root", + mode => 0700, + } + } + + file { "/usr/local/cron": + ensure => directory, + owner => "root", + group => "root", + mode => 0755, + } + + file { "/usr/local/cron/rsync-backup-summary": + ensure => present, + owner => "root", + group => "root", + mode => 0755, + source => "puppet:///modules/rsync/rsync-backup-summary.sh", + require => File["/usr/local/cron"], + } + + cron { "rsync-backup-summary": + command => "/usr/local/cron/rsync-backup-summary", + user => "root", + hour => 06, + minute => 00, + require => File["/usr/local/cron/rsync-backup-summary"], + } + +} + + +# Define rsync backup job +# +# $name: +# Cron job's name +# +# $source_host: +# (optional) Hostname for source host to pull data from. If +# defined $target_host can't be defined. +# +# $target_host: +# (optional) Hostname for target host to push data to. If defined +# $source_host can't be defined. +# +# $source_dirs: +# Single "/path" or [ "/array", "/of", "/paths" ] of sources to +# copy. +# +# $target_dir: +# Single "/path" for rsync destination +# +# $hour: +# The hour at which to run the cron job. See Puppet's "cron" type. +# +# $minute: +# (optional) The minute at which to run the cron job. +# +# $excludes: +# (optional) String or Array of PATTERNS to exclude. See rsync +# manual for exclude PATTERNS. +# +# $enable_acl: +# (optional) Enable rsync ACL. Defaults to enabled. Set to false +# to disable. +# +define rsync::backup_job ( + $source_host="", + $target_host="", + $source_dirs, + $target_dir, + $excludes="", + $hour, + $minute=0, + $enable_acl=true, + ) { + + include rsync + include rsync::backup + + $_script = "/usr/local/cron/${name}" + + if $source_host and $target_host { + fail("Both source_host and target_host can't be defined together.") + } + + if $source_host and $source_dirs { + $_source_dirs = is_array($source_dirs) ? { + # See rsync man page's advanced usage section for this syntax + true => inline_template("<%= @source_dirs.join(\" :\") -%>"), + false => $source_dirs, + } + $_source = "${source_host}:${_source_dirs}" + } elsif $source_dir { + $_source = is_array($source_dirs) ? { + true => inline_template("<%= @source_dirs.join(\" \") -%>"), + false => $source_dirs, + } + } else { + fail("You need to specify source_dirs (and optionally source_host)") + } + + if $target_host and $target_dir { + $_target = "${target_host}:${target_dir}" + } elsif $target_dir { + $_target = "${target_dir}" + } else { + fail("You need to specify target_dir (and optionally target_host)") + } + + if is_array($excludes) { + $_excludes = inline_template('<%= @excludes.map {|x| "--exclude=\"" + x + "\""}.join(" ") -%>') + } elsif $excludes { + # We assume string here + $_excludes = '--exclude="$excludes"' + } else { + $_excludes = "" + } + + if ! $target_host and $target_dir { + file { $target_dir: + ensure => directory, + owner => "root", + group => "root", + mode => 0700, + before => File["/usr/local/cron/${name}"], + } + } + + file { $_script: + ensure => file, + content => template("rsync/rsync-backup.sh.erb"), + owner => "root", + group => "root", + mode => 0744, + require => File["/srv/rsync-log"], + } + + cron { "rsync-backup-${name}": + command => $_script, + user => "root", + hour => $hour, + minute => $minute, + require => File[$_script], + } + +} diff --git a/rsync/templates/rsync-backup.sh.erb b/rsync/templates/rsync-backup.sh.erb new file mode 100644 index 0000000..c7e2b4a --- /dev/null +++ b/rsync/templates/rsync-backup.sh.erb @@ -0,0 +1,35 @@ +#!/bin/bash + +LOGFILE="/srv/rsync-log/<%= @source_host -%>-$(date +"%F-%T").log" + +FLOCK="/usr/bin/flock" +FLOCK_OPTS="--exclusive --nonblock" +FLOCK_FILE="/var/lock/<%= @name -%>" + +RSYNC="/usr/bin/rsync" +RSYNC_OPTS="-ax <%= @acl ? "-A" : "" -%> --delete --itemize-changes <%= @_excludes -%>" + +( + $FLOCK $FLOCK_OPTS 200 || exit $? + /usr/bin/printf "***** <%= @_source -%> -> <%= @_target -%>\n" + /usr/bin/printf "***** Backup started %s\n\n" "$(date +"%F %T UTC")" + /usr/bin/time -p $RSYNC $RSYNC_OPTS <%= @_source -%> <%= @_target %> + RSYNCEXIT=$? + if [ $RSYNCEXIT -ne 0 ]; then + echo "ERROR: rsync job (<%= @name -%>) process returned $RSYNCEXIT" + fi + /usr/bin/printf "\n***** Backup ended %s\n" "$(date +"%F %T UTC")" +) 200>$FLOCK_FILE >$LOGFILE 2>&1 + +FLOCKEXIT=$? # flock's return code + +# Look for unknown lines in log +/bin/egrep -v "^$|([^ ]{11} )|([*]{5} )|(real|user|sys)" $LOGFILE >/dev/null 2>&1 +if [ $? -eq 0 ]; then + /bin/egrep -v "^([^ ]{11} )" $LOGFILE +fi + +if [ $FLOCKEXIT -ne 0 ]; then + echo "ERROR: Rsync job (<%= @name -%>) already running or other error (exit code $FLOCKEXIT)" + exit $FLOCKEXIT +fi