rsync: First version of rsync backup jobs

rsync/files/rsync-backup-summary.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+RSYNC_LOGDIR="/srv/rsync-log"
+
+cd $RSYNC_LOGDIR || exit 1
+for host in `ls *.log | cut -d"-" -f1 | sort -u`; do
+    LOG=`ls $RSYNC_LOGDIR/$host-*.log | sort -nr | head -n1`
+    ls -l --time-style=full-iso $LOG
+    grep "^\*\*\*\*\*" $LOG
+    echo ""
+done

rsync/manifests/init.pp

@@ -24,3 +24,177 @@ class rsync::run-rsync {
         }
     }
 }
+
+
+# Set up environment to execute backup jobs
+#
+#   $rsync_logdir:
+#     (optional) Rsync job logs goes to this directory. Defaults to "/srv/rsync-log"
+#
+class rsync::backup {
+
+  package { "util-linux-ng": ensure => installed, } # Needed for /usr/bin/flock
+  
+  if $rsync_logdir {
+    file { $rsync_logdir:
+      ensure  => directory,
+      owner   => "root",
+      group   => "root",
+      mode    => 0700,
+    }
+    file { "/srv/rsync-log":
+      ensure  => link,
+      target  => $rsync_logdir,
+      require => File[$rsync_logdir],
+    }
+  } else {
+    file { "/srv/rsync-log":
+      ensure  => directory,
+      owner   => "root",
+      group   => "root",
+      mode    => 0700,
+    }
+  }
+
+  file { "/usr/local/cron":
+    ensure => directory,
+    owner  => "root",
+    group  => "root",
+    mode   => 0755,
+  }
+
+  file { "/usr/local/cron/rsync-backup-summary":
+    ensure  => present,
+    owner   => "root",
+    group   => "root",
+    mode    => 0755,
+    source  => "puppet:///modules/rsync/rsync-backup-summary.sh",
+    require => File["/usr/local/cron"],
+  }
+
+  cron { "rsync-backup-summary":
+    command => "/usr/local/cron/rsync-backup-summary",
+    user    => "root",
+    hour    => 06,
+    minute  => 00,
+    require => File["/usr/local/cron/rsync-backup-summary"],
+  }
+
+}
+
+
+# Define rsync backup job
+#
+#   $name:
+#     Cron job's name
+#
+#   $source_host:
+#     (optional) Hostname for source host to pull data from. If
+#     defined $target_host can't be defined.
+#
+#   $target_host:
+#     (optional) Hostname for target host to push data to. If defined
+#     $source_host can't be defined.
+#
+#   $source_dirs:
+#     Single "/path" or [ "/array", "/of", "/paths" ] of sources to
+#     copy.
+#
+#   $target_dir:
+#     Single "/path" for rsync destination
+#
+#   $hour:
+#     The hour at which to run the cron job. See Puppet's "cron" type.
+#
+#   $minute:
+#     (optional) The minute at which to run the cron job.
+#
+#   $excludes:
+#     (optional) String or Array of PATTERNS to exclude. See rsync
+#     manual for exclude PATTERNS.
+#
+#   $enable_acl:
+#     (optional) Enable rsync ACL. Defaults to enabled. Set to false
+#     to disable.
+#
+define rsync::backup_job (
+  $source_host="",
+  $target_host="",
+  $source_dirs,
+  $target_dir,
+  $excludes="",
+  $hour,
+  $minute=0,
+  $enable_acl=true,
+  ) {
+
+    include rsync
+    include rsync::backup
+
+    $_script = "/usr/local/cron/${name}"
+
+    if $source_host and $target_host {
+      fail("Both source_host and target_host can't be defined together.")
+    }
+
+    if $source_host and $source_dirs {
+      $_source_dirs = is_array($source_dirs) ? {
+        # See rsync man page's advanced usage section for this syntax
+        true  => inline_template("<%= @source_dirs.join(\" :\") -%>"),
+        false => $source_dirs,
+      }
+      $_source = "${source_host}:${_source_dirs}"
+    } elsif $source_dir {
+      $_source = is_array($source_dirs) ? {
+        true  => inline_template("<%= @source_dirs.join(\" \") -%>"),
+        false => $source_dirs,
+      }
+    } else {
+      fail("You need to specify source_dirs (and optionally source_host)")
+    }
+
+    if $target_host and $target_dir {
+      $_target = "${target_host}:${target_dir}"
+    } elsif $target_dir {
+      $_target = "${target_dir}"
+    } else {
+      fail("You need to specify target_dir (and optionally target_host)")
+    }
+
+    if is_array($excludes) {
+      $_excludes = inline_template('<%= @excludes.map {|x| "--exclude=\"" + x + "\""}.join(" ") -%>')
+    } elsif $excludes {
+      # We assume string here
+      $_excludes = '--exclude="$excludes"'
+    } else {
+      $_excludes = ""
+    }
+
+    if ! $target_host and $target_dir {
+      file { $target_dir:
+        ensure => directory,
+        owner  => "root",
+        group  => "root",
+        mode   => 0700,
+        before => File["/usr/local/cron/${name}"],
+      }
+    }
+
+    file { $_script:
+      ensure  => file,
+      content => template("rsync/rsync-backup.sh.erb"),
+      owner   => "root",
+      group   => "root",
+      mode    => 0744,
+      require => File["/srv/rsync-log"],
+    }
+
+    cron { "rsync-backup-${name}":
+      command => $_script,
+      user    => "root",
+      hour    => $hour,
+      minute  => $minute,
+      require => File[$_script],
+    }
+
+}

rsync/templates/rsync-backup.sh.erb

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+LOGFILE="/srv/rsync-log/<%= @source_host -%>-$(date +"%F-%T").log"
+
+FLOCK="/usr/bin/flock"
+FLOCK_OPTS="--exclusive --nonblock"
+FLOCK_FILE="/var/lock/<%= @name -%>"
+
+RSYNC="/usr/bin/rsync"
+RSYNC_OPTS="-ax <%= @acl ? "-A" : "" -%> --delete --itemize-changes <%= @_excludes -%>"
+
+(
+    $FLOCK $FLOCK_OPTS 200 || exit $?
+    /usr/bin/printf "***** <%= @_source -%> -> <%= @_target -%>\n"
+    /usr/bin/printf "***** Backup started %s\n\n" "$(date +"%F %T UTC")"
+    /usr/bin/time -p $RSYNC $RSYNC_OPTS <%= @_source -%> <%= @_target %>
+    RSYNCEXIT=$?
+    if [ $RSYNCEXIT -ne 0 ]; then
+	echo "ERROR: rsync job (<%= @name -%>)  process returned $RSYNCEXIT"
+    fi
+    /usr/bin/printf "\n***** Backup ended %s\n" "$(date +"%F %T UTC")"
+) 200>$FLOCK_FILE >$LOGFILE 2>&1
+
+FLOCKEXIT=$? # flock's return code
+
+# Look for unknown lines in log
+/bin/egrep -v "^$|([^ ]{11} )|([*]{5} )|(real|user|sys)" $LOGFILE >/dev/null 2>&1
+if [ $? -eq 0 ]; then
+    /bin/egrep -v "^([^ ]{11} )" $LOGFILE
+fi
+
+if [ $FLOCKEXIT -ne 0 ]; then
+    echo "ERROR: Rsync job (<%= @name -%>) already running or other error (exit code $FLOCKEXIT)"
+    exit $FLOCKEXIT
+fi