Logic implmeented to delete old sync files as well as only run on secondary if primary is not available (this avoids overwriting running configs)

failover/Makefile

@@ -5,3 +5,6 @@ test:
 
 go:
 	./failover.sh -P primary.localdomain -S secondary.localdomain -E
+
+copy:
+	rsync -avu /home/nagios/princeton/failover/ /home/nagios/bin/

failover/colors.sh

@@ -1,11 +1,11 @@
 verbose () {
-  echo "${color_green}$1${color_normal}"
+  echo $2 "${color_green}$1${color_normal}"
 }
 error () {
-  echo "${color_red}$1${color_normal}"
+  echo $2 "${color_red}$1${color_normal}"
         }
 warning () {
-  echo "${color_yellow}$1${color_normal}"
+  echo $2 "${color_yellow}$1${color_normal}"
 }
 # check for color support
 if test -t 1; then

failover/failover.sh

@@ -10,6 +10,9 @@ PATH=/usr/sbin:/sbin:/usr/bin:/bin
 
 #primary="nagiosxi-p.princeton.edu"
 #secondary="nagiosxi-f.princeton.edu"
+backupDir="/store/backups/nagiosxi"
+force=""
+noping="false"
 primary=""
 secondary=""
 synconly="false"
@@ -27,12 +30,19 @@ while [ -n "$1" ]; do
     -P | --primary) primary="$2"; shift 2;;
     -n | --sync-name) syncname="$2"; shift 2;;
     -s | --sync-only) synconly="true"; shift 1;;
+    --force) force="true"; shift 1;;
+    --noping) noping="true"; shift 1;;
     *) extraCmd="$extraCmd $1"; shift 1;;
   esac
 done
 
 [ -z "$hostname" ] && warning "Invalid hostname." && exit 2
 
+ping_host() {
+  $noping && return 0
+  /usr/local/nagios/libexec/check_icmp -H $1 >/dev/null 2>&1
+}
+
 stop_nagios() {
     verbose "Ensuring Nagios services are stopped because we are the secondary"
     sleep 2
@@ -45,13 +55,23 @@ start_nagios() {
     /home/nagios/bin/nagios_startstop.sh start
 }
 
+clean_old_copies() {
+  verbose "Cleaning up old sync files..." -n
+  find ${backupDir} -maxdepth 1 -name sync.\* -mtime +1 -exec rm {} \;
+
+  verbose "Done"
+}
+
 do_nothing() {
   warning "We are not the primary or secondary.  Doing nothing."
   exit 1
 }
 
 do_primary() {
-  verbose "We are the primary.  Creating package to send to secondary."
+  verbose "We are the primary.  Checking to see if secondary is on the network..."
+  ping_host ${secondary}
+  [ $? -ne 0 ] && error "Cannot ping ${secondary}.  Use --noping to continue wihout ping." && exit
+  verbose "Creating package to send to secondary"
   /home/nagios/bin/rsync_xi.sh --prepend ${syncname} --primary "${primary}" --secondary "${secondary}" $extraCmd
   if [ -r "/usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css.primary" -a -w "/usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css" ]; then
     verbose "Copying local gradient files"
@@ -60,8 +80,14 @@ do_primary() {
 }
 
 do_secondary() {
-  verbose "We are the secondary.  Expanding package from the primary."
+  verbose "We are the secondary.  Checking to see if primary is on the network..."
-  syncfile=$(basename `ls -tr1 /store/backups/nagiosxi/${syncname}* | tail -1`)
+  ping_host ${primary}
+  if [ $? -eq 0 -a -z "$force"  ]; then
+    warning "The primary (${primary}) is on the network.  This process will be aborted.  Use --force to force it to continue."
+    exit 1
+  fi
+  verbose "Expanding package from the primary"
+  syncfile=$(basename `ls -tr1 ${backupDir}/${syncname}* | tail -1`)
   [ -z "$syncfile" ] && error "No sync file ($syncfile) found." && exit 2
   /home/nagios/bin/rsync_xi.sh --primary "${primary}" --secondary "${secondary}" --file "/store/backups/nagiosxi/$syncfile"
   if [ $? -eq 0 ]; then
@@ -78,6 +104,7 @@ case "$hostname" in
   $secondary) do_secondary;;
   *) do_nothing
 esac
+clean_old_copies
 
 exit 0