Eric Loyd
4 years ago
commit
31e98f92f7
@ -0,0 +1,55 @@
|
|||||||
|
# Failover for Princeton University
|
||||||
|
Failover from primary Nagios XI to failover Nagios XI is a Disaster Recovery effort aimed at providing a near-time up-to-date failover Nagios server that can take over monitoring and notification options should the primary be unavailable. The primary will always monitor if it is capable. The failover will only monitor when manually enabled and must also be manually disabled when the primary returns to active service.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
* Nagios XI must be installed on both boxes with the same version and underlying directory configuration. If there are any differences in file locations or major configuration between the two boxes, the Nagios failover will have unpredictable results, including complete system failure.
|
||||||
|
* The syncing process will delete files that it does not believe should be on the failover, so all work must be performed on the primary. Any work performed on the failover will be overwritten when the next synchronization process occurs. Note that this includes SSH keys, as /home/nagios will be synced from the parimary to the failover.
|
||||||
|
* /home/nagios/bin exists and contains the files needed for this process. Note that the sync process will sync these from the primary to the failover, so like all other files, they must only be modified on the primary.
|
||||||
|
|
||||||
|
/home/nagios/bin/failover.sh
|
||||||
|
/home/nagios/bin/nagios_startstop.sh
|
||||||
|
/home/nagios/bin/rsync_xi.sh
|
||||||
|
|
||||||
|
* The root user has the ability to SSH from the primary to the failover as the nagios user without entering a passphrase. This is how the rsync and database copies are performed
|
||||||
|
* root on primary (and root on failover) has crontab requirements that will be detailed separately.
|
||||||
|
* nagios on both primary and failover needs to be able ot sudo to root without a password and execute the rsync command:
|
||||||
|
|
||||||
|
NAGIOSXI ALL = NOPASSWD:/usr/bin/rsync *
|
||||||
|
|
||||||
|
* nagios user on primary needs to be able to SSH to nagios on failover without a passphrase (thus, an SSH key and .ssh directory needs to be set up)
|
||||||
|
|
||||||
|
* Make sure any ramdisk (such as /ramdisk) is copied if it exists
|
||||||
|
|
||||||
|
* Note that any gearman addons such as /etc/mod_gearman or whatever are NOT copied as part of this procedure. These types of things need to be set up on both boxes the same way before this process is set up.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
This assumes that Nagios XI is installed and that its base location is /usr/local/nagiosxi. It also assumes that there is a nagios user home directory at /home/nagios. If either of this locations is incorrect, the following commands will need to be updated accordingly.
|
||||||
|
|
||||||
|
* mkdir /home/nagios/bin
|
||||||
|
* cp * /home/nagios/bin
|
||||||
|
* cp -a local /home/nagios/bin
|
||||||
|
|
||||||
|
The first time the sync runs, it will take some time. It has to fully populate the failover Nagios server. It is recommended to run the sync once, manually, by executing the following as the nagios user on the primary Nagios server:
|
||||||
|
|
||||||
|
* /home/nagios/bin/SYNC
|
||||||
|
|
||||||
|
From here on, it can be run automatically via cron at a regular schedule. We recommend at least 30 minutes between successive runs.
|
||||||
|
|
||||||
|
### Cron
|
||||||
|
The following line should be added to cron on the primary, once the initial copy has been performed (this will run every 30 minutes at :00 and :30, for instance):
|
||||||
|
|
||||||
|
* */30 * * * * /home/nagios/bin/SYNC >> /home/nagios/sync.log
|
||||||
|
|
||||||
|
Note that this will create and append the output to /home/nagios/sync.log for future reference. This file can be deleted at any time or added to a logrotate script for management.
|
||||||
|
|
||||||
|
# Failover
|
||||||
|
Normally, the primary will update the failover on a regular schedule (defined in cron) as long as the primary is running and can reach the failover. When the time comes to activate Nagios on the failover box, the following command should be executed as root on the failover Nagios server:
|
||||||
|
|
||||||
|
* /home/nagios/bin/START
|
||||||
|
|
||||||
|
When the primary has returned to service, Nagios should be stopped on the failover box by typing the following command as root on the failover Nagios server:
|
||||||
|
|
||||||
|
* /home/nagios/bin/STOP
|
||||||
|
|
||||||
|
## Local directory
|
||||||
|
Anything in /home/nagios/bin/local will be copied (including the directory structure) to the local Nagios server as part of the syncronization. This is generally for things like custom CSS files. The failover code handles installing those files into their proper locations, if required, and will need to be updated manually if there are files other than the "color bar" that needs to be updated in the future.
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
/home/nagios/bin/failover.sh -P epm202l.princeton.edu -S epm100l.Princeton.EDU -E --force
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
/home/nagios/bin/failover.sh -P epm202l.princeton.edu -S epm100l.Princeton.EDU -E
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
verbose () {
|
||||||
|
echo $2 "${color_green}$1${color_normal}"
|
||||||
|
}
|
||||||
|
error () {
|
||||||
|
echo $2 "${color_red}$1${color_normal}"
|
||||||
|
}
|
||||||
|
warning () {
|
||||||
|
echo $2 "${color_yellow}$1${color_normal}"
|
||||||
|
}
|
||||||
|
# check for color support
|
||||||
|
if test -t 1; then
|
||||||
|
# see if it supports colors...
|
||||||
|
ncolors=$(tput colors)
|
||||||
|
if test -n "$ncolors" && test $ncolors -ge 8; then
|
||||||
|
color_normal="$(tput sgr0)"
|
||||||
|
color_red="$(tput setaf 1)"
|
||||||
|
color_green="$(tput setaf 2)"
|
||||||
|
color_yellow="$(tput setaf 3)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
@ -0,0 +1,113 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
#
|
||||||
|
# Nagios XI failover for Princeton University
|
||||||
|
#
|
||||||
|
# Initial version written by:
|
||||||
|
# Eric Loyd (eloyd@everwatch.global)
|
||||||
|
# EverWatch Global, Inc.
|
||||||
|
|
||||||
|
PATH=/usr/sbin:/sbin:/usr/bin:/bin
|
||||||
|
|
||||||
|
#primary="nagiosxi-p.princeton.edu"
|
||||||
|
#secondary="nagiosxi-f.princeton.edu"
|
||||||
|
backupDir="/store/backups/nagiosxi"
|
||||||
|
force=""
|
||||||
|
noping="false"
|
||||||
|
primary=""
|
||||||
|
secondary=""
|
||||||
|
synconly="false"
|
||||||
|
syncname="sync"
|
||||||
|
extraCmd=""
|
||||||
|
|
||||||
|
hostname=`/bin/hostname`
|
||||||
|
|
||||||
|
. ./colors.sh
|
||||||
|
|
||||||
|
while [ -n "$1" ]; do
|
||||||
|
case "$1" in
|
||||||
|
-H | --hostname) hostname="$2"; shift 2;;
|
||||||
|
-S | --secondary) secondary="$2"; shift 2;;
|
||||||
|
-P | --primary) primary="$2"; shift 2;;
|
||||||
|
-n | --sync-name) syncname="$2"; shift 2;;
|
||||||
|
-s | --sync-only) synconly="true"; shift 1;;
|
||||||
|
--force) force="true"; shift 1;;
|
||||||
|
--noping) noping="true"; shift 1;;
|
||||||
|
*) extraCmd="$extraCmd $1"; shift 1;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -z "$hostname" ] && warning "Invalid hostname." && exit 2
|
||||||
|
|
||||||
|
ping_host() {
|
||||||
|
$noping && return 0
|
||||||
|
/usr/local/nagios/libexec/check_icmp -H $1 >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_nagios() {
|
||||||
|
/home/nagios/bin/nagios_startstop.sh stop
|
||||||
|
}
|
||||||
|
|
||||||
|
start_nagios() {
|
||||||
|
/home/nagios/bin/nagios_startstop.sh start
|
||||||
|
}
|
||||||
|
|
||||||
|
clean_old_copies() {
|
||||||
|
verbose "Cleaning up old sync files..." -n
|
||||||
|
find ${backupDir} -maxdepth 1 -name sync.\* -mtime +1 -exec rm {} \;
|
||||||
|
|
||||||
|
verbose "Done"
|
||||||
|
}
|
||||||
|
|
||||||
|
do_nothing() {
|
||||||
|
warning "We are not the primary or secondary. Doing nothing."
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
do_primary() {
|
||||||
|
verbose "We are the primary. Checking to see if secondary is on the network..."
|
||||||
|
ping_host ${secondary}
|
||||||
|
[ $? -ne 0 ] && error "Cannot ping ${secondary}. Use --noping to continue wihout ping." && exit
|
||||||
|
verbose "Creating package to send to secondary"
|
||||||
|
/home/nagios/bin/rsync_xi.sh --prepend ${syncname} --primary "${primary}" --secondary "${secondary}" $extraCmd
|
||||||
|
if [ -r "/usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css.primary" -a -w "/usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css" ]; then
|
||||||
|
verbose "Copying local gradient files"
|
||||||
|
cp /usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css.primary /usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
do_secondary() {
|
||||||
|
verbose "We are the secondary. Checking to see if primary is on the network..."
|
||||||
|
ping_host ${primary}
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
if [ -z $force ]; then
|
||||||
|
warning "The primary (${primary}) is on the network. This process will be aborted. Use --force to force it to continue."
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
warning "The primary (${primary}) is on the network but --force specified. Proceeding anyway."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
verbose "Expanding package from the primary"
|
||||||
|
syncfile=$(basename `ls -tr1 ${backupDir}/${syncname}* | tail -1`)
|
||||||
|
[ -z "$syncfile" ] && error "No sync file ($syncfile) found." && exit 2
|
||||||
|
/home/nagios/bin/rsync_xi.sh --primary "${primary}" --secondary "${secondary}" --file "/store/backups/nagiosxi/$syncfile"
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
sleep 2
|
||||||
|
if [ -r "/usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css.secondary" -a -w "/usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css" ]; then
|
||||||
|
cp /usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css.secondary /usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css
|
||||||
|
fi
|
||||||
|
verbose "This is intended to be manually executed when needed, so we assume you want to start Nagios..." -n
|
||||||
|
start_nagios
|
||||||
|
rm -f ${syncfile}
|
||||||
|
verbose "Done."
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
case "$hostname" in
|
||||||
|
$primary) do_primary;;
|
||||||
|
$secondary) do_secondary;;
|
||||||
|
*) do_nothing
|
||||||
|
esac
|
||||||
|
clean_old_copies
|
||||||
|
|
||||||
|
exit 0
|
||||||
|
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
#header {
|
||||||
|
background: rgb(59,234,2);
|
||||||
|
background: -moz-linear-gradient(90deg, rgba(59,234,2,1) 0%, rgba(8,124,8,1) 35%);
|
||||||
|
background: -webkit-linear-gradient(90deg, rgba(59,234,2,1) 0%, rgba(8,124,8,1) 35%);
|
||||||
|
background: linear-gradient(90deg, rgba(59,234,2,1) 0%, rgba(8,124,8,1) 35%);
|
||||||
|
filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#3bea02",endColorstr="#087c08",GradientType=1);
|
||||||
|
}
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
#header {
|
||||||
|
background: rgb(0,0,255);
|
||||||
|
background: -moz-linear-gradient(90deg, rgba(0,0,255,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
background: -webkit-linear-gradient(90deg, rgba(0,0,255,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
background: linear-gradient(90deg, rgba(0,0,255,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#0000ff",endColorstr="#000000",GradientType=1);
|
||||||
|
}
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
#header {
|
||||||
|
background: rgb(0,255,0);
|
||||||
|
background: -moz-linear-gradient(90deg, rgba(0,255,0,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
background: -webkit-linear-gradient(90deg, rgba(0,255,0,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
background: linear-gradient(90deg, rgba(0,255,0,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#00ff00",endColorstr="#000000",GradientType=1);
|
||||||
|
}
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
;#header {
|
||||||
|
; background: #3cb371;
|
||||||
|
;}
|
||||||
|
|
||||||
|
#header {
|
||||||
|
background: #c71585;
|
||||||
|
}
|
||||||
|
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
/* Primary CSS */
|
||||||
|
#header {
|
||||||
|
background: rgb(0,255,0);
|
||||||
|
background: -moz-linear-gradient(90deg, rgba(0,255,0,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
background: -webkit-linear-gradient(90deg, rgba(0,255,0,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
background: linear-gradient(90deg, rgba(0,255,0,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#00ff00",endColorstr="#000000",GradientType=1);
|
||||||
|
}
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
/* Secondary CSS */
|
||||||
|
#header {
|
||||||
|
background: rgb(0,0,255);
|
||||||
|
background: -moz-linear-gradient(90deg, rgba(0,0,255,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
background: -webkit-linear-gradient(90deg, rgba(0,0,255,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
background: linear-gradient(90deg, rgba(0,0,255,1) 0%, rgba(0,0,0,1) 50%);
|
||||||
|
filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#0000ff",endColorstr="#000000",GradientType=1);
|
||||||
|
}
|
||||||
@ -0,0 +1,50 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# Set a default path
|
||||||
|
PATH="/usr/bin:/bin"
|
||||||
|
|
||||||
|
# This is where we write to Nagios command pipe
|
||||||
|
# Make sure it's writeable (follow Nagios plugin exit code API)
|
||||||
|
commandfile="/usr/local/nagios/var/rw/nagios.cmd"
|
||||||
|
if [ ! -w "$commandfile" ]; then
|
||||||
|
echo "ERROR: Cannot write to $commandfile"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Send to Nagios
|
||||||
|
send_nagios() {
|
||||||
|
now=`date +%s`
|
||||||
|
/bin/printf "[%lu] $*\n" $now > $commandfile
|
||||||
|
}
|
||||||
|
|
||||||
|
# Stop executing service checks
|
||||||
|
stop_engine() {
|
||||||
|
send_nagios "STOP_EXECUTING_SVC_CHECKS"
|
||||||
|
send_nagios "STOP_EXECUTING_HOST_CHECKS"
|
||||||
|
send_nagios "STOP_ACCEPTING_PASSIVE_SVC_CHECKS"
|
||||||
|
send_nagios "STOP_ACCEPTING_PASSIVE_HOST_CHECKS"
|
||||||
|
send_nagios "DISABLE_EVENT_HANDLERS"
|
||||||
|
send_nagios "DISABLE_NOTIFICATIONS"
|
||||||
|
send_nagios "DISABLE_FLAP_DETECTION"
|
||||||
|
send_nagios "DISABLE_PERFORMANCE_DATA"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Start the monitoring engine
|
||||||
|
start_engine() {
|
||||||
|
send_nagios "START_ACCEPTING_PASSIVE_HOST_CHECKS"
|
||||||
|
send_nagios "START_ACCEPTING_PASSIVE_SVC_CHECKS"
|
||||||
|
send_nagios "START_EXECUTING_HOST_CHECKS"
|
||||||
|
send_nagios "START_EXECUTING_SVC_CHECKS"
|
||||||
|
send_nagios "ENABLE_EVENT_HANDLERS"
|
||||||
|
send_nagios "ENABLE_NOTIFICATIONS"
|
||||||
|
send_nagios "ENABLE_FLAP_DETECTION"
|
||||||
|
send_nagios "ENABLE_PERFORMANCE_DATA"
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ -n "$1" ]; do
|
||||||
|
case "$1" in
|
||||||
|
start) start_engine; shift 1;;
|
||||||
|
stop) stop_engine; shift 1;;
|
||||||
|
*) shift 1;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
@ -0,0 +1,515 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Creates a Full Backup of Nagios XI
|
||||||
|
# Copyright (c) 2011-2020 Nagios Enterprises, LLC. All rights reserved.
|
||||||
|
#
|
||||||
|
|
||||||
|
PATH=/home/nagios/bin:/usr/bin:/bin
|
||||||
|
. /home/nagios/bin/colors.sh
|
||||||
|
|
||||||
|
BASEDIR="/usr/local/nagiosxi/scripts"
|
||||||
|
|
||||||
|
timeStamp=`date +%s`
|
||||||
|
hostname="`/bin/hostname`"
|
||||||
|
pName=""
|
||||||
|
sName=""
|
||||||
|
noping=/bin/false
|
||||||
|
execute="-n"
|
||||||
|
syncfile=""
|
||||||
|
isP=/bin/false
|
||||||
|
isS=/bin/false
|
||||||
|
|
||||||
|
# Import Nagios XI and xi-sys.cfg config vars
|
||||||
|
. $BASEDIR/../var/xi-sys.cfg
|
||||||
|
eval $(php $BASEDIR/import_xiconfig.php)
|
||||||
|
|
||||||
|
###############################
|
||||||
|
# USAGE / HELP
|
||||||
|
###############################
|
||||||
|
usage () {
|
||||||
|
echo ""
|
||||||
|
echo "Use this script to synchronize primary XI to secondary, or to expand primary information on secondary"
|
||||||
|
echo ""
|
||||||
|
echo " -P | --primary Set the FQDN of the primary (what is returned by /bin/hostname)"
|
||||||
|
echo " -S | --secondary Set the FQDN of the secondary (what is returned by /bin/hostname)"
|
||||||
|
echo " -F | --file Set the name of the file that we will be restoring from on the secondary"
|
||||||
|
echo " -E | --execute Perform the sync (otherwise, it just does a dry-run)"
|
||||||
|
echo " -n | --name Set the name of the backup minus the .tar.gz"
|
||||||
|
echo " -p | --prepend Prepend a string to the .tar.gz name"
|
||||||
|
echo " -a | --append Append a string to the .tar.gz name"
|
||||||
|
echo " -d | --directory Change the directory to store the compressed backup"
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
###############################
|
||||||
|
# ADDING LOGIC FOR NEW BACKUPS
|
||||||
|
###############################
|
||||||
|
while [ -n "$1" ]; do
|
||||||
|
case "$1" in
|
||||||
|
-P | --primary)
|
||||||
|
pName="$2"
|
||||||
|
;;
|
||||||
|
-S | --secondary)
|
||||||
|
sName="$2"
|
||||||
|
;;
|
||||||
|
-E | --execute)
|
||||||
|
execute=""
|
||||||
|
;;
|
||||||
|
-F | --file)
|
||||||
|
syncfile="$2"
|
||||||
|
;;
|
||||||
|
--noping)
|
||||||
|
noping=/bin/true
|
||||||
|
;;
|
||||||
|
-h | --help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
-n | --name)
|
||||||
|
fullname=$2
|
||||||
|
;;
|
||||||
|
-p | --prepend)
|
||||||
|
prepend=$2"."
|
||||||
|
;;
|
||||||
|
-a | --append)
|
||||||
|
append="."$2
|
||||||
|
;;
|
||||||
|
-d | --directory)
|
||||||
|
rootdir=$2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
do_primary() {
|
||||||
|
#ping_host $sName
|
||||||
|
#[ $? -ne 0 ] && echo "Cannot ping $sName. Use --noping to continue wihout ping." && exit
|
||||||
|
echo "Starting sync from $pName to $sName"
|
||||||
|
|
||||||
|
# Restart nagios to forcibly update retention.dat
|
||||||
|
verbose "Restarting nagios to forcibly update retention.dat"
|
||||||
|
sudo $BASEDIR/manage_services.sh stop nagios
|
||||||
|
sleep 5
|
||||||
|
sudo $BASEDIR/manage_services.sh start nagios
|
||||||
|
|
||||||
|
if [ -z $rootdir ]; then
|
||||||
|
rootdir="/store/backups/nagiosxi"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Move to root dir to store backups
|
||||||
|
cd $rootdir
|
||||||
|
|
||||||
|
#############################
|
||||||
|
# SET THE NAME & TIME
|
||||||
|
#############################
|
||||||
|
name=$fullname
|
||||||
|
if [ -z $fullname ]; then
|
||||||
|
name=$prepend$timeStamp$append
|
||||||
|
fi
|
||||||
|
# Get current Unix timestamp as name
|
||||||
|
if [ -z $name ]; then
|
||||||
|
name=timeStampts
|
||||||
|
fi
|
||||||
|
|
||||||
|
# My working directory
|
||||||
|
workDir=$rootdir/$name
|
||||||
|
# Make directory for this specific backup
|
||||||
|
mkdir -p $workDir
|
||||||
|
|
||||||
|
do_rsync() {
|
||||||
|
verbose "Syncing $*..."
|
||||||
|
src="$1"
|
||||||
|
[ ! -e "$src" ] && warning " OK: No such file or directory: $src" && return
|
||||||
|
[ -d "$src" ] && sudo rsync -a --delete --delete-after --rsync-path="sudo /bin/rsync" ${execute} ${src}/ nagios@${sName}:${src}/
|
||||||
|
[ -f "$src" ] && sudo rsync -a --delete --delete-after --rsync-path="sudo /bin/rsync" ${execute} ${src} nagios@${sName}:${src}
|
||||||
|
}
|
||||||
|
|
||||||
|
do_backup_files() {
|
||||||
|
##############################
|
||||||
|
# BACKUP DIRS
|
||||||
|
##############################
|
||||||
|
|
||||||
|
# Only backup NagiosQL if it exists
|
||||||
|
if [ -d "/var/www/html/nagiosql" ]; then
|
||||||
|
do_rsync /var/www/html/nagiosql
|
||||||
|
do_rsync /etc/nagiosql
|
||||||
|
fi
|
||||||
|
|
||||||
|
do_rsync /etc/apache2/sites-available/default-ssl.conf
|
||||||
|
do_rsync /etc/logrotate.d/nagiosxi
|
||||||
|
do_rsync /etc/mrtg/conf.d
|
||||||
|
do_rsync /etc/mrtg/mrtg.cfg
|
||||||
|
do_rsync /etc/snmp
|
||||||
|
do_rsync /etc/sysconfig/nagios
|
||||||
|
do_rsync /home/nagios
|
||||||
|
do_rsync $httpdconfdir/nagios.conf
|
||||||
|
do_rsync $httpdconfdir/nagiosmobile.conf
|
||||||
|
do_rsync $httpdconfdir/nagiosxi.conf
|
||||||
|
do_rsync $httpdconfdir/nagvis.conf
|
||||||
|
do_rsync $httpdconfdir/nrdp.conf
|
||||||
|
do_rsync $httpdconfdir/ssl.conf
|
||||||
|
do_rsync /usr/local/nagios
|
||||||
|
do_rsync /usr/local/nagiosmobile
|
||||||
|
do_rsync /usr/local/nagiosxi
|
||||||
|
do_rsync /usr/local/nagvis
|
||||||
|
do_rsync /usr/local/nrdp
|
||||||
|
do_rsync /usr/share/snmp
|
||||||
|
do_rsync /var/lib/mrtg
|
||||||
|
do_rsync /var/spool/cron/apache
|
||||||
|
do_rsync /var/spool/cron/crontabs/$apacheuser
|
||||||
|
|
||||||
|
|
||||||
|
} # End of do_backup_files
|
||||||
|
|
||||||
|
do_backup_sql() {
|
||||||
|
##############################
|
||||||
|
# BACKUP DATABASES
|
||||||
|
##############################
|
||||||
|
echo "Backing up MySQL databases..."
|
||||||
|
mkdir -p $workDir/mysql
|
||||||
|
if [[ "$cfg__db_info__ndoutils__dbserver" == *":"* ]]; then
|
||||||
|
ndoutils_dbport=`echo "$cfg__db_info__ndoutils__dbserver" | cut -f2 -d":"`
|
||||||
|
ndoutils_dbserver=`echo "$cfg__db_info__ndoutils__dbserver" | cut -f1 -d":"`
|
||||||
|
else
|
||||||
|
ndoutils_dbport='3306'
|
||||||
|
ndoutils_dbserver="$cfg__db_info__ndoutils__dbserver"
|
||||||
|
fi
|
||||||
|
mysqldump -h "$ndoutils_dbserver" --port="$ndoutils_dbport" -u $cfg__db_info__ndoutils__user --password="$cfg__db_info__ndoutils__pwd" --add-drop-database -B $cfg__db_info__ndoutils__db > $workDir/mysql/nagios.sql
|
||||||
|
res=$?
|
||||||
|
if [ $res != 0 ]; then
|
||||||
|
echo "Error backing up MySQL database 'nagios' - check the password in this script!"
|
||||||
|
rm -r $workDir
|
||||||
|
exit $res;
|
||||||
|
fi
|
||||||
|
if [[ "$cfg__db_info__nagiosql__dbserver" == *":"* ]]; then
|
||||||
|
nagiosql_dbport=`echo "$cfg__db_info__nagiosql__dbserver" | cut -f2 -d":"`
|
||||||
|
nagiosql_dbserver=`echo "$cfg__db_info__nagiosql__dbserver" | cut -f1 -d":"`
|
||||||
|
else
|
||||||
|
nagiosql_dbport='3306'
|
||||||
|
nagiosql_dbserver="$cfg__db_info__nagiosql__dbserver"
|
||||||
|
fi
|
||||||
|
mysqldump -h "$nagiosql_dbserver" --port="$nagiosql_dbport" -u $cfg__db_info__nagiosql__user --password="$cfg__db_info__nagiosql__pwd" --add-drop-database -B $cfg__db_info__nagiosql__db > $workDir/mysql/nagiosql.sql
|
||||||
|
res=$?
|
||||||
|
if [ $res != 0 ]; then
|
||||||
|
echo "Error backing up MySQL database 'nagiosql' - check the password in this script!"
|
||||||
|
rm -r $workDir
|
||||||
|
exit $res;
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Only backup PostgresQL if we are still using it
|
||||||
|
if [ $cfg__db_info__nagiosxi__dbtype == "pgsql" ]; then
|
||||||
|
echo "Backing up PostgresQL databases..."
|
||||||
|
mkdir -p $workDir/pgsql
|
||||||
|
if [ -z $cfg__db_info__nagiosxi__dbserver ]; then
|
||||||
|
cfg__db_info__nagiosxi__dbserver="localhost"
|
||||||
|
fi
|
||||||
|
pg_dump -h $cfg__db_info__nagiosxi__dbserver -c -U $cfg__db_info__nagiosxi__user $cfg__db_info__nagiosxi__db > $workDir/pgsql/nagiosxi.sql
|
||||||
|
res=$?
|
||||||
|
if [ $res != 0 ]; then
|
||||||
|
echo "Error backing up PostgresQL database 'nagiosxi' !"
|
||||||
|
rm -r $workDir
|
||||||
|
exit $res;
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if [[ "$cfg__db_info__nagiosxi__dbserver" == *":"* ]]; then
|
||||||
|
nagiosxi_dbport=`echo "$cfg__db_info__nagiosxi__dbserver" | cut -f2 -d":"`
|
||||||
|
nagiosxi_dbserver=`echo "$cfg__db_info__nagiosxi__dbserver" | cut -f1 -d":"`
|
||||||
|
else
|
||||||
|
nagiosxi_dbport='3306'
|
||||||
|
nagiosxi_dbserver="$cfg__db_info__nagiosxi__dbserver"
|
||||||
|
fi
|
||||||
|
mysqldump -h "$nagiosxi_dbserver" --port="$nagiosxi_dbport" -u $cfg__db_info__nagiosxi__user --password="$cfg__db_info__nagiosxi__pwd" --add-drop-database -B $cfg__db_info__nagiosxi__db > $workDir/mysql/nagiosxi.sql
|
||||||
|
res=$?
|
||||||
|
if [ $res != 0 ]; then
|
||||||
|
echo "Error backing up MySQL database 'nagiosxi' - check the password in this script!"
|
||||||
|
rm -r $workDir
|
||||||
|
exit $res;
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
##############################
|
||||||
|
# COMPRESS BACKUP
|
||||||
|
##############################
|
||||||
|
verbose "Compressing database file..." -n
|
||||||
|
tar czfp $name.tar.gz $name
|
||||||
|
rm -rf $name
|
||||||
|
verbose "Done."
|
||||||
|
|
||||||
|
# Change ownership
|
||||||
|
chown $nagiosuser:$nagiosgroup $name.tar.gz
|
||||||
|
|
||||||
|
if [ -s $name.tar.gz ];then
|
||||||
|
echo " "
|
||||||
|
echo "==============="
|
||||||
|
echo "BACKUP COMPLETE"
|
||||||
|
echo "==============="
|
||||||
|
echo "Backup stored in $rootdir/$name.tar.gz"
|
||||||
|
else
|
||||||
|
echo " "
|
||||||
|
echo "==============="
|
||||||
|
echo "BACKUP FAILED"
|
||||||
|
echo "==============="
|
||||||
|
echo "File was not created at $rootdir/$name.tar.gz"
|
||||||
|
rm -rf $workDir
|
||||||
|
exit 1;
|
||||||
|
fi
|
||||||
|
do_rsync $rootdir/$name.tar.gz
|
||||||
|
rm -Rf $rootdir/$name.tar.gz
|
||||||
|
} # End of do_backup_sql
|
||||||
|
|
||||||
|
do_backup_files
|
||||||
|
do_backup_sql
|
||||||
|
} # End of do_primary
|
||||||
|
|
||||||
|
restore_secondary() {
|
||||||
|
# Change this password if your root password is different than "nagiosxi"
|
||||||
|
# MySQL root password
|
||||||
|
themysqlpass="nagiosxi"
|
||||||
|
|
||||||
|
# Tests mysql connection by opening a connection and selecting the DB we want to use
|
||||||
|
test_mysql_connection() {
|
||||||
|
local db_host="$1"
|
||||||
|
local db_port="$2"
|
||||||
|
local db_username="$3"
|
||||||
|
local db_password="$4"
|
||||||
|
db_error=$(mysql -h "$db_host" --port="$db_port" -u $db_username --password=$db_password -e "show databases;" 2>&1)
|
||||||
|
return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
rootdir=/store/backups/nagiosxi
|
||||||
|
backupfile=$1
|
||||||
|
if [ ! -f $backupfile ]; then
|
||||||
|
error "Unable to find backup file $backupfile!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ts=`date +%s`
|
||||||
|
mydir=${rootdir}/${ts}-restore
|
||||||
|
mkdir -p $mydir
|
||||||
|
if [ ! -d $mydir ]; then
|
||||||
|
error "Unable to create restore directory $mydir!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Extract backup
|
||||||
|
verbose "Extracting backup to $mydir..."
|
||||||
|
cd $mydir
|
||||||
|
tar xzfps $backupfile
|
||||||
|
# Change to subdirectory
|
||||||
|
cd `ls | head -1`
|
||||||
|
backupdir=`pwd`
|
||||||
|
|
||||||
|
verbose "Backup files look okay. Preparing to restore..."
|
||||||
|
|
||||||
|
# Shutdown services
|
||||||
|
verbose "Shutting down services..."
|
||||||
|
[ -r "/usr/local/nagios/var/retention.dat" ] && cp /usr/local/nagios/var/retention.dat $backupdir
|
||||||
|
sudo $BASEDIR/manage_services.sh stop nagios
|
||||||
|
sudo $BASEDIR/manage_services.sh stop ndo2db
|
||||||
|
sudo $BASEDIR/manage_services.sh stop npcd
|
||||||
|
[ -r "$backupdir/retention.dat" ] && cp $backupdir/retention.dat /usr/local/nagios/var/retention.dat
|
||||||
|
|
||||||
|
|
||||||
|
rootdir=/
|
||||||
|
# Overwrite the mysqlpass with the hardcoded one at the top
|
||||||
|
mysqlpass="$themysqlpass"
|
||||||
|
|
||||||
|
# Restore databases
|
||||||
|
verbose "Restoring MySQL databases..."
|
||||||
|
if [[ "$cfg__db_info__ndoutils__dbserver" == *":"* ]]; then
|
||||||
|
ndoutils_dbport=`echo "$cfg__db_info__ndoutils__dbserver" | cut -f2 -d":"`
|
||||||
|
ndoutils_dbserver=`echo "$cfg__db_info__ndoutils__dbserver" | cut -f1 -d":"`
|
||||||
|
else
|
||||||
|
ndoutils_dbport='3306'
|
||||||
|
ndoutils_dbserver="$cfg__db_info__ndoutils__dbserver"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Test mysql and see if we can connect before continuing
|
||||||
|
x=1
|
||||||
|
while [ $x -le 5 ];
|
||||||
|
do
|
||||||
|
test_mysql_connection $ndoutils_dbserver $ndoutils_dbport "root" $mysqlpass
|
||||||
|
if [ $? == 1 ]; then
|
||||||
|
error "ERROR: Could not connect to $ndoutils_dbserver:$ndoutils_dbport with root password supplied."
|
||||||
|
read -s -r -p "Please enter the MySQL root password: " mysqlpass
|
||||||
|
echo ""
|
||||||
|
else
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ $x -eq 5 ]; then
|
||||||
|
error "ERROR: Aborting restore: Could not connect to MySQL."
|
||||||
|
error "$db_error"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
x=$(($x+1))
|
||||||
|
done
|
||||||
|
|
||||||
|
mysql -h "$ndoutils_dbserver" --port="$ndoutils_dbport" -u root --password=$mysqlpass < $backupdir/mysql/nagios.sql
|
||||||
|
res=$?
|
||||||
|
if [ $res != 0 ]; then
|
||||||
|
error "Error restoring MySQL database 'nagios'"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$cfg__db_info__nagiosql__dbserver" == *":"* ]]; then
|
||||||
|
nagiosql_dbport=`echo "$cfg__db_info__nagiosql__dbserver" | cut -f2 -d":"`
|
||||||
|
nagiosql_dbserver=`echo "$cfg__db_info__nagiosql__dbserver" | cut -f1 -d":"`
|
||||||
|
else
|
||||||
|
nagiosql_dbport='3306'
|
||||||
|
nagiosql_dbserver="$cfg__db_info__nagiosql__dbserver"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Test mysql again and see if we can connect before continuing
|
||||||
|
x=1
|
||||||
|
while [ $x -le 5 ];
|
||||||
|
do
|
||||||
|
test_mysql_connection $nagiosql_dbserver $nagiosql_dbport "root" $mysqlpass
|
||||||
|
if [ $? == 1 ]; then
|
||||||
|
error "ERROR: Could not connect to $nagiosql_dbserver:$nagiosql_dbport with root password supplied."
|
||||||
|
read -s -r -p "Please enter the MySQL root password: " mysqlpass
|
||||||
|
echo ""
|
||||||
|
else
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ $x -eq 5 ]; then
|
||||||
|
error "ERROR: Aborting restore: Could not connect to MySQL."
|
||||||
|
error "$db_error"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
x=$(($x+1))
|
||||||
|
done
|
||||||
|
|
||||||
|
mysql -h "$nagiosql_dbserver" --port="$nagiosql_dbport" -u root --password=$mysqlpass < $backupdir/mysql/nagiosql.sql
|
||||||
|
res=$?
|
||||||
|
if [ $res != 0 ]; then
|
||||||
|
error "Error restoring MySQL database 'nagiosql'"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Only restore PostgresQL if we are still using it
|
||||||
|
if [ "$cfg__db_info__nagiosxi__dbtype" == "pgsql" ]; then
|
||||||
|
service postgresql initdb &>/dev/null || true
|
||||||
|
verbose "Restoring Nagios XI PostgresQL database..."
|
||||||
|
if [ -f /var/lib/pgsql/data/pg_hba.conf ]; then
|
||||||
|
pghba="/var/lib/pgsql/data/pg_hba.conf"
|
||||||
|
cp -pr $pghba $pghba.old
|
||||||
|
else
|
||||||
|
#Ubuntu/Debian
|
||||||
|
pghba=$(find /etc/postgresql -name "*pg_hba.conf")
|
||||||
|
cp -pr $pghba $pghba.old
|
||||||
|
fi
|
||||||
|
echo "local all all trust
|
||||||
|
host all all 127.0.0.1/32 trust
|
||||||
|
host all all ::1/128 trust" > $pghba
|
||||||
|
$BASEDIR/manage_services.sh start postgresql
|
||||||
|
sudo -u postgres psql -c "create user nagiosxi with password 'n@gweb';"
|
||||||
|
sudo -u postgres psql -c "create database nagiosxi owner nagiosxi;"
|
||||||
|
$BASEDIR/manage_services.sh restart postgresql
|
||||||
|
# Sleep a bit (required so Postgres finishes startup before we connect again)
|
||||||
|
verbose "Sleeping for a few seconds (please wait)..."
|
||||||
|
sleep 7
|
||||||
|
psql -U nagiosxi nagiosxi < $backupdir/pgsql/nagiosxi.sql
|
||||||
|
res=$?
|
||||||
|
if [ $res != 0 ]; then
|
||||||
|
error "Error restoring PostgresQL database 'nagiosxi' !"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
$BASEDIR/manage_services.sh restart postgresql
|
||||||
|
if [ "$dist" == "el7" ] || [ "$dist" == "el8" ]; then
|
||||||
|
systemctl enable postgresql.service
|
||||||
|
elif [[ "$distro" == "Ubuntu" ]] || [[ "$distro" == "Debian" ]]; then
|
||||||
|
update-rc.d postgresql enable
|
||||||
|
else
|
||||||
|
chkconfig postgresql on
|
||||||
|
fi
|
||||||
|
# Remove nagiosxi db from mysql if postgres is used instead
|
||||||
|
if [[ "$cfg__db_info__nagiosql__dbserver" == *":"* ]]; then
|
||||||
|
nagiosql_dbport=`echo "$cfg__db_info__nagiosql__dbserver" | cut -f2 -d":"`
|
||||||
|
nagiosql_dbserver=`echo "$cfg__db_info__nagiosql__dbserver" | cut -f1 -d":"`
|
||||||
|
else
|
||||||
|
nagiosql_dbport='3306'
|
||||||
|
nagiosql_dbserver="$cfg__db_info__nagiosql__dbserver"
|
||||||
|
fi
|
||||||
|
mysql -h "$nagiosql_dbserver" --port="$nagiosql_dbport" -u root --password=$mysqlpass < "DROP TABLE IF EXISTS nagiosxi;"
|
||||||
|
else
|
||||||
|
verbose "Restoring Nagios XI MySQL database..."
|
||||||
|
if [[ "$cfg__db_info__nagiosxi__dbserver" == *":"* ]]; then
|
||||||
|
nagiosxi_dbport=`echo "$cfg__db_info__nagiosxi__dbserver" | cut -f2 -d":"`
|
||||||
|
nagiosxi_dbserver=`echo "$cfg__db_info__nagiosxi__dbserver" | cut -f1 -d":"`
|
||||||
|
else
|
||||||
|
nagiosxi_dbport='3306'
|
||||||
|
if [ "x$cfg__db_info__nagiosxi__dbserver" == "x" ]; then
|
||||||
|
nagiosxi_dbserver="localhost"
|
||||||
|
else
|
||||||
|
nagiosxi_dbserver="$cfg__db_info__nagiosxi__dbserver"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Test mysql again and see if we can connect before continuing
|
||||||
|
x=1
|
||||||
|
while [ $x -le 5 ];
|
||||||
|
do
|
||||||
|
test_mysql_connection $nagiosxi_dbserver $nagiosxi_dbport "root" $mysqlpass
|
||||||
|
if [ $? == 1 ]; then
|
||||||
|
error "ERROR: Could not connect to $nagiosxi_dbserver:$nagiosxi_dbport with root password supplied."
|
||||||
|
read -s -r -p "Please enter the MySQL root password: " mysqlpass
|
||||||
|
echo ""
|
||||||
|
else
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ $x -eq 5 ]; then
|
||||||
|
error "ERROR: Aborting restore: Could not connect to MySQL."
|
||||||
|
error "$db_error"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
x=$(($x+1))
|
||||||
|
done
|
||||||
|
|
||||||
|
mysql -h "$nagiosxi_dbserver" --port="$nagiosxi_dbport" -u root --password=$mysqlpass < $backupdir/mysql/nagiosxi.sql
|
||||||
|
res=$?
|
||||||
|
if [ $res != 0 ]; then
|
||||||
|
error "Error restoring MySQL database 'nagiosxi' !"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
##############################
|
||||||
|
# RESTART SERVICES
|
||||||
|
##############################
|
||||||
|
verbose "Restarting services..."
|
||||||
|
sudo $BASEDIR/manage_services.sh restart mysqld
|
||||||
|
sudo $BASEDIR/manage_services.sh restart httpd
|
||||||
|
sudo $BASEDIR/manage_services.sh start npcd
|
||||||
|
sudo $BASEDIR/manage_services.sh start ndo2db
|
||||||
|
sudo $BASEDIR/manage_services.sh start nagios
|
||||||
|
verbose "Restarting Nagios XI"
|
||||||
|
cd /usr/local/nagiosxi/scripts
|
||||||
|
./reconfigure_nagios.sh
|
||||||
|
rm -rf $mydir
|
||||||
|
echo " "
|
||||||
|
echo "==============="
|
||||||
|
echo "RESTORE COMPLETE"
|
||||||
|
echo "==============="
|
||||||
|
}
|
||||||
|
|
||||||
|
do_secondary() {
|
||||||
|
[ ! -r "$syncfile" ] && error "No sync file ($syncfile) found." && exit 2
|
||||||
|
restore_secondary "$syncfile"
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
if [ -r "/usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css.secondary" -a -w "/usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css" ]; then
|
||||||
|
cp /usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css.secondary /usr/local/nagiosxi/html/includes/components/custom-includes/css/header-gradient.css
|
||||||
|
fi
|
||||||
|
rm -f ${syncfile}
|
||||||
|
fi
|
||||||
|
} # End of do_secondary
|
||||||
|
|
||||||
|
ping_host() {
|
||||||
|
$noping && return 0
|
||||||
|
/usr/local/nagios/libexec/check_icmp -H $1 >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
[ -z "$pName" -o -z "$sName" ] && echo "Must specify primary name and secondary name" && exit 0
|
||||||
|
[ "$hostname" = "$pName" ] && isP=/bin/true
|
||||||
|
[ "$hostname" = "$sName" ] && isS=/bin/true
|
||||||
|
$isP && do_primary
|
||||||
|
$isS && do_secondary
|
||||||
Loading…
Reference in New Issue