diff --git a/event_handler.sh b/event_handler.sh new file mode 100755 index 0000000..3ec6e21 --- /dev/null +++ b/event_handler.sh @@ -0,0 +1,141 @@ +#!/bin/sh + +PATH=/usr/local/bin:/usr/bin:/bin + +# Mail settings in case we want to do to OOB email notifications +email="eloyd@everwatch.global" + +print_usage() { + cat << EOF +Generic event handler. Should not be executed by hand. + +Order of arguments is: + + hostname + description of the service that failed + state of the service that failed + type of state (HARD or SOFT) + number of times it's been in this state + [-d] debug (must be last parameter - ONLY FOR DEBUGGING) + +We have to figure out what action to take based on the description. +EOF + exit 1 +} + +do_foobar() { + echo "Foobar debug=$debug host=$host what=$what service=$service state=$state hardSoft=$hardSoft tries=$tries" +} + +do_HTTP() { + case "$host" in + "192.168.1.88") /usr/local/nagios/libexec/check_3gstore_outlet -H "192.168.1.3" -o 2 -c RESTART;; + esac +} + +do_email() { + [ -z "$email" ] && return + myHost=`echo $host | tr " " "+"` + myService=`echo $service | tr " " "+"` + echo "https://192.168.1.19/nagiosxi/includes/components/xicore/status.php?show=servicedetail&host=$myHost&service=$myService&dest=auto" | mail -s "Restarted $host:$service" $email +} + +################################################################################ +# Here's where the program really starts (main) +################################################################################ + +# Command line help for debugging +[ "$1" = "-h" ] && print_usage + +host="$1" +service="$2" +colonService=`echo $service | awk -F: '{print $1}'` +dashService=`echo $service | awk -F- '{print $1}' | tr -d " "` +state="$3" +hardSoft="$4" +tries="$5" +debug="false" +[ "$6" = "-d" ] && debug="true" + +# Make SSH do something useful or add keys, username, or other options, in case we need it +ssh="/bin/ssh" +[ ! -x "$ssh" ] && ssh="/usr/bin/ssh" +[ ! -x "$ssh" ] && ssh="/usr/local/bin/ssh" +[ ! -x "$ssh" ] && exit 1 + +# do_restart is the main entry point for ALL services that use this event handler. +do_restart() { + $debug && echo "We're going to see if we can restart $service on $host..." + case "$service" in + "Service Status - ntpd") sudo systemctl restart ntpd; do_email;; + "HTTP") do_HTTP;; + *) do_nothing;; + esac +} + +date >> /tmp/event_handler.log +env >> /tmp/event_handler.log +echo "" >> /tmp/event_handler.log + +# Nothing to do (yet?) +do_nothing() { + $debug && echo "$service is in $state state ($hardSoft) for $tries tries." +} + +# Do we restart soft services? Yes, if this is the 3rd soft state +do_soft() { + case "$tries" in + 3) do_restart;; # Okay, let's restart it before it goes hard + *) do_nothing;; # Don't restart yet + esac +} + +# See if there's something to do +do_unknown() { + case "$hardSoft" in + SOFT) do_soft;; # Take action before it's too late? + HARD) do_restart;; # Hard CRITICAL - Our last chance to take action + *) do_nothing;; + esac +} + +# See if there's something to do +do_warning() { + case "$hardSoft" in + SOFT) do_soft;; # Take action before it's too late? + HARD) do_restart;; # Hard CRITICAL - Our last chance to take action + *) do_nothing;; + esac +} + +# See if there's something to do +do_critical() { + case "$hardSoft" in + SOFT) do_soft;; # Take action before it's too late? + HARD) do_restart;; # Hard CRITICAL - Our last chance to take action + *) do_nothing;; + esac +} + +# See if there's something to recover +# So far, we only do it for Apache workers +do_fixit() { + case "$colonService" in + Priority) do_restart;; + *) do_nothing;; + esac + case "$dashService" in + ImportantService) do_restart;; + *) do_nothing;; + esac +} + +# Main entry point - what's the status of the thing? +echo "state is $state" +case "$state" in + 0|OK) do_fixit;; # If it ain't broke, maybe fix it. + 1|WARNING|warning|Warning) do_warning;; # If it ain't really broke, probably don't fix it. + 2|CRITICAL|critical|Critical) do_critical;; # Figure out what we should do + 3|UNKNOWN|unknown|Unknown) do_unknown;; # Probably can't touch it anyway. + *) do_nothing;; +esac