#!/usr/bin/perl -w
#
# By Igor Gubenko (igubenko@Princeton.EDU) 09/07/2012
#
# Script to create or update a Cacti graph from given Nagios output
#
# $Header: /usr/local/nagios/libexec/eventhandlers/RCS/cacti_update,v 1.5 2013/05/03 14:55:23 nagios Exp nagios $
#

use lib qw(/usr/local/monitoring/perl/lib/perl5);
#use lib qw(/usr/local/perl/modules/lib/perl5/site_perl/5.8.8 /usr/local/perl/modules/lib/perl5/5.8.8 /usr/local/perl/modules/lib64/perl5/site_perl/5.8.8 /usr/local/perl/modules/lib64/perl5/5.8.8 /usr/local/perl/lib/perl5/site_perl/5.8.8 /usr/local/perl/lib/perl5/5.8.8 /usr/local/perl/lib64/perl5/site_perl/5.8.8 /usr/local/perl/lib64/perl5/5.8.8);

use strict;
use warnings;
use Config::Any;
use WWW::Mechanize;
use HTML::TableExtract;
use Data::Dumper;
use Digest::MD5 qw(md5);
use threads;
use threads::shared;
use Thread::Queue;
use POSIX;
use DBI;
use File::Pid;

$ENV{PERL_LWP_SSL_VERIFY_HOSTNAME}=0;
my $myname = $0;
my ($mech, $cacdb);
my $cacti_path = '/usr/local/cacti';
my $rrapath = '/usr/local/cacti/nagiosrrd';
my $rrdtool = '/usr/bin/rrdtool';
my $php = '/usr/bin/php';
my $spec_file = '/usr/local/monitoring/libexec/eventhandlers/cacti_service_spec.json';
my $close_file = '/usr/local/monitoring/libexec/eventhandlers/posttasks';
my $perf_file = '/usr/local/monitoring/var/naemon/service-perfdata';
#my $perf_file = '/usr/local/nagios/libexec/eventhandlers/data.temp';
my $log_file = '/usr/local/monitoring/log/cacti_update.log';
my $persistence_file = '/usr/local/monitoring/var/naemon/cacti_persist.cache';
my $pidpath = "/var/run/cacti/" . $myname . ".pid";
my $pidfile;

my $cacti_user = 'csgtst05';
my $cacti_pass = 'C5Gt$t42!!!!!!!!';
#my $cacti_user = 'groundwk';
#my $cacti_pass = 'th4de/DC';

my $cacti_host = "ims204";
my $cacti_url = "https://${cacti_host}.princeton.edu/cactigraphs";

my $rracacspec = 'RRA:AVERAGE:0.5:1:500 RRA:AVERAGE:0.5:1:600 RRA:AVERAGE:0.5:6:700 RRA:AVERAGE:0.5:24:775 RRA:AVERAGE:0.5:288:797 RRA:MAX:0.5:1:500 RRA:MAX:0.5:1:600 RRA:MAX:0.5:6:700 RRA:MAX:0.5:24:775 RRA:MAX:0.5:288:797';

my $spec;
my %cacti_hosts;
my %data_templates;
my %perfhist :shared;
my %graph_templates;
my $verbose = 1;
my $outtype = "log";
my $terminate :shared = 0;
my $waiton :shared = 0;
my $pnag;
my $pnagcac;

#my @data_templates_created;
my %data_source_created;

sub begin;
sub load;
sub unload;
sub reload;
sub read_perfdata;
sub update_rrd;

sub cacti_update;
sub dt_check_create;
sub gt_check_create;
sub ds_check_create;
sub g_check_create;

sub end_program;
sub graceful_die;
sub verbose;

sub DB_get;
sub get_cacti_data_templates;
sub get_cacti_data_template_component_id;
#sub get_cacti_data_sources;
sub get_cacti_host_template_ds;
sub get_cacti_graph_templates;
sub get_cacti_host_ds_components;
sub get_cacti_graph_template_component;
sub get_cacti_graph;
sub get_cacti_hosts;

begin;

graceful_die 1, 1;

##################################
#### Subroutines

# This routine creates/updates Cacti to use external RRD's
sub cacti_update {
	# Next part is to create the Cacti mapping, for whatever graphs if wasn't done yet
	# 1) Create Data Template  -- per RRD, per host
	# 2) Create Graph Template -- per Graph spec
	# 3) Create Data Source -- per RRD, per host
	# 4) Create New Graph -- per Graph spec  --- doesn't have a "name" - just association
	#
	verbose 1, "------ Cacti update thread started ------";

	verbose 1, "Trying to log to Cacti...";
	
	$mech = WWW::Mechanize->new(agent => 'Mozilla/5.0', stack_depth => '2', noproxy => '1');

	$mech->credentials($cacti_user, $cacti_pass);

	## Get initial Cacti login page
	$mech->get($cacti_url);

	unless ($mech -> success() && $mech->response()->decoded_content =~ m#Console#) {
		verbose 0, "Failed to login. Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		graceful_die 1, 1;
	}

	verbose 1, "Login to Cacti successful\n";
	verbose 2, "Response:\n" . $mech->response()->decoded_content . "\n\n##########################";

	$cacdb = DBI -> connect('DBI:mysql:cactigraphs', 'cactus', 'th4de/DC', {'RaiseError' => 1});

	unless ($cacdb) {
		verbose 0, "Failed to connect to the Cacti database: $DBI::errstr";
		graceful_die 1, 1;
	}

	verbose 1, "Successfully connected to the Cacti database\n";

	my $hst_tmp = get_cacti_hosts;
	%cacti_hosts = %$hst_tmp;

	my $dt_tmp = get_cacti_data_templates;
	%data_templates = %$dt_tmp;

	my $gt_tmp = get_cacti_graph_templates;
	%graph_templates = %$gt_tmp;

	open PERS, ">> $persistence_file";

	CAC: while ( $_ = $pnagcac -> dequeue ) {
		last if /Terminate|Reload/;

		next unless /Next service/;

		my %perfdata;
		my ($curhost, $curserv, $hstspec);

		while ( $_ = $pnagcac -> dequeue ) {
verbose (1, "Cacti: New perf arrived: ##$_##");
			last if /End service/;
			last CAC if /Terminate|Reload/;

			/Host: (.*)/ and do {
				$curhost = $1;
				$perfdata{$curhost} = {};
			};
			/Service: (.*)/ and do {
				verbose 1, "Cacti: New perf for Cacti: \"$curhost\" -- \"$1\"";
				$curserv = $1;
				$perfdata{$curhost} -> {$curserv} = {};
			};
			/Host Spec: (.*)/ and $hstspec = $1;
			/Service Spec: (.*)/ and do {
				$perfdata{$curhost} -> {$curserv} -> {spec} = $spec->{$1} -> {$hstspec};
			};
			/Perf: (.*)/ and do {
				my $lbl = $1;
				$_ = $pnagcac -> dequeue;
verbose (1, "Cacti: Perf: ##$_##");
				my @perfsplit = split /::: /;
				$perfdata{$curhost} -> {$curserv} -> {perf} -> {$lbl} = \@perfsplit;
			};
		}

		my $curperf = $perfdata{$curhost}->{$curserv};

		# Check if the host exists, first. If it's not in Cacti, there's nothing to do really
		next unless grep /^${curhost}(|\.princeton\.edu)$/i, keys %cacti_hosts;
		verbose 1, "Cacti: \"$curhost\" exists. Need to check for graph data";

		# Check if the data template exists, and if not, create it
		next if dt_check_create ($curperf, $curserv, $curhost);
		verbose 1, "Cacti: Data template exists for \"$curserv\". Need to check for graph data";

		# Check if the graph template exists, and if not, create it
		next if gt_check_create ($curperf, $curserv, $curhost);
		verbose 1, "Cacti: Graph template exists for \"$curserv\". Need to check for data sources, and graphs";

		# Check if the data source exists, and if not, create it
		next if ds_check_create ($curperf, $curserv, $curhost);
		verbose 1, "Cacti: Data source exists for \"$curhost\" - \"$curserv\". Need to check for the graph";

		# Check if the graph exists, and if not, create it
		#g_check_create ($curperf, $curserv, $curhost);

		if (g_check_create ($curperf, $curserv, $curhost)) {
			lock %perfhist;
			lock %{$perfhist{$curhost}};
			$perfhist{$curhost}->{$curserv} = 1;   # Error, do not write to persistence; also try processing again
		} else {
			lock %perfhist;
			lock %{$perfhist{$curhost}};
			$perfhist{$curhost}->{$curserv} = 2;   # OK, processed fine - write to persistence

			print PERS "${curhost}::::${curserv}\n";
		}
	}

	close PERS;

	$cacdb -> disconnect;
	$mech->get ("${cacti_url}/logout.php");

	verbose 1, "------ Cacti update thread terminated ------";

	return 0;
}

# This routine checks if a data template exists, and if not, it creates it
sub dt_check_create {
	my $data = shift;
	my $serv = shift;
	my $host = shift;

	# We need a unique name for a template, unique for service name AND its perfdata items
	my $cactiserv = "${serv}_" . unpack ("L", md5(join "", sort keys %{$data->{perf}}));

	# If we have the data template on record, don't waste time
	#return 0 if grep m#^$cactiserv$#i, @data_templates_created;
	return 0 if (grep m#^$cactiserv$#i, keys %data_templates);

	verbose (1, "Cacti: Data Template not found for \"$cactiserv\" - will create");

	# Initial Cacti page containing data template data
#	$mech->get ("${cacti_url}/data_templates.php");

#	unless ($mech -> success() && $mech->title() =~ m#Console -> Data Templates#) {
#		verbose 0, "Cacti: Failed to fetch \"${cacti_url}/data_templates.php\". Answer: " . $mech->status();
#		verbose 2, "Response:\n" . $mech->response()->decoded_content;
#		return 1;
#	}

#	verbose 2, "Cacti: Query of ${cacti_url}/data_templates.php resulted in:\n" . $mech->response()->decoded_content . "\n\n######################";

	##############################################################
	# Search for the data template - check if it exists
#	$mech->submit_form(
#		form_name => 'form_data_template',
#		fields => {
#			filter => $cactiserv
#		},
#	);

#	unless ($mech -> success() && $mech->title() =~ m#Console -> Data Templates#) {
#		verbose 0, "Cacti: Failed to submit to \"${cacti_url}/data_templates.php\". Answer: " . $mech->status();
#		verbose 2, "Response:\n" . $mech->response()->decoded_content;
#		return 1;
#	}

#	verbose 2, "Cacti: Submit to ${cacti_url}/data_templates.php resulted in:\n" . $mech->response()->decoded_content . "\n\n######################";
	##############################################################
	# Found the data template?
	#<a class='linkEditMain' href='data_templates.php?action=template_edit&amp;id=27'><span style='background-color: #F8D93D;'>Cisco Router - 5 Minute CPU</span>
#	if (grep m#<a class='linkEditMain' href='data_templates.php.*>${cactiserv}</span>#, $mech->response()->decoded_content) {
#		verbose (1, "Cacti: Data Template already exists for \"$cactiserv\"");
#		return 0;
#	}
#	verbose (1, "Cacti: Data Template not found for \"$cactiserv\" - will create");

	# No, need to create it
	# Go to data template add page
#	$mech->follow_link (text => 'Add');

	$mech->get ("${cacti_url}/data_templates.php?action=template_edit");

	unless ($mech -> success() && $mech->title() =~ m#Console -> Data Templates -> \(Edit\)#) {
		verbose 0, "Cacti: Failed to go to the data template addition page. Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Accessed the page to add a data template:\n" . $mech->response()->decoded_content . "\n\n######################";
	###############################################################

	my @dsnames;
	@dsnames = keys %{$data->{perf}};    # Our pefdata labels, which are also internal data source names for Cacti

	my $inids = shift @dsnames;
	my @create_item_regexs;
	my $create_item_spec_match;
	my %typemap = ( GAUGE => 1, COUNTER => 2, DERIVE => 3, ABSOLUTE => 4 );

	@create_item_regexs = keys %{$data->{spec}->{create_items}};
	$create_item_spec_match = join "", grep { $inids =~ /^$_$/i } @create_item_regexs;

#print "data_source_name => $inids,
#t_rrd_minimum => $data->{spec}->{create_items}->{$create_item_spec_match}->{min},
#t_rrd_maximum => $data->{spec}->{create_items}->{$create_item_spec_match}->{max},
#data_source_type_id => $typemap{$data->{spec}->{create_items}->{$create_item_spec_match}->{type}},\n";
#exit 1;

	# In Cacti, we first create an initial data template, with an initial data source name, and then we can add more
	$mech->submit_form(
		fields => {
			template_name => "$cactiserv",
			name => "|host_description| - $cactiserv",
			active => 'unchecked',
			data_source_name => $inids,
			rrd_minimum => $data->{spec}->{create_items}->{$create_item_spec_match}->{min},
			rrd_maximum => $data->{spec}->{create_items}->{$create_item_spec_match}->{max},
			data_source_type_id => $typemap{$data->{spec}->{create_items}->{$create_item_spec_match}->{type}},
		},
	);
				
	unless ($mech -> success() && $mech->response()->decoded_content =~ m#Save Successful#) {
		verbose 0, "Cacti: Failed to save the new data template \"$serv\" -- \"${cacti_url}/data_templates.php?action=template_edit\". Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Saved the new data template \"$serv\" to ${cacti_url}/data_templates.php?action=template_edit:\n" . $mech->response()->decoded_content . "\n\n######################";
	#################################################################

	# Now we can add additional items to the data template (perfdata labels), if needed
	my $dsprev = $inids;
	foreach my $nextds (@dsnames) {
		$create_item_spec_match = join "", grep { $nextds =~ /^$_$/i } @create_item_regexs;	

		# First we ask to create a new component
		$mech -> follow_link (text => 'New');

		unless ($mech -> success() && $mech->response()->decoded_content =~ m#\d+: $dsprev#) {
			verbose 0, "Cacti: Failed to create new item in the new data template \"$serv\" -- \"${cacti_url}/data_templates.php?action=template_edit\". Answer: " . $mech->status();
			verbose 2, "Response:\n" . $mech->response()->decoded_content;
			return 1;
		}

		verbose 2, "Cacti: Created new component for new data template \"$serv\", ds component \"$nextds\" in ${cacti_url}/data_templates.php?action=template_edit:\n" . $mech->response()->decoded_content . "\n\n######################";

		##################################################
		# Then we save it with necessary parameters
		$mech->submit_form(
			fields => {
				data_source_name => $nextds,
				rrd_minimum => $data->{spec}->{create_items}->{$create_item_spec_match}->{min},
				rrd_maximum => $data->{spec}->{create_items}->{$create_item_spec_match}->{max},
				data_source_type_id => $typemap{$data->{spec}->{create_items}->{$create_item_spec_match}->{type}},
			},
		);
				
		unless ($mech -> success() && $mech->response()->decoded_content =~ m#Save Successful#) {
			verbose 0, "Cacti: Failed to save the new data template \"$serv\" with new item \"$nextds\" - \"${cacti_url}/data_templates.php?action=template_edit\". Answer: " . $mech->status();
			verbose 2, "Response:\n" . $mech->response()->decoded_content;
			return 1;
		}

		verbose 2, "Cacti: Saved new component for new data template \"$serv\", ds component \"$nextds\" in ${cacti_url}/data_templates.php?action=template_edit:\n" . $mech->response()->decoded_content . "\n\n######################";

		$dsprev = $nextds;
	}

	#push @data_templates_created, $cactiserv;
	my $dt_tmp = get_cacti_data_templates;
	%data_templates = %$dt_tmp;

	verbose (1, "Cacti: Successfully created a new data template for service \"$cactiserv\"");
	return 0;  # Successful creation of all components
}

# Check if the graph template exists, and if not, create
sub gt_check_create {
	my $data = shift;
	my $serv = shift;
	my $host = shift;
	my ($cactiserv, $uniqname);

	$uniqname = unpack ("L", md5(join "", sort keys %{$data->{perf}}));
	$cactiserv = "${serv}_${uniqname}";

	# Does the template exist?
	#return 0 if grep /^$data->{spec}->{graph_template_name}_${cactiserv}$/i, @graph_templates;
	return 0 if exists $graph_templates{"$data->{spec}->{graph_template_name}_${cactiserv}"};

	verbose 1, "Cacti: Graph template not found. Will create";

	# Access initial page containing the graph templates
	$mech->get ("${cacti_url}/graph_templates.php");

	unless ($mech -> success() && $mech->title() =~ m#Console -> Graph Templates#) {
		verbose 0, "Cacti: Failed to fetch \"${cacti_url}/graph_templates.php\". Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Query of ${cacti_url}/graph_templates.php resulted in:\n" . $mech->response()->decoded_content . "\n\n######################";
	#######################################################
	# Add a new one
	$mech -> follow_link (text => 'Add');

	unless ($mech -> success() && $mech->title() =~ m#Console -> Graph Templates -> \(Edit\)#) {
		verbose 0, "Cacti: Failed to follow the \"Add\" link. Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Followed the \"Add\" link:\n" . $mech->response()->decoded_content . "\n\n######################";
	########################################################
	# Next we need to save the new graph template with global graph template data

	my ($title, $hgt, $wdt, $t_upper, $t_lower, $t_base, $g_templ_name, $g_templ_label);

	$title = "|host_description| - " . (exists $data->{spec}->{graph_title} ? $data->{spec}->{graph_title} : $serv) . "($uniqname)";
	$hgt = exists $data->{spec}->{graph_height} ? $data->{spec}->{graph_height} : 120;
	$wdt = exists $data->{spec}->{graph_width} ? $data->{spec}->{graph_width} : 500;
	$t_upper = exists $data->{spec}->{graph_upper_limit} ? $data->{spec}->{graph_upper_limit} : 100;
	$t_lower = exists $data->{spec}->{graph_lower_limit} ? $data->{spec}->{graph_lower_limit} : 0;
	$t_base = exists $data->{spec}->{graph_base} ? $data->{spec}->{graph_base} : 1000;
	$g_templ_name = $data->{spec}->{graph_template_name};

	# If the desired graph label is the same as the data source item name (for a single DS, we need to get it out)
	my ($firstlbl) = keys %{$data->{perf}};
	$g_templ_label = $data->{spec}->{graph_label} =~ m#\^item\^#i ? $data->{perf}->{$firstlbl}->[5] : $data->{spec}->{graph_label};

#print "$title###$data->{spec}->{graph_label}###\n";
#return 1;

	$mech -> submit_form (
		fields => {
			name => "${g_templ_name}_${cactiserv}",
			title => $title,
			height => $hgt,
			width => $wdt,
			upper_limit => $t_upper,
			lower_limit => $t_lower,
			base_value => $t_base,
			vertical_label => $g_templ_label,
		},
	);

	unless ($mech -> success() && $mech->response()->decoded_content =~ m#Save Successful#) {
		verbose 0, "Cacti: Failed to save the new graph template \"${g_templ_name}_${cactiserv}\". Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Saved new graph template \"${g_templ_name}_${cactiserv}\":\n" . $mech->response()->decoded_content . "\n\n######################";
	############################################################
	# Now we need to add graph components, and associate them with items in the relevant data template

	my (@dsnames, @graph_item_regexs);
	@dsnames = keys %{$data->{perf}};
	@graph_item_regexs = keys %{$data->{spec}->{graph_items}};

	my (%data_template_sel, %colors_sel, %function_type_sel, %graph_type_sel, %cdef_sel, %gprint_sel);
	my ($data_template_sel_items, $colors_sel_items, $function_type_sel_items, $graph_type_sel_items, $cdef_sel_items, $gprint_sel_items);

	# For each data source item
	foreach my $curds (@dsnames) {
		my $graph_item_spec_match = join "", grep { $curds =~ /^$_$/i } @graph_item_regexs;
		my @gr_items;
		
		@gr_items = @{$data->{spec}->{graph_items}->{$graph_item_spec_match}};

		# For each graph component of relevant ds item (line, area, etc)
		foreach my $curgritem (@gr_items) {
			$mech -> follow_link (text => 'Add', n => 1);
			my $curcont = $mech->response()->decoded_content;

			unless ($mech -> success() && $mech->title() =~ m#Console -> Graph Templates -> \(Edit\) -> Graph Template Items#) {
				verbose 0, "Cacti: Failed to add a new graph component by following the \"Add\" link. Answer: " . $mech->status();
				verbose 2, "Response:\n" . $curcont;
				return 1;
			}

			verbose 2, "Cacti: Followed the \"Add\" link to add a new graph component:\n" . $curcont . "\n\n######################";

			# Load a list of data templates
			($data_template_sel_items) = $mech -> find_all_inputs (
							type => 'option',
							name_regex => qr/^task_item_id$/,
						);
			# We only need to determine the list of data templates once
#			@data_template_sel{$data_template_sel_items->value_names} = $data_template_sel_items->possible_values if keys %data_template_sel == 0;

#print Dumper (\%data_template_sel);

			# We need to determine the relevant data template id to select each time
			# We do this, in case the key matched case-insensitive
#			my $keytemp = join "", grep m#^${cactiserv}\s+\-\s+\(${curds}\)$#i, keys %data_template_sel;
			my $keytemp = get_cacti_data_template_component_id ($cactiserv, $curds);
#verbose 0, Dumper ($keytemp);
			unless ($keytemp && @$keytemp == 1) {
				verbose 0, "Cacti: Could not locate the data_template ID for \"$cactiserv\"";
				return 1;
			}
			$keytemp = $keytemp->[0];
#verbose 0, "\n###$keytemp###\n";
#print $data_template_sel{$keytemp}, "\n";
			# Select the right data template
		#	$data_template_sel_items -> value ($data_template_sel{$keytemp});
			$data_template_sel_items -> value ($keytemp);

			# Load a list of colors
			($colors_sel_items) = $mech -> find_all_inputs (
						type => 'option',
						name_regex => qr/^color_id$/,
					);
			@colors_sel{$colors_sel_items->value_names} = $colors_sel_items->possible_values if keys %colors_sel == 0;

			# Load a list of graph types (AREA, GRPINT, etc)
			($graph_type_sel_items) = $mech -> find_all_inputs (
						type => 'option',
						name_regex => qr/^graph_type_id$/,
					);
			@graph_type_sel{$graph_type_sel_items->value_names} = $graph_type_sel_items->possible_values if keys %graph_type_sel == 0;

			# Load a list of functions (AVERAGE, MIN, MAX)
			($function_type_sel_items) = $mech -> find_all_inputs (
						type => 'option',
						name_regex => qr/^consolidation_function_id$/,
					);
			@function_type_sel{$function_type_sel_items->value_names} = $function_type_sel_items->possible_values if keys %function_type_sel == 0;

			# Load a list of CDEF's
			($cdef_sel_items) = $mech -> find_all_inputs (
						type => 'option',
						name_regex => qr/^cdef_id$/,
					);
			@cdef_sel{$cdef_sel_items->value_names} = $cdef_sel_items->possible_values if keys %cdef_sel == 0;

			# Load a list of GPRINT types
			($gprint_sel_items) = $mech -> find_all_inputs (
						type => 'option',
						name_regex => qr/^gprint_id$/,
					);
			@gprint_sel{$gprint_sel_items->value_names} = $gprint_sel_items->possible_values if keys %gprint_sel == 0;


			my $gritem = join "", keys %{$curgritem};  # AREA, LINE1, GRPINT, etc

			# Select the right graph type
			$keytemp = join "", grep m#^$gritem$#i, keys %graph_type_sel;
			$graph_type_sel_items -> value ($graph_type_sel{$keytemp});

			# Function is average for graphical items, and whatever is specified for everything else
			my $func = $gritem =~ m#area|stack|line\d+#i ? 'AVERAGE' : $curgritem -> {$gritem} -> {function};

			# Select the right function type
			$keytemp = join "", grep m#^$func$#i, keys %function_type_sel;
			$function_type_sel_items -> value ($function_type_sel{$keytemp});

			# Select the right CDEF if necessary
			if (exists $curgritem->{$gritem}->{cdef}) {
				$keytemp = join "", grep m#^$curgritem->{$gritem}->{cdef}$#i, keys %cdef_sel;
				$cdef_sel_items -> value ($cdef_sel{$keytemp});
			}

			# Select the right GPRINT type if necessary
			if (exists $curgritem->{$gritem}->{type}) {
				$keytemp = join "", grep m#^$curgritem->{$gritem}->{type}$#i, keys %gprint_sel;
				$gprint_sel_items -> value ($gprint_sel{$keytemp});
			}
			
			my $text;

			$text = $curgritem -> {$gritem} -> {text} =~ m#\^item\^# ? $data->{perf}->{$curds}->[5] : $curgritem -> {$gritem} -> {text};

			# Select the right color, or if missing/random - select a random color
			exists $curgritem -> {$gritem} -> {color} and do {
				$keytemp = join "", grep m#^$curgritem->{$gritem}->{color}$#i, keys %colors_sel;

				# Color not found or random color
				my @colorkeys = keys %colors_sel;
				while ($keytemp eq 'FFFFFF' || $keytemp eq '') {  # White color is ignored
					$keytemp = $colorkeys[int (rand (scalar (@colorkeys)))];
				}

				$colors_sel_items -> value ($colors_sel{$keytemp});
			};

			# Hard return checked if this is the last item
			#my $hard_return = $gr_items[@gr_items-1] eq $curgritem ? 'checked' : 'unchecked';
			$mech -> field (hard_return => 'on') if ($gr_items[@gr_items-1] eq $curgritem || exists $curgritem -> {$gritem} -> {eol});

			$mech -> submit_form (
				fields => {
					text_format => $text,
				#	hard_return => $hard_return,
				},
			);
				
			unless ($mech -> success() && $mech->response()->decoded_content =~ m#Save Successful#) {
				verbose 0, "Cacti: Failed to save a new graph item $curds -> $func for new graph template \"$g_templ_name\". Answer: " . $mech->status();
				verbose 2, "Response:\n" . $mech->response()->decoded_content;
				return 1;
			}

			verbose 2, "Cacti: Saved new graph item $curds -> $func for new graph template \"$g_templ_name\":\n" . $mech->response()->decoded_content . "\n\n######################";

		}
	}

	verbose 1, "Cacti: Graph template \"${g_templ_name}_${cactiserv}\" successfully created";

	my $gt_tmp = get_cacti_graph_templates;
	%graph_templates = %$gt_tmp;

	#@graph_templates = `$php -q ${cacti_path}/cli/add_graphs.php --list-graph-templates`;
	#shift @graph_templates;
	#map { chomp; s/^\d+\s+//; } @graph_templates;

	return 0; # Successful creation of all components
}

# Check if the data source exists, and if not, create
sub ds_check_create {
	my $data = shift;
	my $serv = shift;
	my $host = shift;
	my $cactiserv = "${serv}_" . unpack ("L", md5(join "", sort keys %{$data->{perf}}));

	$cactiserv = join ("", grep (m#^$cactiserv$#i, keys %data_templates));

	return 0 if (get_cacti_host_template_ds ($host, $cactiserv));
	#if (get_cacti_host_template_ds ($host, $cactiserv)) {
		#verbose 1, "Data source already exists for host \"$host\" and service \"$cactiserv\"";
		#return 0;
	#}

	# If we have the data source on record, don't waste time
#	return 0 if exists $data_source_created{$cactiserv} && grep m#^$host$#i, @{$data_source_created{$cactiserv}};

	# First see if the data sources already exist
	# Get the page with data sources
#	$mech -> get ("${cacti_url}/data_sources.php");

#	unless ($mech -> success() && $mech->title() =~ m#Console -> Data Sources#) {
#		verbose 0, "Cacti: Failed to fetch \"${cacti_url}/data_sources.php\". Answer: " . $mech->status();
#		verbose 2, "Response:\n" . $mech->response()->decoded_content;
#		return 1;
#	}

#	verbose 2, "Cacti: Query of ${cacti_url}/data_sources.php resulted in:\n" . $mech->response()->decoded_content . "\n\n######################";
	##################################################
	my (%data_template_sel, %host_sel);
	my ($data_template_sel_items, $host_sel_items);

	# Load a list of data templates
#	($data_template_sel_items) = $mech -> find_all_inputs (
#							type => 'option',
#							name_regex => qr/^template_id$/,
#						);
#	@data_template_sel{$data_template_sel_items->value_names} = $data_template_sel_items->possible_values;

	# Locate the needed data template in the list, and select it
#	my $servkey = join "", grep m#^$cactiserv$#i, keys %data_template_sel;
#	my $hostkey;
#	if (exists $data_template_sel{$servkey}) {   # No hosts with the template???
#		$data_template_sel_items -> value ($data_template_sel{$servkey});
		####################
		# Load a list of hosts
#		($host_sel_items) = $mech -> find_all_inputs (
#					type => 'option',
#					name_regex => qr/^host_id$/,
#					);
#		@host_sel{$host_sel_items->value_names} = $host_sel_items->possible_values;

		# Locate the needed host in the list, and select it
#		$hostkey = join "", grep m#^$host #i, keys %host_sel;
#		$hostkey = join "", grep m# \(${host}\.#i, keys %host_sel if $hostkey eq '';
#		if (exists $host_sel{$hostkey}) {
#			$host_sel_items -> value ($host_sel{$hostkey});
			###################

			# Now search for it
#			$mech -> submit_form (
#				form_name => 'form_data_sources',
#			);

#			unless ($mech -> success() && $mech->title() =~ m#Console -> Data Sources#) {
#				verbose 0, "Cacti: Failed to search \"${cacti_url}/data_sources.php\". Answer: " . $mech->status();
#				verbose 2, "Response:\n" . $mech->response()->decoded_content;
#				return 1;
#			}

#			verbose 2, "Cacti: Query of ${cacti_url}/data_sources.php resulted in:\n" . $mech->response()->decoded_content . "\n\n######################";
			# Data source exists?
#			return 0 if $mech->response()->decoded_content !~ m#No Data Sources#;
#		}
#	}
	########################################################

	verbose 1, "Cacti: Data source not found for host \"$host\" and service \"$cactiserv\". Will create";

	# Add a new data source
#	$mech->follow_link (text => 'Add');
	$mech -> get ("${cacti_url}/data_sources.php?action=ds_edit&host_id=-1");

	unless ($mech -> success() && $mech->title() =~ m#Console -> Data Sources -> \(Edit\)#) {
		verbose 0, "Cacti: Failed to add a new data source by following the \"Add\" link. Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Followed the \"Add\" link to add a new data source:\n" . $mech->response()->decoded_content . "\n\n######################";
	########################################################
	### We reload the lists because these are OBJECTS specific to the current page of $mech
	# Load a list of data templates
	($data_template_sel_items) = $mech -> find_all_inputs (
							type => 'option',
							name_regex => qr/^data_template_id$/,
						);
#	@data_template_sel{$data_template_sel_items->value_names} = $data_template_sel_items->possible_values;

	# Locate the needed data template in the list, and select it
#	$servkey = join "", grep m#^$cactiserv$#i, keys %data_template_sel;
#	$data_template_sel_items -> value ($data_template_sel{$servkey});
	$data_template_sel_items -> value ($data_templates{$cactiserv});

	# Load a list of hosts
	($host_sel_items) = $mech -> find_all_inputs (
					type => 'option',
					name_regex => qr/^host_id$/,
					);
	@host_sel{$host_sel_items->value_names} = $host_sel_items->possible_values;

	# Locate the needed host in the list, and select it
	my $hostkey = join "", grep m#^$host #i, keys %host_sel;
	$hostkey = join "", grep m# \(${host}\.#i, keys %host_sel if $hostkey eq '';
	$host_sel_items -> value ($host_sel{$hostkey});

	# Save the new data source with selected host and data template
	$mech -> submit_form;

	unless ($mech -> success()) {
		verbose 0, "Cacti: Failed to create a new data source by associating a host with data template. Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Created a new data source:\n" . $mech->response()->decoded_content . "\n\n######################";
	#####################################################################

	# Need to save another page having a path to the RRD file
	$mech -> submit_form (
		fields => {
			data_source_path => "${rrapath}/${host}_${serv}.rrd",
		},
	);
				
	unless ($mech -> success() && $mech->response()->decoded_content =~ m#Save Successful#) {
		verbose 0, "Cacti: Failed to save a new data source for template \"$cactiserv\" and host \"$host\". Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

#	$data_source_created{$cactiserv} = [] unless exists $data_source_created{$cactiserv};
#	push @{$data_source_created{$cactiserv}}, $host;

	verbose 2, "Cacti: Saved a new data source for template \"$cactiserv\" and host \"$host\":\n" . $mech->response()->decoded_content . "\n\n######################";

	verbose 1, "Cacti: Successfully created a new data source for host \"$host\" and service \"$cactiserv\"";
	return 0; # Successful creation of a new data source
}


# Check if the graph exists, and if not, create
sub g_check_create {
	my $data = shift;
	my $serv = shift;
	my $host = shift;
	my ($cactiserv, $uniqname);

	$uniqname = unpack ("L", md5(join "", sort keys %{$data->{perf}}));
	$cactiserv = "${serv}_${uniqname}";

	my (%graph_template_sel, %host_sel);
	my ($graph_template_sel_items, $host_sel_items, $g_templ_title);

	my $hst = join "", grep m#^${host}(|\.princeton\.edu)$#i, keys %cacti_hosts;  # look up hostname
	#$hst = grep m#^${host}$#i, keys %cacti_hosts if $hst eq '';  # look up description (might be different)
	#return 1 unless exists $cacti_hosts{$hst};   # Host could not be located in Cacti for some reason (unprobable since this was already checked)
	return 1 unless $hst;   # Host could not be located in Cacti for some reason (unprobable since this was already checked)

	my $host_ds_comp = get_cacti_host_ds_components $host;

	#my @host_graphs = `$php ${cacti_path}/cli/add_tree.php --list-graphs --host-id=$cacti_hosts{$hst}`;
	my $hst_grphs = get_cacti_graph $host;

	# Does the graph exist for the host?
	$g_templ_title = exists $data->{spec}->{graph_title} ? $data->{spec}->{graph_title} : $serv;
	#if (grep m#^\d+\s+\S+\s+\-\s+$g_templ_title\($uniqname\)\s+#i, @host_graphs) {
	if ($hst_grphs && exists $hst_grphs->{"${host} - ${g_templ_title}(${uniqname})"}) {
		verbose 1, "Cacti: Graph exists for host \"$host\" and service \"$serv\"";
		return 0;
	}

	verbose 1, "Cacti: Graph for host \"$host\" and service \"$serv\" could not be located. Will create";

	$mech -> get ("${cacti_url}/graphs.php?action=graph_edit&host_id=-1");

	unless ($mech -> success() && $mech->title() =~ m#Console -> Graph Management -> \(Edit\)#) {
		verbose 0, "Cacti: Failed to add a new graph for template \"$cactiserv\" and host \"$host\". Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Created a new graph for template \"$cactiserv\" and host \"$host\":\n" . $mech->response()->decoded_content . "\n\n######################";
	##################################################

	# Load a list of graph templates
	($graph_template_sel_items) = $mech -> find_all_inputs (
							type => 'option',
							name_regex => qr/^graph_template_id$/,
						);
#	@graph_template_sel{$graph_template_sel_items->value_names} = $graph_template_sel_items->possible_values;

	# Locate the needed graph template in the list, and select it
	my ($graphkey, $g_templ_name);

	$g_templ_name = $data->{spec}->{graph_template_name} . "_$cactiserv";
	$graphkey = $graph_templates{$g_templ_name};
	#$graphkey = join "", grep m#^${g_templ_name}$#i, keys %graph_template_sel;
	#$graph_template_sel_items -> value ($graph_template_sel{$graphkey});
	$graph_template_sel_items -> value ($graphkey);
	####################
	# Load a list of hosts
	($host_sel_items) = $mech -> find_all_inputs (
					type => 'option',
					name_regex => qr/^host_id$/,
					);
#	@host_sel{$host_sel_items->value_names} = $host_sel_items->possible_values;

	# Locate the needed host in the list, and select it
#	my $hostkey = join "", grep m#^$host #i, keys %host_sel;
#	$hostkey = join "", grep m# \(${host}\.princeton\.edu\)#i, keys %host_sel if $hostkey eq '';
#	$host_sel_items -> value ($host_sel{$hostkey});
	$host_sel_items -> value ($cacti_hosts{$host});

	# This is done solely to get the name of the host if the description differs from the hostname
#	$hostkey =~ m#^(\S+)#;
#	my $hostkey_temp = $1;

	###################
	# Save
	$mech -> submit_form;
				
	#unless ($mech -> success() && $mech->response()->decoded_content =~ m#${hostkey_temp} \- ${g_templ_title}\(${uniqname}\)#) {
	unless ($mech -> success() && $mech->response()->decoded_content =~ m#${host} \- ${g_templ_title}\(${uniqname}\)#) {
		verbose 0, "Cacti: Failed to save the new graph for graph template \"${g_templ_title}(${uniqname})\" and host \"$host\". Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Saved the new graph for graph template \"${g_templ_title}(${uniqname})\" and host \"$host\":\n" . $mech->response()->decoded_content . "\n\n######################";
	##########################################################

	# Now let's select (map) the relevant data sources to data template items of the graph

	my $curcont = join "", $mech -> response() -> decoded_content;
	my $ds_sel_items;
	my %ds_sel;
	#<tr id='row_task_item_id_310' bgcolor='#E5E5E5'>
	#<td width='50%'>
	#<font class='textEditTitle'>Data Source [home]</font><br>
	while ($curcont =~ m#<tr .* id='row_task_item_id_([^']+)[^\n]+\n[^\n]+[^>]+>Data Source \[([^\]]+)\]</font><br>#mg) {
		my ($item_id, $found_ds_name) = ($1, $2);

		($ds_sel_items) = $mech -> find_all_inputs (
						type => 'option',
						name_regex => qr/^task_item_id_${item_id}$/,
					);
		#@ds_sel{$ds_sel_items->value_names} = $ds_sel_items->possible_values;

		#my $dskey = join "", grep m#^${host} \- ${cactiserv} \($found_ds_name\)$#i, keys %ds_sel;

		#$ds_sel_items -> value ($ds_sel{$dskey});
		$ds_sel_items -> value ($host_ds_comp->{"${host} - ${cactiserv}(${found_ds_name})"});
	}

	$mech -> submit_form;
				
	unless ($mech -> success() && $mech->response()->decoded_content =~ m#Save Successful#) {
		verbose 0, "Cacti: Failed to save the components of the new graph for graph template \"${g_templ_name}_${cactiserv}\" and host \"$host\". Answer: " . $mech->status();
		verbose 2, "Response:\n" . $mech->response()->decoded_content;
		return 1;
	}

	verbose 2, "Cacti: Saved the components of the new graph for graph template \"${g_templ_name}_${cactiserv}\" and host \"$host\":\n" . $mech->response()->decoded_content . "\n\n######################";
	verbose 1, "Cacti: Successfully created a graph for host \"$host\"";

	return 0; # Graph creation successful
}

sub begin {
	# If we are not outputting to STDOUT, then we daemonize
	if ($outtype eq 'log') {
		my $pid;
		use POSIX qw(setsid);
		chdir '/';
		open STDIN, '/dev/null' or die "Cannot read \"/dev/null\"";
		open STDOUT, '>> /dev/null' or die "Cannot write to \"/dev/null\"";
		#open STDERR, '>> /dev/null' or die "Cannot write to \"/dev/null\"";
		open STDERR, '>> /usr/local/monitoring/log/cacti_update.err' or die "Cannot write to \"/usr/local/monitoring/log/cacti_update.err\"";
		defined ($pid = fork) or die "Cannot fork: $!";
		exit 0 if $pid;

		POSIX::setsid or die "Cannot start a new session";

		$SIG{PIPE} = 'ignore';

		$pidfile = File::Pid -> new ({file => $pidpath,});
		$pidfile -> write or die "Cannot write PID file: $!";
	}

	$SIG{HUP} = \&reload;   # SIGHUP causes configuration re-read
	$SIG{TERM} = \&end_program;  # SIGTERM causes graceful termination
	$SIG{INT} = \&end_program;  # CTRL+C causes graceful termination

	my $descr = load;

	# Nagios pipe read
	read_perfdata $descr->[0], $descr->[1];

	$pidfile -> remove if $outtype eq 'log';
}


# This routine does things that might later be reloaded on request
sub load {
	$pnag = Thread::Queue -> new;
	$pnagcac = Thread::Queue -> new;

	if ($outtype eq 'log') {
		open LOG, ">> $log_file" or die "Cannot open logfile \"$log_file\"";
	}

	verbose 0, "###### Program Started ###########\n";

	# Load performance history
	open PERS, "< $persistence_file" and do {
		foreach (<PERS>) {
			chomp;
			my ($hst, $svc) = split /::::/;
			$perfhist{$hst} = &share ({}) unless exists $perfhist{$hst};
			$perfhist{$hst}->{$svc} = 2;
		}
		close PERS;
	};

	my @spec_files = ($spec_file);

	verbose 1, "---- Reading RRD specification ----";

	$spec = Config::Any -> load_files ({ files => \@spec_files, use_ext => 1 });

	die ("Cannot read configuration file \"$spec_file\"") unless $spec;

	$spec = $spec->[0]->{$spec_file};

	my $rrd_thr = threads -> new (\&update_rrd);
	$rrd_thr -> detach;
	my $cac_thr = threads -> new (\&cacti_update);
	$cac_thr -> detach;

	return [ $rrd_thr, $cac_thr ];
}

# This routine terminates things that might then be reloaded, or the program can be terminated
sub unload {
	
	lock $waiton and $waiton = 0;

	# Clear performance history
	foreach (keys %perfhist) {
		delete $perfhist{$_};
	}
	%perfhist = ();

	verbose 0, "########## Program terminated ##########";

	close LOG;

	# This is a way to execute various external tasks before final termination, or reload
	# Things such as cleanup, and log rotation should go in this file
	open POSTTASKS, "< $close_file" and do {
		foreach (<POSTTASKS>) {
			chomp;
			`$_`;
		}
		close POSTTASKS;
	};

}

# The following will continuously read the perfdata Nagios pipe
sub read_perfdata {
	my ($rrd_thr, $cac_thr) = @_;
	my $qitems = $pnag -> pending;
	my $qincr = 0;
	my @rrd_threads;
	my $tries = 0;

#$LASTSERVICECHECK$\t$HOSTNAME$\t$SERVICEDESC$\t$SERVICEOUTPUT$\t$SERVICEPERFDATA$
	LP: while ($tries < 12) {
		open PERFDATA, "< $perf_file" or do {
			$tries++;
			sleep 10;
			next;
		};

		verbose (1, "Performance data pipe \"$perf_file\" successfully opened for reading");

		# The following should loop forever, since we're reading from a pipe
		# Termination should be done by a signal, such as SIGTERM
		# Each line contains perfdata info for a host/service
		while (<PERFDATA>) {
			$tries = 0;
			chomp;

			# Fast ignore timeouts
			next if m#\(Service Check Timed Out\)|ERROR: Alarm signal \(Nagios time-out|Timeout: No Response#;

			verbose (4, "Perfdata line read: #$_#\n");

			# If we've been commanded to finish
			last LP if (lock $terminate && $terminate);

			# Are we commanded to reload?
			{
				lock $waiton;
			if ($waiton) {
				if ($waiton == 1) {
					#$pnag -> insert (0, "Reload");
					#$pnagcac -> insert (0, "Reload");
					$pnag -> enqueue ("Reload");
					$pnagcac -> enqueue ("Reload");
					$waiton = 2;
				}
				if ($rrd_thr -> is_running || $cac_thr -> is_running) { verbose 1, "Waiting for the threads to complete..."; next; }
				unload;
				my $descr = load;
				($rrd_thr, $cac_thr) = @$descr;
			}
			}

			# Queue is a bit full. If the RRD thread died, reload the whole thing
			# otherwise, wait a bit for items to be processed
			if ($pnag -> pending > 1000) {
				unless ($rrd_thr -> is_running) {
					verbose 0, "RRD Thread died for some reason! Reloading";

					lock $waiton; $waiton = 1;
				} else {
					verbose 1, "WARNING: queue size is " . $pnag -> pending;
					sleep 240;
					next;
				}
			}

			$pnag -> enqueue ("$_");

		}

		close PERFDATA;
		#	open PERFDATA, "< $perf_file" or graceful_die (0, 1, "Unable to open the perfdata file \"$perf_file\"!!!");

		verbose (1, "Performance data pipe \"$perf_file\" closed");

		$tries++;
	}

	close PERFDATA;
	verbose (1, "Performance data pipe \"$perf_file\" closed");
#sleep 60;
	#$pnag -> insert (0, "Terminate");
	#$pnagcac -> insert (0, "Terminate");
	$pnag -> enqueue ("Terminate");
	$pnagcac -> enqueue ("Terminate");

	# If one of the threads is still running, just display a message
	while ($rrd_thr -> is_running || $cac_thr -> is_running) { verbose 1, "Waiting for the threads to complete..."; sleep 10; }

	unload;
}

sub update_rrd {
	my %perfdata;
	my $rrd_thr_wait = 0;

	verbose (1, "------ Update RRD thread started ------");

	while ( $_ = $pnag -> dequeue ) {
		verbose (4, "Update RRD dequeued: ##$_##");

		last if /^(Terminate|Reload)$/;

		my ($tm, $hst, $svc, $svcout, $svcperf) = split /\t/;

		next unless defined $tm && defined $hst && defined $svc && defined $svcout && defined $svcperf;
#$svcperf = $svcout if not defined $svcperf || $svcperf eq '';    ##### need to put in possibility of regex parsing the output if needed. This will require some regex matching in create_ spec

		verbose (3, "Perfdata line parsed: ####$_#####$tm###$svc###$hst###$svcout###$svcperf###");

		# This skips perfdata lines not matching the spec - don't care about these
		next if $svcperf =~ /^\s*$/i
			|| ! grep ($svc =~ m#$_#i, keys %$spec)
			|| ! grep { my $svtmp = $_; $svc =~ m#$svtmp#i and do {
				grep { $hst =~ m#$_#i || $_ eq '^self^' } keys %{$spec->{$svtmp}}
					};
				} keys %$spec;
		$svc =~ s/\s+/_/g;

		# change to 2 later!!!
		verbose (1, "Perfdata line matching spec: ###$svc###$hst###$svcout###$svcperf###; Queue size: " . $pnag->pending);

		$perfdata{$hst} = {} unless exists $perfdata{$hst};
		$perfdata{$hst}->{$svc} = {} unless exists $perfdata{$hst}->{$svc};
		my $curperf = $perfdata{$hst}->{$svc};
		$curperf->{perf} = {};
		$curperf->{time} = $tm;

		# Each perfdata line might carry perf output (separated by space) for several items that we need to parse
		# 'label'=value[UOM];[warn];[crit];[min];[max]  -- per rrdtool docs, label should be [a-zA-Z0-9_]+, 1-19chars long
		while ($svcperf =~ m#'?([^'=]*)'?=([\S]+)\s*#g) {
#verbose 1, "################$&################\n";
			my ($lbl, $rest) = ($1, $2);
			next unless defined $rest;

			my $graphlabel = $lbl;
			$lbl =~ s/[^a-zA-Z0-9_]//g;
			$lbl = substr $lbl, 0, 19;
			my ($val, $wrn, $crt, $min, $max) = split /;/, $rest;
			foreach (\$val, \$wrn, \$crt, \$min, \$max) { $$_ = 'U' unless defined $$_ && $$_ !~ /^\s*$/; }
			$val =~ s/[^0-9\.\-]//g;

			if (exists $curperf->{perf}->{$lbl}) { $lbl = "_$lbl"; }
			$curperf->{perf}->{$lbl} = [];
			push @{$curperf->{perf}->{$lbl}}, $val, $wrn, $crt, $min, $max, $graphlabel;

			# change to 2 later!!!
			verbose (1, "Perfdata deciphered: ###label=$lbl###val=$val###graphlabel=$graphlabel###");
		}

		# Link the relevant part of the spec hash to the %perfdata for further use
		# For every service specification, remove ignored items, and map items to map
		foreach my $svcspec (keys %$spec) {
			next unless $svc =~ m#${svcspec}#;

			# For every host specification
			foreach my $hostspec (keys %{$spec->{$svcspec}}) {
				next unless ($hst =~ /^${hostspec}$/i || $hostspec eq '^self^');

				my $curspec = $spec->{$svcspec}->{$hostspec};
				$curperf->{spec} = $curspec;

				# Remove perfdata DS's from the ignore map
				exists $curspec->{ignore_items} and do {
					foreach (split (/\s*,\s*/, $curspec->{ignore_items})) {
						next unless exists $curperf->{perf}->{$_};
#print Dumper ($curperf->{perf}->{$_});
						delete $curperf->{perf}->{$_};
					}
				};

				# Map DS names if needed
				exists $curspec->{map_items} and do {
					foreach (keys %{$curspec->{map_items}}) {
						next unless exists $curperf->{perf}->{$_};
						$curperf->{perf}->{$curspec->{map_items}->{$_}} = $curperf->{perf}->{$_};
						delete $curperf->{perf}->{$_};
					}
				};

				# This service is probably useless, since it yields no stats that we want to graph according to the spec
				if (keys %{$curperf->{perf}} == 0) {
					delete $curperf->{perf};
					delete $perfdata{$hst}->{$svc};
					delete $perfdata{$hst} unless keys %{$perfdata{$hst}} > 0;
				}

				#/maestro=2196MB;2406;2706;0;3007
				#/cognos/iowa=68% /cognos/ohio=53%

				#/usr/local/groundwork/common/bin/rrdtool create \
				#/var/local/cacti-0.8.7h/rra/sdpadfs300w_avgdiskreadpersec_9866.rrd \
				#--step 300  \
				#DS:AvgDiskWriteQueueLe:GAUGE:600:0:100 \
				#DS:AvgDiskReadQueueLen:GAUGE:600:0:100 \
				#DS:PercentFreeSpace:GAUGE:600:0:100 \
				#DS:PercentDiskWriteTim:GAUGE:600:0:100 \
				#DS:PercentDiskReadTime:GAUGE:600:0:100 \
				#DS:AvgDiskReadPerSec:GAUGE:600:0:U \
				#DS:AvgDiskWritePerSec:GAUGE:600:0:U \
				#DS:FreeMegabytes:GAUGE:600:0:U \
				#RRA:AVERAGE:0.5:1:500 \
				#RRA:AVERAGE:0.5:1:600 \
				#RRA:AVERAGE:0.5:6:700 \
				#RRA:AVERAGE:0.5:24:775 \
				#RRA:AVERAGE:0.5:288:797 \
				#RRA:MAX:0.5:1:500 \
				#RRA:MAX:0.5:1:600 \
				#RRA:MAX:0.5:6:700 \
				#RRA:MAX:0.5:24:775 \
				#RRA:MAX:0.5:288:797 \
				#

				# rrdcreate?
				# rrdtool create filename [--start|-b start time] [--step|-s step] [--no-overwrite] [DS:ds-name:DST:dst arguments] [RRA:CF:cf arguments]
				# If the RRD data file for the current data set does not exist, create it
				if (! -f "${rrapath}/${hst}_${svc}.rrd") {
#print "###", keys %{$perfdata{$hst}->{$svc}}, "###\n";
					my @create_item_regexs = keys %{$curspec->{create_items}};
					my $ds_create_spec = '';

					foreach my $dsitem (keys %{$curperf->{perf}}) {
						my $create_item_spec_match = join "", grep { $dsitem =~ /^$_$/i } @create_item_regexs;
						$ds_create_spec .= ($create_item_spec_match eq '') ?
							"DS:$dsitem:$curspec->{create_items}->{$create_item_spec_match}->{type}:600:$curspec->{create_items}->{$create_item_spec_match}->{min}:$curspec->{create_items}->{$create_item_spec_match}->{max} " :
							"DS:$dsitem:GAUGE:600:0:U ";
					}
#print "####$ds_create_spec####\n";
					chomp $ds_create_spec;
					my @rrdout = `$rrdtool create ${rrapath}/${hst}_${svc}.rrd --step 300 $ds_create_spec $rracacspec 2>&1`;
					verbose (1, "Create RRD: $rrdtool create ${rrapath}/${hst}_${svc}.rrd --step 300 $ds_create_spec $rracacspec\nResponse: " . join ("\n", @rrdout));
				}

				# rrdupdate
				# rrdtool {update | updatev} filename [--template|-t ds-name[:ds-name]...] [--daemon address] [--] N|timestamp:value[:value...] at-timestamp@value[:value...] [timestamp:value[:value...] ...]
				my $valsupdate = join ":", (map { $curperf->{perf}->{$_}->[0]; } keys %{$curperf->{perf}});
				my @rrdout = `$rrdtool update ${rrapath}/${hst}_${svc}.rrd $curperf->{time}:$valsupdate 2>&1`;
				verbose (1, "Update RRD: $rrdtool update ${rrapath}/${hst}_${svc}.rrd $curperf->{time}:$valsupdate\nResponse: " . join ("\n", @rrdout));

#next;
				# Record processed data
				# 0 - initial unprocessed, 1 - error, try again, 2 - successfully processed previously
				lock %perfhist;
				$perfhist{$hst} = &share ({}) unless exists $perfhist{$hst};
				lock %{$perfhist{$hst}};
				unless (exists $perfhist{$hst}->{$svc} && $perfhist{$hst}->{$svc} != 1) {
					$perfhist{$hst}->{$svc} = 0;   # Initially mark it as unprocessed by Cacti
					$pnagcac -> enqueue ("Next service");
					$pnagcac -> enqueue ("Host: ${hst}");
					$pnagcac -> enqueue ("Service: ${svc}");
					$pnagcac -> enqueue ("Host Spec: ${hostspec}");
					$pnagcac -> enqueue ("Service Spec: ${svcspec}");
					foreach my $lbl (keys %{$curperf->{perf}}) {
						$pnagcac -> enqueue ("Perf: $lbl");
						$pnagcac -> enqueue (join "::: ", @{$curperf->{perf}->{$lbl}});
					}
					$pnagcac -> enqueue ("End service");
				}

			} # End foreach hostspec
		} # End foreach svcspec

	} # End while (1)

	verbose 1, "Pending queue items: " . $pnag->pending();

	verbose (1, "------ Update RRD thread terminated ------");

	return 0;
}

sub verbose {
	my $level = shift;
	my $string = shift;

#		print STDERR "### ", $string, "\n" if $verbose && $verbose >= $level;
	if ($outtype eq 'log') {
		my $dt = `date +"%a %b %d %T"`; chomp $dt;
		print LOG "$dt => $string", "\n" if $verbose && $verbose >= $level;
	} elsif ($outtype eq 'stderr') {
		print STDERR "### ", $string, "\n" if $verbose && $verbose >= $level;
	}
}

sub graceful_die {
	my $immediate = shift;
	my $state = shift;
	my $str = shift;
	verbose 0, $str if $str && $str ne '';

	if ($immediate) {
		if ($outtype eq 'log') {
			close LOG;
			$pidfile -> remove;
		}
		exit $state;
	} else {
		lock $terminate;
		$terminate = 1;
	}
}

sub reload {
	verbose (0, "SIGHUP caught. Reloading configuration");
	lock $waiton;
	$waiton = 1 unless $waiton;   # If waiton is already non-zero - we're in the process of reloading, so DND
}

sub end_program {
	verbose (0, "Program terminate requested by user");
	lock $terminate;
	$terminate = 1;
}

# Args: cols, tables, conditions
sub DB_get {
	my $colnames_p = shift; 
	my @colnames = split /\s*,\s*/, $colnames_p;

	my $tblnames_p = shift;
	my @tblnames = split /\s*,\s*/, $tblnames_p;

	my $cond_p = shift;

	my %res;

	my $query = "SELECT $colnames_p FROM $tblnames_p";
	$query .= " WHERE $cond_p" if $cond_p;
verbose 1, "Cacti: ###$query###";
	my $sth = $cacdb -> prepare ($query);
	graceful_die 1, 1, "Error initializing DB query \"$query\". Received: " . $cacdb->errstr unless $sth;
#verbose 0, "\n###$query prepared\n";
	my $ref;
	graceful_die 1, 1, "Error executing DB query \"$query\". Received: " . $cacdb->errstr unless $sth -> execute;
#verbose 0, "\n###$query executed\n";

	while ($ref = $sth->fetchrow_hashref()) {
#verbose 0, (Dumper ($ref));
#verbose 0, "#########Next row\n";
		foreach (@colnames) {
			s#.*\.##;
#verbose 0, "#########$_##\n";
		#	$res{"$_"} = [] unless exists $res{"$_"};
		#	push @{$res{"$_"}}, $ref -> {"$_"};
			push @{$res{$_}||=[]}, $ref->{$_};
#verbose 0, "####" . $ref -> {"$_"} . "####\n";
		}
	}
#verbose 0, Dumper (\%res);
#verbose 0, "#######Done\n";
	$sth -> finish;

	return \%res;
}

sub get_cacti_data_templates {
	my $res = DB_get ("id, name", "data_template");
	return undef unless keys %$res > 0;

	my %res2;
#print Dumper ($res);
	for (my $ctr = 0; $ctr < @{$res->{name}}; $ctr++) {
		$res2{$res->{name}->[$ctr]} = $res->{id}->[$ctr];
	}

	return \%res2;
}

sub get_cacti_data_template_component_id {
	my $templ_name = shift;
	my $comp_name = shift;

#select data_template_rrd.id, data_template.name, data_template_rrd.data_source_name from data_template,data_template_data,data_template_rrd where data_template_rrd.data_template_id=data_template.id and data_template_data.data_template_id=data_template.id and data_template_data.local_data_id=0 and data_template_rrd.local_data_id=0;

	my $res = DB_get ("data_template_rrd.id", "data_template, data_template_data, data_template_rrd", "data_template_rrd.data_template_id=data_template.id AND data_template_data.data_template_id=data_template.id AND data_template_data.local_data_id='0' AND data_template_rrd.local_data_id='0' AND data_template.name='" . $templ_name . "' AND data_template_rrd.data_source_name='" . $comp_name . "'");

	return keys %$res == 1 ? $res -> {id} : undef;
}

#sub get_cacti_data_sources {
#	my $res = DB_get ("id, name", "graph_templates");
#	return undef unless keys %$res > 0;

#	my %res2;
#print Dumper ($res);
#	for (my $ctr = 0; $ctr < @{$res->{name}}; $ctr++) {
#		$res2{$res->{name}->[$ctr]} = $res->{id}->[$ctr];
#	}

#	return \%res2;
#}

sub get_cacti_host_template_ds {
	my $host_name = shift;
	my $templ_name = shift;

	my $res = DB_get ("data_template_data.local_data_id", "data_local, data_template_data", "data_local.id=data_template_data.local_data_id AND data_local.host_id='" . $cacti_hosts{$host_name} . "' AND data_template_data.data_template_id='" . $data_templates{$templ_name} . "'");

	return keys %$res == 1 ? $res->{local_data_id} : undef;
}

sub get_cacti_host_ds_components {
	my $host_name = shift;

	my $res = DB_get ("data_template_rrd.id, data_source_name, name_cache", "data_template_data, data_template_rrd, data_local", "host_id='" . $cacti_hosts{$host_name} . "' and data_template_rrd.local_data_id=data_local.id and data_local.id=data_template_data.local_data_id");
	return undef unless keys %$res > 0;

	my %res2;
#print Dumper ($res);
	for (my $ctr = 0; $ctr < @{$res->{id}}; $ctr++) {
		$res2{"$res->{name_cache}->[$ctr]" . "($res->{data_source_name}->[$ctr])"} = $res->{id}->[$ctr];
	}

	return \%res2;
}

sub get_cacti_graph_templates {
	my $res = DB_get ("id, name", "graph_templates");
	return undef unless keys %$res > 0;

	my %res2;
#print Dumper ($res);
	for (my $ctr = 0; $ctr < @{$res->{name}}; $ctr++) {
		$res2{$res->{name}->[$ctr]} = $res->{id}->[$ctr];
	}

	return \%res2;
}

sub get_cacti_graph_template_component {
	my $templ_name = shift;

	my $res = DB_get ("name", "graph_template_input", "graph_template_id='" . $graph_templates{$templ_name} . "'");
	return keys %$res == 0 ? undef : $res->{name}->[0];
}

sub get_cacti_graph {
	my $host_name = shift;

	my $res = DB_get ("graph_templates_graph.local_graph_id, graph_templates_graph.title_cache", "graph_local, graph_templates_graph", "graph_local.id=graph_templates_graph.local_graph_id AND graph_local.host_id='" . $cacti_hosts{$host_name} . "'");

	return undef unless exists $res->{local_graph_id};

	my %res2;
#print Dumper ($res);
	for (my $ctr = 0; $ctr < @{$res->{local_graph_id}}; $ctr++) {
		$res2{$res->{title_cache}->[$ctr]} = $res->{local_graph_id}->[$ctr];
	}

	return \%res2;
}

sub get_cacti_hosts {
	my $res = DB_get ("id, description, hostname", "host");
	return undef unless keys %$res > 0;

	my %res2;
#print Dumper ($res);
	for (my $ctr = 0; $ctr < @{$res->{id}}; $ctr++) {
		$res2{$res->{hostname}->[$ctr]} = $res->{id}->[$ctr];
		$res2{$res->{description}->[$ctr]} = $res->{id}->[$ctr];
	}

	return \%res2;
}

