NMS: Rewrite the SNMP fetcher and whatnot

Again.

It's using the prototype, and the objects are configurable. I assume this
will need some work.

Also: I touched ping up to make it work without linknets.

6 files changed, 80 insertions, 92 deletions

diff --git a/clients/ping.pl b/clients/ping.pl
index 8d216ef..80c389a 100755
--- a/clients/ping.pl
+++ b/clients/ping.pl
@@ -13,7 +13,7 @@ my $dbh = nms::db_connect();
 $dbh->{AutoCommit} = 0;
 $dbh->{RaiseError} = 1;
 
-my $q = $dbh->prepare("SELECT switch,ip,secondary_ip FROM switches WHERE ip<>'127.0.0.1'");
+my $q = $dbh->prepare("SELECT switch,ip,secondary_ip FROM switches WHERE ip is not null");
 my $lq = $dbh->prepare("SELECT linknet,addr1,addr2 FROM linknets");
 
 while (1) {
@@ -36,7 +36,6 @@ while (1) {
 			$ping->host_add($secondary_ip);
 			$secondary_ip_to_switch{$secondary_ip} = $switch;
 		}
-		print "ip: $ip\n";
 	}
 	my $result = $ping->ping();
 	die $ping->get_error if (!defined($result));
@@ -73,18 +72,19 @@ while (1) {
 		$ping->host_add($ref->{'addr1'});
 		$ping->host_add($ref->{'addr2'});
 	}
-	$result = $ping->ping();
-	die $ping->get_error if (!defined($result));
+	if (@linknets) { 
+		$result = $ping->ping();
+		die $ping->get_error if (!defined($result));
 
-	$dbh->do('COPY linknet_ping (linknet, latency1_ms, latency2_ms) FROM STDIN');  # date is implicitly now.
-	for my $linknet (@linknets) {
-		my $id = $linknet->{'linknet'};
-		my $latency1 = $result->{$linknet->{'addr1'}} // '\N';
-		my $latency2 = $result->{$linknet->{'addr2'}} // '\N';
-		$dbh->pg_putcopydata("$id\t$latency1\t$latency2\n");
+		$dbh->do('COPY linknet_ping (linknet, latency1_ms, latency2_ms) FROM STDIN');  # date is implicitly now.
+			for my $linknet (@linknets) {
+				my $id = $linknet->{'linknet'};
+				my $latency1 = $result->{$linknet->{'addr1'}} // '\N';
+				my $latency2 = $result->{$linknet->{'addr2'}} // '\N';
+				$dbh->pg_putcopydata("$id\t$latency1\t$latency2\n");
+			}
+		$dbh->pg_putcopyend();
 	}
-	$dbh->pg_putcopyend();
 	$dbh->commit;
-	
 }
 
diff --git a/clients/snmpfetchng.pl b/clients/snmpfetchng.pl
index fa7c227..5d1fd42 100755
--- a/clients/snmpfetchng.pl
+++ b/clients/snmpfetchng.pl
@@ -3,12 +3,19 @@ use strict;
 use warnings;
 use DBI;
 use POSIX;
-use Time::HiRes;
+#use Time::HiRes qw(time);
 use SNMP;
 use Data::Dumper;
 use lib '../include';
 use nms;
 
+SNMP::addMibDirs("/tmp/tmp.esQYrkg9MW/v2");
+SNMP::loadModules('SNMPv2-MIB');
+SNMP::loadModules('ENTITY-MIB');
+SNMP::loadModules('IF-MIB');
+SNMP::loadModules('LLDP-MIB');
+SNMP::loadModules('IP-MIB');
+SNMP::loadModules('IP-FORWARD-MIB');
 our $dbh = nms::db_connect();
 $dbh->{AutoCommit} = 0;
 $dbh->{RaiseError} = 1;
@@ -17,7 +24,7 @@ $dbh->{RaiseError} = 1;
 
 my $qualification = <<"EOF";
 (last_updated IS NULL OR now() - last_updated > poll_frequency)
-AND (locked='f' OR now() - last_updated > '5 minutes'::interval)
+AND (locked='f' OR now() - last_updated > '15 minutes'::interval)
 AND ip is not null
 EOF
 
@@ -28,13 +35,11 @@ SELECT
   DATE_TRUNC('second', now() - last_updated - poll_frequency) AS overdue
 FROM
   switches
-  NATURAL LEFT JOIN switchtypes
 WHERE
 $qualification
 ORDER BY
-  priority DESC,
   overdue DESC
-LIMIT 10
+LIMIT ?
 FOR UPDATE OF switches
 EOF
 	or die "Couldn't prepare qswitch";
@@ -44,8 +49,10 @@ our $qunlock = $dbh->prepare("UPDATE switches SET locked='f', last_updated=now()
 	or die "Couldn't prepare qunlock";
 my @switches = ();
 
-our $temppoll = $dbh->prepare("INSERT INTO switch_temp (switch,temp,time) VALUES((select switch from switches where sysname = ?),?,now())")
-	or die "Couldn't prepare temppoll";
+my $sth = $dbh->prepare("INSERT INTO snmp (switch,data) VALUES((select switch from switches where sysname=?), ?)");
+
+our $outstanding = 0;
+
 sub mylog
 {
 	my $msg = shift;
@@ -57,7 +64,15 @@ sub mylog
 sub populate_switches
 {
 	@switches = ();
-	$qswitch->execute()
+	my $limit = $nms::config::snmp_max - $outstanding;
+	if ($limit < 0) {
+		mylog("Something wrong. Too many outstanding polls going.");
+		$limit = 1;
+	}
+	if ($outstanding > 0) {
+		mylog("Outstanding polls: $outstanding . Current limit: $limit");
+	}
+	$qswitch->execute($limit)
 		or die "Couldn't get switch";
 	
 	while (my $ref = $qswitch->fetchrow_hashref()) {
@@ -68,103 +83,72 @@ sub populate_switches
 			'community' => $ref->{'community'}
 		};
 	}
-		$dbh->commit;
+	$dbh->commit;
 }
 
 sub inner_loop
 {
-	mylog("Starting run");
 	populate_switches();
+	my $poll_todo = "";
 	for my $refswitch (@switches) {
+		$outstanding++;
 		my %switch = %{$refswitch};
-		mylog( "START: Polling $switch{'sysname'} ($switch{'mgtip'}) ");
+		$poll_todo .= "$switch{'sysname'} ($switch{'mgtip'}) ";
 
 		$switch{'start'} = time;
 		$qlock->execute($switch{'id'})
 			or die "Couldn't lock switch";
 		$dbh->commit;
-		my $s = new SNMP::Session(DestHost => $switch{'mgtip'},
+		my $s = SNMP::Session->new(DestHost => $switch{'mgtip'},
 					  Community => $switch{'community'},
+					  UseEnums => 1,
 					  Version => '2');
-		my @vars = ();
-		push @vars, [ "sysName", 0];
-		push @vars, [ "sysDescr", 0];
-		push @vars, [ "1.3.6.1.4.1.2636.3.1.13.1.7.7.1.0", 0];
-		my $varlist = SNMP::VarList->new(@vars);
-		$s->get($varlist, [ \&ckcall, \%switch ]);
-		$s->gettable('ifXTable',callback => [\&callback, \%switch]);
+		$s->bulkwalk(0, 10, @nms::config::snmp_objects, sub{ callback(\%switch, @_); });
 	}
-	mylog( "Added " . @switches . " ");
+	mylog( "Polling " . @switches . " switches: $poll_todo");
 	SNMP::MainLoop(5);
 }
 
-sub ckcall
-{
+sub callback{
+	my @top = $_[1];
 	my %switch = %{$_[0]};
-
-	my $vars = $_[1];
-	my ($sysname,$sysdescr,$temp) = (undef,undef,undef);
-	for my $var (@$vars) {
-		if ($var->[0] eq "sysName") {
-			$sysname = $var->[2];
-		} elsif ($var->[0] eq "sysDescr") {
-			$sysdescr = $var->[2];
-		} elsif ($var->[0] eq "enterprises.2636.3.1.13.1.7.7.1.0.0") {
-			$temp = $var->[2];
+	my %tree;
+	my %ttop;
+	my %nics;
+	my @nicids;
+
+	for my $ret (@top) {
+		for my $var (@{$ret}) {
+			for my $inner (@{$var}) {
+				my ($tag,$type,$name,$iid, $val) = ( $inner->tag ,$inner->type , $inner->name, $inner->iid, $inner->val);
+				if ($tag eq "ifPhysAddress") {
+					next;
+				}
+				$tree{$iid}{$tag} = $val;
+				if ($tag eq "ifIndex") {
+					push @nicids, $iid;
+				}
+			}
 		}
 	}
-	if (defined $temp && $temp =~ /^\d+$/) {
-		$temppoll->execute($switch{'sysname'},$temp);
-	} else {
-		warn "Couldn't read temp for " . $switch{'sysname'} . ", got " . (defined $temp ? $temp : "undef");
-	}
-	$dbh->commit;
-}
-my @values = ('ifName','ifHighSpeed','ifHCOutOctets','ifHCInOctets');
-my $query = "INSERT INTO polls (switch,time";
-foreach my $val (@values) {
-	$query .= ",$val";
-}
-$query .= ") VALUES(?,timeofday()::timestamp";
-foreach my $val (@values) {
-	$query .= ",?";
-}
-$query .= ");";
-
-our $qpoll = $dbh->prepare($query)
-	or die "Couldn't prepare qpoll";
-sub callback
-{
-	my %switch = %{$_[0]};
-	my $table = $_[1];
-
-	my %ifs = ();
 
-	foreach my $key (keys %{$table}) {
-		my $descr = $table->{$key}->{'ifName'};
-
-		if ($descr =~ m/(ge|xe|et)-/ && $descr !~ m/\./) {
-			$ifs{$descr} = $table->{$key};
-		}
+	my %tree2;
+	for my $nic (@nicids) {
+		$tree2{'ports'}{$tree{$nic}{'ifName'}} = $tree{$nic};
+		delete $tree{$nic};
 	}
-
-
-	foreach my $key (keys %ifs) {
-		my @vals = ();
-		foreach my $val (@values) {
-			if (!defined($ifs{$key}{$val})) {
-				die "Missing data";
-			}
-			push @vals, $ifs{$key}{$val};
+	for my $iid (keys %tree) {
+		for my $key (keys %{$tree{$iid}}) {
+			$tree2{'misc'}{$key}{$iid} = $tree{$iid}{$key};
 		}
-		$qpoll->execute($switch{'id'},@vals) || die "ops";
 	}
-	mylog( "STOP: Polling $switch{'sysname'} took " . (time - $switch{'start'}) . "s");
+	$sth->execute($switch{'sysname'}, JSON::XS::encode_json(\%tree2));
+	mylog( "Polled $switch{'sysname'} in " . (time - $switch{'start'}) . "s. ($outstanding outstanding polls)");
 	$qunlock->execute($switch{'id'})
 		or die "Couldn't unlock switch";
 	$dbh->commit;
+	$outstanding--;
 }
-print $query . "\n";
 while (1) {
 	inner_loop();
 }
diff --git a/include/config.pm.dist b/include/config.pm.dist
index cce82df..382b4e2 100755
--- a/include/config.pm.dist
+++ b/include/config.pm.dist
@@ -10,8 +10,10 @@ our $db_host = "gerald.tg15.gathering.org";
 our $db_username = "nms";
 our $db_password = "<removed>";
 
-# NMS hash used for public NMS obfuscation of interface names
-our $nms_hash = "<removed>";
+# NMS: What SNMP objects to fetch.
+our @snmp_objects = [['ifTable'], ['ifXTable'], ['sysDescr'], ['sysName'],['ipForward']];
+# Max SNMP polls to fire off at the same time.
+our $snmp_max = 20;
 
 # DHCP-servers
 our $dhcp_server1 = "185.12.59.66"; # primary
diff --git a/include/nms.pm b/include/nms.pm
index 6a9598e..2ec922b 100755
--- a/include/nms.pm
+++ b/include/nms.pm
@@ -26,7 +26,7 @@ sub db_connect {
 
 	my $dbh = DBI->connect($connstr,
 	                       $nms::config::db_username,
-	                       $nms::config::db_password)
+	                       $nms::config::db_password, {AutoCommit => 0})
 	        or die "Couldn't connect to database";
 	return $dbh;	
 }
diff --git a/include/nms/web.pm b/include/nms/web.pm
index 0f37a59..2a336c6 100755
--- a/include/nms/web.pm
+++ b/include/nms/web.pm
@@ -59,13 +59,14 @@ sub setwhen {
 		$now = db_safe_quote('now') . "::timestamp ";
 		$cc{'max-age'} = "3600";
 	}
-	$when = " time > " . $now . " - '5m'::interval and time < " . $now . " ";
+	$when = " time > " . $now . " - '15m'::interval and time < " . $now . " ";
 	return $when;
 }
 
 sub finalize_output {
 	my $query;
 	my $hash = Digest::SHA::sha512_base64(FreezeThaw::freeze(%json));
+	$dbh->commit;
 	$query = $dbh->prepare('select to_char(' . $now . ', \'YYYY-MM-DD"T"HH24:MI:SS\') as time;');
 	$query->execute();
 
diff --git a/web/nms.gathering.org/js/nms.js b/web/nms.gathering.org/js/nms.js
index ec7ed06..0484ffa 100644
--- a/web/nms.gathering.org/js/nms.js
+++ b/web/nms.gathering.org/js/nms.js
@@ -457,6 +457,7 @@ function initNMS() {
 	nmsData.registerSource("switchstate","/api/public/switch-state");
 
 	// Private	
+	nmsData.registerSource("snmp","/api/private/snmp");
 	nmsData.registerSource("portstate","/api/private/port-state");
 	nmsData.registerSource("comments", "/api/private/comments");
 	nmsData.registerSource("smanagement","/api/private/switches-management");