tools: redo discovery in the form of lolwhat

what?

lol...

just figure it out for me.

Ok, so the tool isn't entirely done, but it departs from the simple
lldpdiscver.pl method that was very limited when things weren't working as
they should.

The long-term idea of lolwhat is that it should be able to do all sorts of
discovery. Right now it just takes a list of IPs and figures out how things
are connected by SNMP. It uses LLDP info from SNMP, but it does not
_currently_ traverse new hosts based on what it finds.

4 files changed, 467 insertions, 14 deletions

diff --git a/extras/tools/lldp/dotnet.sh b/extras/tools/lldp/dotnet.sh
index 5c1b369..40e5024 100755
--- a/extras/tools/lldp/dotnet.sh
+++ b/extras/tools/lldp/dotnet.sh
@@ -1,9 +1,8 @@
 #!/usr/bin/env bash
-
-DATE="$(date +%s)"
-if [ -z "$1" ] || [ -z "$2" ]; then
-	echo "Usage: $0 <ip> <community>"
-	exit 1;
-fi
-./lldpdiscover.pl $1 $2 | ./draw-neighbors.pl | dot -Tpng > dotnet-${DATE}.png
-echo File name: dotnet-${DATE}.png
+set -xe
+DATE="$(date --iso-8601=second | sed s/:/./g)"
+JSON=lolwhat-${DATE}.json
+./lolwhat.pl $* > ${JSON}
+./draw-neighbors.pl < ${JSON} | dot -Tpng > lolwhat-${DATE}.png
+./draw-neighbors.pl full < ${JSON} | dot -Tpng > lolwhat-${DATE}-full.png
+echo File name: lolwhat-${DATE}*png
diff --git a/extras/tools/lldp/draw-neighbors.pl b/extras/tools/lldp/draw-neighbors.pl
index 323e676..0a3f9d0 100755
--- a/extras/tools/lldp/draw-neighbors.pl
+++ b/extras/tools/lldp/draw-neighbors.pl
@@ -4,32 +4,55 @@ use strict;
 use JSON;
 
 my $in;
+my ($full) = @ARGV;
 while (<STDIN>) {
 	$in .= $_;
 }
 
 my %assets = %{JSON::XS::decode_json($in)};
+my %map = %{$assets{hood}};
+my %map2 = %{$assets{extended}};
 
 print "strict graph network {\n";
-while (my ($key, $value) = each %assets) {
+while (my ($key, $value) = each %map) {
 	print_tree ($key,0,undef);
 }
+if ($full eq "full") {
+while (my ($key, $value) = each %map2) {
+	print_tree2 ($key,0,undef);
+}}
 print "}\n";
 
 sub print_tree
 {
-	my ($chassis_id,$indent,$parent,$max) = @_;
+	my ($name,$indent,$parent,$max) = @_;
 	if (!defined($parent)) {
 		$parent = "";
 	}
 	if ($indent > 50) {
 		die "Possible loop detected.";
 	}
-	print " \"$assets{$chassis_id}{sysName}\" -- {";
+	print " \"$name\" -- {";
 	my @n;
-	while (my ($key, $value) = each %{$assets{$chassis_id}{neighbors}}) {
-		push @n, "\"$assets{$key}{sysName}\"";
+	while (my ($key, $value) = each %{$map{$name}}) {
+		push @n, "\"$key\"";
 	}
 	print join(",",@n) . "};\n";
 }
 
+sub print_tree2
+{
+	my ($name,$indent,$parent,$max) = @_;
+	if (!defined($parent)) {
+		$parent = "";
+	}
+	if ($indent > 50) {
+		die "Possible loop detected.";
+	}
+	print " \"$name\" -- {";
+	my @n;
+	while (my ($key, $value) = each %{$map2{$name}}) {
+		push @n, "\"$key\"";
+	}
+	print join(",",@n) . "};\n";
+}
diff --git a/extras/tools/lldp/lldpdiscover.pl b/extras/tools/lldp/lldpdiscover.pl
index d6f2992..e5ac46f 100755
--- a/extras/tools/lldp/lldpdiscover.pl
+++ b/extras/tools/lldp/lldpdiscover.pl
@@ -36,6 +36,7 @@ use nms::snmp;
 
 # Actual assets detected, indexed by chassis ID
 my %assets;
+my %arp;
 
 # Tracking arrays. Continue scanning until they are of the same length.
 my @chassis_ids_checked;
@@ -77,7 +78,40 @@ if (defined($cmdline_ip) && defined($cmdline_community)) {
 		       push @chassis_ids_checked,$id;
 	       }
 	}
-	print JSON::XS::encode_json(\%assets);
+	while (my ($mac, $entry) = each %arp) {
+		if (!defined($entry->{'neighbors'})) {
+			delete $arp{$mac};
+			next;
+		}
+		my $sysn = $entry->{sysName};
+		for my $aentry (@{$entry->{'neighbors'}}) {
+			if (defined($entry->{chassis_id}) and $aentry->{origin} eq $entry->{chassis_id}) {
+				next;
+			}
+                        if ($aentry->{ipNetToMediaType} eq "static") {
+				next;
+			}
+			$arp{$mac}{ips}{$aentry->{ipNetToMediaNetAddress}} = 1;
+			if (!defined($arp{$mac}{sysName})) {
+				$arp{$mac}{sysName}= $aentry->{ipNetToMediaNetAddress};
+				$arp{$mac}{namefromip} = 1;
+			}
+			$arp{$mac}{nsystems}{$aentry->{origin_sysname}} = 1;
+		}
+	}
+	my %map = ();
+	while (my ($mac, $entry) = each %arp) {
+		while (my ($sys, $ientry) = each %{$entry->{nsystems}}) {
+			$map{$entry->{sysName}}{$sys} = 1;
+			$map{$sys}{$entry->{sysName}} = 1;
+		}
+	}
+
+	my %major = ();
+	$major{assets} = \%assets;
+	$major{arp} = \%arp;
+	$major{map} = \%map;
+	print JSON::XS::encode_json(\%major);
 # Creates corrupt output, hooray.
 #	print JSON::XS->new->pretty(1)->encode(\%assets);
 	exit;
@@ -297,9 +331,37 @@ sub add_switch {
 	$assets{$chassis_id}{'ip'} = $addr;
 	$assets{$chassis_id}{'snmp'} = $snmp;
 	$assets{$chassis_id}{'snmp_parsed'} = parse_snmp($snmp);
+	populate_arp($chassis_id);
 	return;
 }
 
+sub populate_arp {
+	my ($id) = @_;
+	while (my ($key, $value) = each %{$assets{$id}{snmp_parsed}}) {
+		my $mac = $value->{'ifPhysAddress'};
+		if (!defined($mac) || $mac eq "") {
+			next;
+		}
+		mylog("mac: $mac - $assets{$id}{sysName}");
+		$arp{$mac}{chassis_id} = $id;
+		$arp{$mac}{port} = $key;
+		$arp{$mac}{sysName} = $assets{$id}{sysName};
+		if (!defined($value->{ARP})) {
+			next;
+		}
+		for my $entry (@{$value->{ARP}}) {
+			my $nmac = $entry->{ipNetToMediaPhysAddress};
+			if (!defined($arp{$nmac}{neighbors})) {
+				@{$arp{$nmac}{neighbors}} = ();
+			}
+			$entry->{'origin'} = $id;
+			$entry->{'origin_sysname'} = $assets{$id}{sysName};
+			$entry->{'origin_mgmt'} = $assets{$id}{ip};
+			push @{$arp{$nmac}{neighbors}}, $entry;
+		}
+	}
+}
+
 sub get_lldp_chassis_id {
 	my ($session) = @_;
 	my $response;
diff --git a/extras/tools/lldp/lolwhat.pl b/extras/tools/lldp/lolwhat.pl
new file mode 100755
index 0000000..a763be2
--- /dev/null
+++ b/extras/tools/lldp/lolwhat.pl
@@ -0,0 +1,369 @@
+#! /usr/bin/perl
+# 
+# What? WHAAAAT? What's going on?
+#
+# Simple to to figure out what those tools in the network department have
+# actually done. Uses SNMP to gather information about your network, using
+# multiple techniques to father information about "next hop".
+#
+# Usage: ./lolwhat.pl <ip> <community>
+#
+# This will connect to <ip> and poll it for SNMP-data, then add that to an
+# asset database. 
+use POSIX;
+use Time::HiRes;
+use strict;
+use warnings;
+use Data::Dumper;
+
+use lib '/opt/gondul/include';
+use FixedSNMP;
+use nms;
+use nms::snmp;
+
+# Actual assets detected, indexed by chassis ID
+my %assets;
+my %arp;
+
+# Tracking arrays. Continue scanning until they are of the same length.
+my @ips_checked;
+my @ips_to_check;
+
+
+# If we are given one switch on the command line, add that and then exit.
+my $cmdline_community = shift;
+my @ips = @ARGV;
+
+sub mylog {
+	my $msg = shift;
+	my $time = POSIX::ctime(time);
+	$time =~ s/\n.*$//;
+	printf STDERR "[%s] %s\n", $time, $msg;
+}
+
+
+
+my %ipmap = ();
+my %peermap = ();
+foreach my $target (@ips) {
+	my $snmp = get_snmp_data($target, $cmdline_community);
+	my $parsed = parse_snmp($snmp);
+	#print Dumper(\$parsed);
+}
+#print Dumper(\%ipmap);
+#print Dumper(\%peermap);
+
+my %hood = ();
+my %extended = ();
+
+while (my ($ip, $value)  = each %peermap) {
+	my $sys = $ipmap{$ip} || $ip;
+	my $real = defined($ipmap{$ip});
+	foreach my $sys2 (@{$value}) {
+		if ($real) {
+		$hood{$sys}{$sys2} = 1;
+		$hood{$sys2}{$sys} = 1;
+		} else {
+		$extended{$sys2}{$sys} = 1;
+		}
+	}
+}
+#print Dumper(\%hood);
+#print Dumper(\%extended);
+
+my %result = ( hood => \%hood, extended => \%extended, ipmap => \%ipmap, peermap => \%peermap);
+
+print JSON::XS::encode_json(\%result);
+exit;
+
+# Filter out stuff we don't scan. Return true if we care about it.
+# XXX: Several of these things are temporary to test (e.g.: AP).
+sub filter {
+	my %sys = %{$_[0]};
+	if (!defined($sys{'lldpRemSysCapEnabled'})) {
+		return 0;
+	}
+	my %caps = %{$sys{'lldpRemSysCapEnabled'}};
+	my $sysdesc = $sys{'lldpRemSysDesc'};
+	my $sysname = $sys{'lldpRemSysName'};
+
+	if ($caps{'cap_enabled_ap'}) {
+		return 1;
+	}
+	if ($caps{'cap_enabled_telephone'}) {
+		return 0;
+	}
+	if (!defined($sysdesc)) {
+		return 1;
+	}
+	if ($sysdesc =~ /\b(C1530|C3600|C3700)\b/) {
+		return 0;
+	}
+	if (!$caps{'cap_enabled_bridge'} && !$caps{'cap_enabled_router'}) {
+		return 1;
+	}
+	if ($sysname =~ /BCS-OSL/) {
+		return 1;
+	}
+	return 1;
+}
+
+# Discover neighbours of a switch. The data needed is already present int
+# %assets , so this shouldn't cause any extra SNMP requests. It will add
+# new devices as it finds them.
+sub discover_lldp_neighbors {
+	my $local_id = $_[0];
+	#print "local id: $local_id\n";
+	my $ip = $assets{$local_id}{mgmt};
+	my $local_sysname = $assets{$local_id}{snmp}{sysName};
+	my $community = $assets{$local_id}{community};
+	my $addrtable;
+	while (my ($key, $value) = each %{$assets{$local_id}{snmp_parsed}}) {
+		my $chassis_id = $value->{'lldpRemChassisId'};
+
+		my $sysname = $value->{'lldpRemSysName'};
+		if (!defined($sysname)) {
+			$sysname = $chassis_id;
+		}
+
+		if (defined($value->{lldpRemManAddr})) {
+			$sysname =~ s/\..*$//;
+		} else {
+			next;
+		}
+		# Do not try to poll servers.
+		if (!filter(\%{$value})) {
+			mylog("\tFiltered out $sysname  ($local_sysname -> $sysname)");
+			next;
+		}
+		mylog("\tFound $sysname ($chassis_id) ($local_sysname -> $sysname )");
+		if (defined($assets{$chassis_id}{'sysName'})) {
+			mylog("\t\tDuplicate $sysname: \"$sysname\" vs \"$assets{$chassis_id}{'sysName'}\" - $value->{'lldpRemManAddr'} vs $assets{$chassis_id}{'ip'}");
+			if ($assets{$chassis_id}{'sysName'} eq "") {
+				$assets{$chassis_id}{'sysName'} = $sysname;
+			}
+		} else {
+			$assets{$chassis_id}{'sysName'} = $sysname;
+		}
+
+		# FIXME: We should handle duplicates better and for more
+		# than just sysname. These happen every time we are at
+		# least one tier down (given A->{B,C,D,E}, switch B, C, D
+		# and E will all know about A, thus trigger this). We also
+		# want to _add_ information only, since two nodes might
+		# know about the same switch, but one might have incomplete
+		# information (as is the case when things start up).
+
+		# We simply guess that the community is the same as ours.
+		$assets{$chassis_id}{'community'} = $community;
+		$assets{$chassis_id}{'ip'} = $value->{lldpRemManAddr};
+
+		$assets{$chassis_id}{'neighbors'}{$local_id} = 1;
+		$assets{$local_id}{'neighbors'}{$chassis_id} = 1;
+		check_neigh($chassis_id);
+		#print "checking $chassis_id\n";
+	}
+}
+
+
+# Get raw SNMP data for an ip/community.
+# FIXME: This should be seriously improved. Three get()'s and four
+# gettables could definitely be streamlined, but then again, I doubt it
+# matters much unless we start running this tool constantly.
+sub get_snmp_data {
+	my ($ip, $community) = @_;
+	my %ret = ();
+	eval {
+		my $session = nms::snmp::snmp_open_session($ip, $community);
+		$ret{'sysName'} = $session->get('sysName.0');
+		$ret{'sysDescr'} = $session->get('sysDescr.0');
+		$ret{'lldpRemManAddrTable'} = $session->gettable("lldpRemManAddrTable");
+		$ret{'lldpRemTable'} = $session->gettable("lldpRemTable");
+		$ret{'ipNetToMediaTable'} = $session->gettable('ipNetToMediaTable');
+		$ret{'ifTable'} = $session->gettable('ifTable');
+		$ret{'ifXTable'} = $session->gettable('ifXTable');
+		$ret{'ipAddressTable'} = $session->gettable('ipAddressTable');
+		#print Dumper(\%ret);
+	};
+	if ($@) {
+		my $tmp = "$@";
+		chomp($tmp);
+		mylog("\t" . $tmp);
+		return undef;
+	}
+	return \%ret;
+}
+
+# Filter raw SNMP data over to something more legible.
+# This is the place to add all post-processed results so all parts of the
+# tool can use them.
+sub parse_snmp
+{
+	my $snmp = $_[0];
+	my %result = ();
+	my %lol = ();
+	$result{sysName} = $snmp->{sysName};
+	$result{sysDescr} = $snmp->{sysDescr};
+	@{$result{ips}} = ();
+	@{$result{peers}} = ();
+	while (my ($key, $value) = each %{$snmp->{lldpRemTable}}) {
+		my $chassis_id = nms::convert_mac($value->{'lldpRemChassisId'});
+		foreach my $key2 (keys %$value) {
+			$lol{$value->{lldpRemIndex}}{$key2} = $value->{$key2};
+		}
+		$lol{$value->{lldpRemIndex}}{'lldpRemChassisId'} = $chassis_id;
+		my %caps = ();
+		nms::convert_lldp_caps($value->{'lldpRemSysCapEnabled'}, \%caps);
+		$lol{$value->{lldpRemIndex}}{'lldpRemSysCapEnabled'} = \%caps;
+	}
+	while (my ($key, $value) = each %{$snmp->{lldpRemManAddrTable}}) {
+		my $old = 0;
+		if (defined($lol{$value->{lldpRemIndex}}{lldpRemManAddr})) {
+			mylog("\t\tFound existing address: $lol{$value->{lldpRemIndex}}{lldpRemManAddr}");
+			$old = $lol{$value->{lldpRemIndex}}{lldpRemManAddrSubtype};
+		}
+		my $addr = $value->{'lldpRemManAddr'};
+		my $addrtype = $value->{'lldpRemManAddrSubtype'};
+		foreach my $key2 (keys %$value) {
+			if ($key2 eq 'lldpRemManAddr') {
+				next;
+			}
+			$lol{$value->{lldpRemIndex}}{$key2} = $value->{$key2};
+		}
+		my $remip;
+		if ($addrtype == 1) {
+			$remip = nms::convert_ipv4($addr);
+			$lol{$value->{lldpRemIndex}}{lldpRemManAddr} = nms::convert_ipv4($addr);
+		} elsif ($addrtype == 2 && $old == 0) {
+			$remip = nms::convert_ipv6($addr);
+			$lol{$value->{lldpRemIndex}}{lldpRemManAddr} = nms::convert_ipv6($addr);
+		} else {
+			next;
+		}
+		my $idx = $value->{lldpRemIndex};
+		my $remname = $lol{$idx}{lldpRemSysName};
+		if (!defined($ipmap{$remip})) {
+			$ipmap{$remip} = $remname;
+		}
+		push @{$result{peers}}, $remip;
+		push @{$lol{$idx}{peers}}, $remip;
+	}
+	while (my ($key, $value) = each %{$snmp->{ifTable}}) {
+		$value->{ifPhysAddress} = nms::convert_mac($value->{ifPhysAddress});
+		foreach my $key2 (keys %$value) {
+			$lol{$value->{ifIndex}}{$key2} = $value->{$key2};
+		}
+	}
+	while (my ($key, $value) = each %{$snmp->{ipNetToMediaTable}}) {
+		my $mac = nms::convert_mac($value->{ipNetToMediaPhysAddress});
+		my $idx = $value->{ipNetToMediaIfIndex};
+		$value->{ipNetToMediaPhysAddress} = $mac;
+		push @{$lol{$idx}{ARP}}, $value;
+		if ($lol{$idx}{ifPhysAddress} eq $mac) {
+			push @{$lol{$idx}{ips}}, $value->{ipNetToMediaNetAddress};
+			push @{$result{ips}}, $value->{ipNetToMediaNetAddress};
+		} else {
+			push @{$lol{$idx}{peers}}, $value->{ipNetToMediaNetAddress};
+			push @{$result{peers}}, $value->{ipNetToMediaNetAddress};
+		}
+	}
+	while (my ($key, $value) = each %{$snmp->{ifXTable}}) {
+		foreach my $key2 (keys %$value) {
+			$lol{$key}{$key2} = $value->{$key2};
+		}
+	}
+	while (my ($key, $value) = each %{$snmp->{ipAddressTable}}) {
+		my $addr = $value->{ipAddressAddr};
+		my $addrtype = $value->{ipAddressAddrType};
+		if ($addrtype == 1) {
+			$addr = nms::convert_ipv4($addr);
+		} elsif ($addrtype == 2) {
+			$addr = nms::convert_ipv6($addr);
+		} else {
+			next;
+		}
+		push @{$result{ips}}, $addr;
+	}
+	@{$result{peers}} = sort @{$result{peers}};
+	@{$result{ips}} = sort @{$result{ips}};
+	$result{interfaces} = \%lol;
+	foreach my $localip (@{$result{ips}}) {
+		$ipmap{$localip} = $result{sysName};
+	}
+	foreach my $peer (@{$result{peers}}) {
+		if (!defined($peermap{$peer})) {
+			@{$peermap{$peer}} = ();
+		}
+		push @{$peermap{$peer}}, $result{sysName};
+	}
+	return \%result;
+}
+
+# Add a chassis_id to the list to be checked, but only if it isn't there.
+# I'm sure there's some better way to do this, but meh, perl. Doesn't even
+# have half-decent prototypes.
+sub check_neigh {
+	my $n = $_[0];
+	for my $v (@ips_to_check) {
+		if ($v eq $n) {
+			return 0;
+		}
+	}
+	push @ips_to_check,$n;
+	return 1;
+}
+
+# We've got a switch. Populate it with SNMP data (if we can).
+sub add_switch {
+	my $chassis_id = shift;
+	my $addr;
+	my $snmp = undef;
+	$addr = $assets{$chassis_id}{'ip'};
+	my $name = $assets{$chassis_id}{'sysName'} || "$addr";
+	mylog("\tProbing $addr ($name)");
+	$snmp = get_snmp_data($addr, $assets{$chassis_id}{'community'});
+	
+	return if (!defined($snmp));
+	my $sysname = $snmp->{sysName};
+	$sysname =~ s/\..*$//;
+	$assets{$chassis_id}{'sysName'} = $sysname;
+	$assets{$chassis_id}{'ip'} = $addr;
+	$assets{$chassis_id}{'snmp'} = $snmp;
+	populate_arp($chassis_id);
+	return;
+}
+
+sub populate_arp {
+	my ($id) = @_;
+	while (my ($key, $value) = each %{$assets{$id}{snmp_parsed}}) {
+		my $mac = $value->{'ifPhysAddress'};
+		if (!defined($mac) || $mac eq "") {
+			next;
+		}
+		mylog("mac: $mac - $assets{$id}{sysName}");
+		$arp{$mac}{chassis_id} = $id;
+		$arp{$mac}{port} = $key;
+		$arp{$mac}{sysName} = $assets{$id}{sysName};
+		if (!defined($value->{ARP})) {
+			next;
+		}
+		for my $entry (@{$value->{ARP}}) {
+			my $nmac = $entry->{ipNetToMediaPhysAddress};
+			if (!defined($arp{$nmac}{neighbors})) {
+				@{$arp{$nmac}{neighbors}} = ();
+			}
+			$entry->{'origin'} = $id;
+			$entry->{'origin_sysname'} = $assets{$id}{sysName};
+			$entry->{'origin_mgmt'} = $assets{$id}{ip};
+			push @{$arp{$nmac}{neighbors}}, $entry;
+		}
+	}
+}
+
+sub get_lldp_chassis_id {
+	my ($session) = @_;
+	my $response;
+	$response = $session->get('lldpLocChassisId.0');
+	my $real = nms::convert_mac($response);
+	return $real;
+}