From ee2ae15c9f4c8ea6426d90cf215720836627f1cd Mon Sep 17 00:00:00 2001 From: Kristian Lyngstol Date: Tue, 12 Apr 2016 19:41:16 +0200 Subject: Move clients->collectors Better name --- clients/dhcptail.pl | 55 ------------------ clients/ping.pl | 100 -------------------------------- clients/snmpfetchng.pl | 141 ---------------------------------------------- collectors/dhcptail.pl | 55 ++++++++++++++++++ collectors/ping.pl | 100 ++++++++++++++++++++++++++++++++ collectors/snmpfetchng.pl | 141 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 296 insertions(+), 296 deletions(-) delete mode 100755 clients/dhcptail.pl delete mode 100755 clients/ping.pl delete mode 100755 clients/snmpfetchng.pl create mode 100755 collectors/dhcptail.pl create mode 100755 collectors/ping.pl create mode 100755 collectors/snmpfetchng.pl diff --git a/clients/dhcptail.pl b/clients/dhcptail.pl deleted file mode 100755 index e5d7d4a..0000000 --- a/clients/dhcptail.pl +++ /dev/null @@ -1,55 +0,0 @@ -#! /usr/bin/perl -use DBI; -use POSIX; -use lib '../include'; -use nms; -use strict; -use Data::Dumper; -use warnings; - -BEGIN { - require "../include/config.pm"; -} - -my (undef,undef,undef,undef,undef,$year,undef,undef,undef) = gmtime(time); - -my %months = ( - Jan => 1, - Feb => 2, - Mar => 3, - Apr => 4, - May => 5, - Jun => 6, - Jul => 7, - Aug => 8, - Sep => 9, - Oct => 10, - Nov => 11, - Dec => 12 -); - -my $realtime = 0; -my ($dbh, $q,$check); -$dbh = nms::db_connect(); -$q = $dbh->prepare("INSERT INTO dhcp (switch,time,ip,mac) VALUES((SELECT switch FROM switches WHERE ?::inet << subnet4 ORDER BY sysname LIMIT 1),?,?,?)"); -$check = $dbh->prepare("SELECT max(time)::timestamp - ?::timestamp < '0s'::interval as doit FROM dhcp;"); - -open(SYSLOG, "tail -n 9999999 -F /var/log/syslog |") or die "Unable to tail syslog: $!"; -while () { - /(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d+)\s+(\d+:\d+:\d+).*DHCPACK on (\d+\.\d+\.\d+\.\d+) to (\S+) / or next; - my $date = $year . "-" . $months{$1} . "-" . $2 . " " . $3 . " Europe/Oslo"; - my $machine = $5; - my $via = $6; - $check->execute($date); - $dbh->commit; - my $cond = $check->fetchrow_hashref(); - if (!defined($cond) or !defined($cond->{'doit'}) or $cond->{'doit'} eq '1') { - if ($realtime != 1) { - $realtime = 1; - print "real time achieved...\n"; - } - $q->execute($4,$date,$4,$machine); - $dbh->commit; - } -} -close SYSLOG; diff --git a/clients/ping.pl b/clients/ping.pl deleted file mode 100755 index d945917..0000000 --- a/clients/ping.pl +++ /dev/null @@ -1,100 +0,0 @@ -#! /usr/bin/perl -use DBI; -use POSIX; -use Time::HiRes; -use Net::Oping; -use strict; -use warnings; -use Data::Dumper; - -use lib '../include'; -use nms; - -$|++; -my $dbh = nms::db_connect(); -$dbh->{AutoCommit} = 0; -$dbh->{RaiseError} = 1; - -my $q = $dbh->prepare("SELECT switch,host(mgmt_v4_addr) as ip,host(mgmt_v6_addr) as secondary_ip FROM switches WHERE mgmt_v4_addr is not null ORDER BY random()"); -my $lq = $dbh->prepare("SELECT linknet,addr1,addr2 FROM linknets WHERE addr1 is not null and addr2 is not null"); - -while (1) { - # ping loopbacks - my $ping = Net::Oping->new; - $ping->timeout(0.3); - - $q->execute; - my %ip_to_switch = (); - my %secondary_ip_to_switch = (); - while (my $ref = $q->fetchrow_hashref) { - my $switch = $ref->{'switch'}; - - my $ip = $ref->{'ip'}; - $ping->host_add($ip); - $ip_to_switch{$ip} = $switch; - - my $secondary_ip = $ref->{'secondary_ip'}; - if (defined($secondary_ip)) { - $ping->host_add($secondary_ip); - $secondary_ip_to_switch{$secondary_ip} = $switch; - } - } - my $result = $ping->ping(); - my %dropped = %{$ping->get_dropped()}; - die $ping->get_error if (!defined($result)); - - $dbh->do('COPY ping (switch, latency_ms) FROM STDIN'); # date is implicitly now. - my $drops = 0; - while (my ($ip, $latency) = each %$result) { - my $switch = $ip_to_switch{$ip}; - next if (!defined($switch)); - - if (!defined($latency)) { - $drops += $dropped{$ip}; - } - $latency //= "\\N"; - $dbh->pg_putcopydata("$switch\t$latency\n"); - } - if ($drops > 0) { - print "$drops "; - } - $dbh->pg_putcopyend(); - - $dbh->do('COPY ping_secondary_ip (switch, latency_ms) FROM STDIN'); # date is implicitly now. - while (my ($ip, $latency) = each %$result) { - my $switch = $secondary_ip_to_switch{$ip}; - next if (!defined($switch)); - - $latency //= "\\N"; - $dbh->pg_putcopydata("$switch\t$latency\n"); - } - $dbh->pg_putcopyend(); - - $dbh->commit; - # ping linknets - $ping = Net::Oping->new; - $ping->timeout(0.3); - - $lq->execute; - my @linknets = (); - while (my $ref = $lq->fetchrow_hashref) { - push @linknets, $ref; - $ping->host_add($ref->{'addr1'}); - $ping->host_add($ref->{'addr2'}); - } - if (@linknets) { - $result = $ping->ping(); - die $ping->get_error if (!defined($result)); - - $dbh->do('COPY linknet_ping (linknet, latency1_ms, latency2_ms) FROM STDIN'); # date is implicitly now. - for my $linknet (@linknets) { - my $id = $linknet->{'linknet'}; - my $latency1 = $result->{$linknet->{'addr1'}} // '\N'; - my $latency2 = $result->{$linknet->{'addr2'}} // '\N'; - $dbh->pg_putcopydata("$id\t$latency1\t$latency2\n"); - } - $dbh->pg_putcopyend(); - } - $dbh->commit; -} - diff --git a/clients/snmpfetchng.pl b/clients/snmpfetchng.pl deleted file mode 100755 index 2f5e785..0000000 --- a/clients/snmpfetchng.pl +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/perl -use strict; -use warnings; -use DBI; -use POSIX; -#use Time::HiRes qw(time); -use SNMP; -use Data::Dumper; -use lib '../include'; -use nms; - -SNMP::initMib(); -SNMP::addMibDirs("/srv/tgmanage/mibs/StandardMibs"); -SNMP::addMibDirs("/srv/tgmanage/mibs/JuniperMibs"); -SNMP::addMibDirs("/srv/tgmanage/mibs"); -SNMP::loadModules('ALL'); - -our $dbh = nms::db_connect(); -$dbh->{AutoCommit} = 0; -$dbh->{RaiseError} = 1; - -my $qualification = <<"EOF"; -(last_updated IS NULL OR now() - last_updated > poll_frequency) -AND (locked='f' OR now() - last_updated > '15 minutes'::interval) -AND mgmt_v4_addr is not null -EOF - -# Borrowed from snmpfetch.pl -our $qswitch = $dbh->prepare(<<"EOF") -SELECT - sysname,switch,host(mgmt_v4_addr) as ip,community, - DATE_TRUNC('second', now() - last_updated - poll_frequency) AS overdue -FROM - switches -WHERE -$qualification -ORDER BY - overdue DESC -LIMIT ? -EOF - or die "Couldn't prepare qswitch"; -our $qlock = $dbh->prepare("UPDATE switches SET locked='t', last_updated=now() WHERE switch=?") - or die "Couldn't prepare qlock"; -our $qunlock = $dbh->prepare("UPDATE switches SET locked='f', last_updated=now() WHERE switch=?") - or die "Couldn't prepare qunlock"; -my @switches = (); - -my $sth = $dbh->prepare("INSERT INTO snmp (switch,data) VALUES((select switch from switches where sysname=?), ?)"); - -sub mylog -{ - my $msg = shift; - my $time = POSIX::ctime(time); - $time =~ s/\n.*$//; - printf STDERR "[%s] %s\n", $time, $msg; -} - -sub populate_switches -{ - @switches = (); - my $limit = $nms::config::snmp_max; - $qswitch->execute($limit) - or die "Couldn't get switch"; - $dbh->commit; - while (my $ref = $qswitch->fetchrow_hashref()) { - push @switches, { - 'sysname' => $ref->{'sysname'}, - 'id' => $ref->{'switch'}, - 'mgtip' => $ref->{'ip'}, - 'community' => $ref->{'community'} - }; - } -} - -sub inner_loop -{ - populate_switches(); - my $poll_todo = ""; - for my $refswitch (@switches) { - my %switch = %{$refswitch}; - $poll_todo .= "$switch{'sysname'} "; - - $switch{'start'} = time; - $qlock->execute($switch{'id'}) - or die "Couldn't lock switch"; - $dbh->commit; - my $s = SNMP::Session->new(DestHost => $switch{'mgtip'}, - Community => $switch{'community'}, - UseEnums => 1, - Version => '2'); - my $ret = $s->bulkwalk(0, 10, @nms::config::snmp_objects, sub{ callback(\%switch, @_); }); - if (!defined($ret)) { - mylog("Fudge: ". $s->{'ErrorStr'}); - } - } - mylog( "Polling " . @switches . " switches: $poll_todo"); - SNMP::MainLoop(10); -} - -sub callback{ - my @top = $_[1]; - my %switch = %{$_[0]}; - my %tree; - my %ttop; - my %nics; - my @nicids; - - for my $ret (@top) { - for my $var (@{$ret}) { - for my $inner (@{$var}) { - my ($tag,$type,$name,$iid, $val) = ( $inner->tag ,$inner->type , $inner->name, $inner->iid, $inner->val); - if ($tag eq "ifPhysAddress") { - next; - } - $tree{$iid}{$tag} = $val; - if ($tag eq "ifIndex") { - push @nicids, $iid; - } - } - } - } - - my %tree2; - for my $nic (@nicids) { - $tree2{'ports'}{$tree{$nic}{'ifName'}} = $tree{$nic}; - delete $tree{$nic}; - } - for my $iid (keys %tree) { - for my $key (keys %{$tree{$iid}}) { - $tree2{'misc'}{$key}{$iid} = $tree{$iid}{$key}; - } - } - $sth->execute($switch{'sysname'}, JSON::XS::encode_json(\%tree2)); - $qunlock->execute($switch{'id'}) - or die "Couldn't unlock switch"; - $dbh->commit; - mylog( "Polled $switch{'sysname'} in " . (time - $switch{'start'}) . "s."); -} -while (1) { - inner_loop(); -} diff --git a/collectors/dhcptail.pl b/collectors/dhcptail.pl new file mode 100755 index 0000000..e5d7d4a --- /dev/null +++ b/collectors/dhcptail.pl @@ -0,0 +1,55 @@ +#! /usr/bin/perl +use DBI; +use POSIX; +use lib '../include'; +use nms; +use strict; +use Data::Dumper; +use warnings; + +BEGIN { + require "../include/config.pm"; +} + +my (undef,undef,undef,undef,undef,$year,undef,undef,undef) = gmtime(time); + +my %months = ( + Jan => 1, + Feb => 2, + Mar => 3, + Apr => 4, + May => 5, + Jun => 6, + Jul => 7, + Aug => 8, + Sep => 9, + Oct => 10, + Nov => 11, + Dec => 12 +); + +my $realtime = 0; +my ($dbh, $q,$check); +$dbh = nms::db_connect(); +$q = $dbh->prepare("INSERT INTO dhcp (switch,time,ip,mac) VALUES((SELECT switch FROM switches WHERE ?::inet << subnet4 ORDER BY sysname LIMIT 1),?,?,?)"); +$check = $dbh->prepare("SELECT max(time)::timestamp - ?::timestamp < '0s'::interval as doit FROM dhcp;"); + +open(SYSLOG, "tail -n 9999999 -F /var/log/syslog |") or die "Unable to tail syslog: $!"; +while () { + /(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d+)\s+(\d+:\d+:\d+).*DHCPACK on (\d+\.\d+\.\d+\.\d+) to (\S+) / or next; + my $date = $year . "-" . $months{$1} . "-" . $2 . " " . $3 . " Europe/Oslo"; + my $machine = $5; + my $via = $6; + $check->execute($date); + $dbh->commit; + my $cond = $check->fetchrow_hashref(); + if (!defined($cond) or !defined($cond->{'doit'}) or $cond->{'doit'} eq '1') { + if ($realtime != 1) { + $realtime = 1; + print "real time achieved...\n"; + } + $q->execute($4,$date,$4,$machine); + $dbh->commit; + } +} +close SYSLOG; diff --git a/collectors/ping.pl b/collectors/ping.pl new file mode 100755 index 0000000..d945917 --- /dev/null +++ b/collectors/ping.pl @@ -0,0 +1,100 @@ +#! /usr/bin/perl +use DBI; +use POSIX; +use Time::HiRes; +use Net::Oping; +use strict; +use warnings; +use Data::Dumper; + +use lib '../include'; +use nms; + +$|++; +my $dbh = nms::db_connect(); +$dbh->{AutoCommit} = 0; +$dbh->{RaiseError} = 1; + +my $q = $dbh->prepare("SELECT switch,host(mgmt_v4_addr) as ip,host(mgmt_v6_addr) as secondary_ip FROM switches WHERE mgmt_v4_addr is not null ORDER BY random()"); +my $lq = $dbh->prepare("SELECT linknet,addr1,addr2 FROM linknets WHERE addr1 is not null and addr2 is not null"); + +while (1) { + # ping loopbacks + my $ping = Net::Oping->new; + $ping->timeout(0.3); + + $q->execute; + my %ip_to_switch = (); + my %secondary_ip_to_switch = (); + while (my $ref = $q->fetchrow_hashref) { + my $switch = $ref->{'switch'}; + + my $ip = $ref->{'ip'}; + $ping->host_add($ip); + $ip_to_switch{$ip} = $switch; + + my $secondary_ip = $ref->{'secondary_ip'}; + if (defined($secondary_ip)) { + $ping->host_add($secondary_ip); + $secondary_ip_to_switch{$secondary_ip} = $switch; + } + } + my $result = $ping->ping(); + my %dropped = %{$ping->get_dropped()}; + die $ping->get_error if (!defined($result)); + + $dbh->do('COPY ping (switch, latency_ms) FROM STDIN'); # date is implicitly now. + my $drops = 0; + while (my ($ip, $latency) = each %$result) { + my $switch = $ip_to_switch{$ip}; + next if (!defined($switch)); + + if (!defined($latency)) { + $drops += $dropped{$ip}; + } + $latency //= "\\N"; + $dbh->pg_putcopydata("$switch\t$latency\n"); + } + if ($drops > 0) { + print "$drops "; + } + $dbh->pg_putcopyend(); + + $dbh->do('COPY ping_secondary_ip (switch, latency_ms) FROM STDIN'); # date is implicitly now. + while (my ($ip, $latency) = each %$result) { + my $switch = $secondary_ip_to_switch{$ip}; + next if (!defined($switch)); + + $latency //= "\\N"; + $dbh->pg_putcopydata("$switch\t$latency\n"); + } + $dbh->pg_putcopyend(); + + $dbh->commit; + # ping linknets + $ping = Net::Oping->new; + $ping->timeout(0.3); + + $lq->execute; + my @linknets = (); + while (my $ref = $lq->fetchrow_hashref) { + push @linknets, $ref; + $ping->host_add($ref->{'addr1'}); + $ping->host_add($ref->{'addr2'}); + } + if (@linknets) { + $result = $ping->ping(); + die $ping->get_error if (!defined($result)); + + $dbh->do('COPY linknet_ping (linknet, latency1_ms, latency2_ms) FROM STDIN'); # date is implicitly now. + for my $linknet (@linknets) { + my $id = $linknet->{'linknet'}; + my $latency1 = $result->{$linknet->{'addr1'}} // '\N'; + my $latency2 = $result->{$linknet->{'addr2'}} // '\N'; + $dbh->pg_putcopydata("$id\t$latency1\t$latency2\n"); + } + $dbh->pg_putcopyend(); + } + $dbh->commit; +} + diff --git a/collectors/snmpfetchng.pl b/collectors/snmpfetchng.pl new file mode 100755 index 0000000..2f5e785 --- /dev/null +++ b/collectors/snmpfetchng.pl @@ -0,0 +1,141 @@ +#!/usr/bin/perl +use strict; +use warnings; +use DBI; +use POSIX; +#use Time::HiRes qw(time); +use SNMP; +use Data::Dumper; +use lib '../include'; +use nms; + +SNMP::initMib(); +SNMP::addMibDirs("/srv/tgmanage/mibs/StandardMibs"); +SNMP::addMibDirs("/srv/tgmanage/mibs/JuniperMibs"); +SNMP::addMibDirs("/srv/tgmanage/mibs"); +SNMP::loadModules('ALL'); + +our $dbh = nms::db_connect(); +$dbh->{AutoCommit} = 0; +$dbh->{RaiseError} = 1; + +my $qualification = <<"EOF"; +(last_updated IS NULL OR now() - last_updated > poll_frequency) +AND (locked='f' OR now() - last_updated > '15 minutes'::interval) +AND mgmt_v4_addr is not null +EOF + +# Borrowed from snmpfetch.pl +our $qswitch = $dbh->prepare(<<"EOF") +SELECT + sysname,switch,host(mgmt_v4_addr) as ip,community, + DATE_TRUNC('second', now() - last_updated - poll_frequency) AS overdue +FROM + switches +WHERE +$qualification +ORDER BY + overdue DESC +LIMIT ? +EOF + or die "Couldn't prepare qswitch"; +our $qlock = $dbh->prepare("UPDATE switches SET locked='t', last_updated=now() WHERE switch=?") + or die "Couldn't prepare qlock"; +our $qunlock = $dbh->prepare("UPDATE switches SET locked='f', last_updated=now() WHERE switch=?") + or die "Couldn't prepare qunlock"; +my @switches = (); + +my $sth = $dbh->prepare("INSERT INTO snmp (switch,data) VALUES((select switch from switches where sysname=?), ?)"); + +sub mylog +{ + my $msg = shift; + my $time = POSIX::ctime(time); + $time =~ s/\n.*$//; + printf STDERR "[%s] %s\n", $time, $msg; +} + +sub populate_switches +{ + @switches = (); + my $limit = $nms::config::snmp_max; + $qswitch->execute($limit) + or die "Couldn't get switch"; + $dbh->commit; + while (my $ref = $qswitch->fetchrow_hashref()) { + push @switches, { + 'sysname' => $ref->{'sysname'}, + 'id' => $ref->{'switch'}, + 'mgtip' => $ref->{'ip'}, + 'community' => $ref->{'community'} + }; + } +} + +sub inner_loop +{ + populate_switches(); + my $poll_todo = ""; + for my $refswitch (@switches) { + my %switch = %{$refswitch}; + $poll_todo .= "$switch{'sysname'} "; + + $switch{'start'} = time; + $qlock->execute($switch{'id'}) + or die "Couldn't lock switch"; + $dbh->commit; + my $s = SNMP::Session->new(DestHost => $switch{'mgtip'}, + Community => $switch{'community'}, + UseEnums => 1, + Version => '2'); + my $ret = $s->bulkwalk(0, 10, @nms::config::snmp_objects, sub{ callback(\%switch, @_); }); + if (!defined($ret)) { + mylog("Fudge: ". $s->{'ErrorStr'}); + } + } + mylog( "Polling " . @switches . " switches: $poll_todo"); + SNMP::MainLoop(10); +} + +sub callback{ + my @top = $_[1]; + my %switch = %{$_[0]}; + my %tree; + my %ttop; + my %nics; + my @nicids; + + for my $ret (@top) { + for my $var (@{$ret}) { + for my $inner (@{$var}) { + my ($tag,$type,$name,$iid, $val) = ( $inner->tag ,$inner->type , $inner->name, $inner->iid, $inner->val); + if ($tag eq "ifPhysAddress") { + next; + } + $tree{$iid}{$tag} = $val; + if ($tag eq "ifIndex") { + push @nicids, $iid; + } + } + } + } + + my %tree2; + for my $nic (@nicids) { + $tree2{'ports'}{$tree{$nic}{'ifName'}} = $tree{$nic}; + delete $tree{$nic}; + } + for my $iid (keys %tree) { + for my $key (keys %{$tree{$iid}}) { + $tree2{'misc'}{$key}{$iid} = $tree{$iid}{$key}; + } + } + $sth->execute($switch{'sysname'}, JSON::XS::encode_json(\%tree2)); + $qunlock->execute($switch{'id'}) + or die "Couldn't unlock switch"; + $dbh->commit; + mylog( "Polled $switch{'sysname'} in " . (time - $switch{'start'}) . "s."); +} +while (1) { + inner_loop(); +} -- cgit v1.2.3