TG17 stuff ?

16 files changed, 126 insertions, 40 deletions

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..8fce603
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1 @@
+data/
diff --git a/build/storage-schemas.conf b/build/storage-schemas.conf
index 551f753..192328a 100644
--- a/build/storage-schemas.conf
+++ b/build/storage-schemas.conf
@@ -19,6 +19,14 @@ retentions = 10s:1d, 1m:20d
 pattern = ^ping\.
 retentions = 1s:1d, 1m:20d
 
+[tele]
+pattern = ^snmp\.fw1\.tele\.
+retentions = 10s:3d, 1m:40d
+
+[tele2]
+pattern = ^snmp\.vc1\.tele\.
+retentions = 10s:3d, 1m:40d
+
 [default_1min_for_14day]
 pattern = .*
 retentions = 1m:14d
diff --git a/build/test/gondul-grafana-test.Dockerfile b/build/test/gondul-grafana-test.Dockerfile
index be808d3..1b26f3a 100644
--- a/build/test/gondul-grafana-test.Dockerfile
+++ b/build/test/gondul-grafana-test.Dockerfile
@@ -1,9 +1,9 @@
 FROM grafana/grafana:4.0.0-beta2
-ENV GF_SERVER_ROOT_URL http://nms-dev.gathering.org/grafana/
+ENV GF_SERVER_ROOT_URL http://gondul.gathering.org/grafana/
 ENV GF_METRICS_GRAPHITE_ADDRESS graphite:2003
 ENV GF_METRICS_GRAPHITE_PREFIX grafana.%(instance_name)s.
 ENV GF_DATABASE_TYPE postgres
-ENV GF_DATABASE_HOST db:5432
+ENV GF_DATABASE_HOST cryptolocker.tg17.gathering.org:5432
 ENV GF_DATABASE_NAME grafana
 ENV GF_DATABASE_USER grafana
 ENV GF_DATABASE_PASSWORD grafana
@@ -11,6 +11,3 @@ ENV GF_DATABASE_SSL_MODE require
 ENV GF_AUTH_PROXY_ENABLED true
 ENV GF_AUTH_DISABLE_LOGIN_FORM true
 ENV GF_EXTERNAL_IMAGE_STORAGE_PROVIDER internal
-ENV GF_EXTERNAL_IMAGE_STORAGE_S3_BUCKET_URL http://grafana.situla.bitbit.net/
-ENV GF_EXTERNAL_IMAGE_STORAGE_S3_ACCESS_KEY 8KMMX9F3VZZ6MAZOGFF6-yes-I-knw
-ENV GF_EXTERNAL_IMAGE_STORAGE_S3_SECRET_KEY dTuAqxPGE5SFbtEmJxoZ9Y3AHINOZ5ju0IPfoqfA-no-it-doesnt-work
diff --git a/collectors/ping.pl b/collectors/ping.pl
index 6cb3bb5..2d181b3 100755
--- a/collectors/ping.pl
+++ b/collectors/ping.pl
@@ -20,7 +20,7 @@ my $q = $dbh->prepare("SELECT switch,sysname,host(mgmt_v4_addr) as ip,host(mgmt_
 my $lq = $dbh->prepare("SELECT linknet,addr1,addr2 FROM linknets WHERE addr1 is not null and addr2 is not null;");
 
 my $last = time();
-my $target = 0.7;
+my $target = 1.5;
 # Hack to avoid starting the collector before graphite is up.
 sleep(5);
 my $sock = IO::Socket::IP->new(
@@ -38,7 +38,7 @@ while (1) {
 	$last = time();
 	# ping loopbacks
 	my $ping = Net::Oping->new;
-	$ping->timeout(0.4);
+	$ping->timeout(0.9);
 
 	$q->execute;
 	my %ip_to_switch = ();
@@ -78,10 +78,10 @@ while (1) {
 	my $now_graphite = time();
 	while (my ($ip, $latency) = each %$result) {
 		my $switch = $ip_to_switch{$ip};
-		my $sysname = $sw_to_sysname{$switch};
 		if (!defined($switch)) {
 			next;
 		}
+		my $sysname = $sw_to_sysname{$switch};
 
 		if (!defined($latency)) {
 			$drops += $dropped{$ip};
diff --git a/collectors/snmpfetchng.pl b/collectors/snmpfetchng.pl
index b961cac..df90138 100755
--- a/collectors/snmpfetchng.pl
+++ b/collectors/snmpfetchng.pl
@@ -61,7 +61,7 @@ sleep(5);
 my $sock = IO::Socket::IP->new(
        PeerHost => "$nms::config::graphite_host:$nms::config::graphite_port",
         Timeout => 20,
-       ) or die "Cannot connect - $@";
+       ) or die "Cannot connect to graphite - $@";
  
  $sock->blocking( 0 );
 
diff --git a/extras/misc/varnish.vcl b/extras/misc/varnish.vcl
index 8f67a4d..271211b 100644
--- a/extras/misc/varnish.vcl
+++ b/extras/misc/varnish.vcl
@@ -12,6 +12,10 @@ backend default {
 backend graphite {
     .host = "gondul-graphite";
     .port = "80";
+    .first_byte_timeout	= 3s;
+    .between_bytes_timeout = 3s;
+    .connect_timeout = 1s;
+    .max_connections = 20;
 }
 
 backend templating {
@@ -39,7 +43,7 @@ sub vcl_recv {
         return (synth(418,"LOLOLOL"));
     }
 
-    if (req.url ~ "^/render") {
+    if (req.url ~ "^/render" || req.url ~ "^/metric") {
         set req.backend_hint = graphite;
     }
     if (req.url ~ "^/grafana") {
diff --git a/extras/tools/add_switches.txt.pl b/extras/tools/add_switches.txt.pl
index b4f4ef2..0ec38b1 100755
--- a/extras/tools/add_switches.txt.pl
+++ b/extras/tools/add_switches.txt.pl
@@ -8,7 +8,7 @@
 use strict;
 use warnings;
 use Data::Dumper;
-use lib '/opt/gondul/include';
+use lib '/home/tech/gondul/include';
 use JSON;
 use nms::util;
 
diff --git a/include/nms/util.pm b/include/nms/util.pm
index 3e8596e..1cc34e9 100644
--- a/include/nms/util.pm
+++ b/include/nms/util.pm
@@ -17,7 +17,7 @@ sub parse_switch {
 		'mgmt_v4_addr' => "$mgtmt4",
 		'mgmt_v6_addr' => "$mgtmt6",
 		'traffic_vlan' => "$lolid",
-		'distro' => "$distro"
+		'distro_name' => "$distro"
 	);
 	%{$ret{'placement'}} = guess_placement($switch);
 	return %ret;
diff --git a/playbook-prod.yml b/playbook-prod.yml
new file mode 100644
index 0000000..c452438
--- /dev/null
+++ b/playbook-prod.yml
@@ -0,0 +1,31 @@
+---
+- hosts: all
+  become: false
+  roles:
+    - basics
+  vars:
+  - images:
+    - name: "gondul-graphite-test"
+      volumes: [ "{{ pwd.stdout }}/:/opt/gondul" , "{{ pwd.stdout }}/data/graphite:/var/lib/graphite" ]
+      links: []
+      ports: [ "2003:2003" ]
+    - name: "gondul-grafana-test"
+      volumes: [ "{{ pwd.stdout }}/:/opt/gondul" , "{{ pwd.stdout }}/data/grafana:/var/lib/grafana" ]
+      links: ["gondul-graphite-test:graphite" ]
+      ports: []
+    - name: "gondul-front-test"
+      volumes: [ "{{ pwd.stdout }}/:/opt/gondul" ]
+      links: [  ]
+      ports: "{{ front_ports }}"
+    - name: "gondul-templating-test"
+      volumes: [ "{{ pwd.stdout }}/:/opt/gondul" ]
+      links: [ "gondul-front-test:gondul-front"]
+      ports: []
+    - name: "gondul-varnish-test"
+      volumes: [ "{{ pwd.stdout }}/:/opt/gondul" ]
+      links: [ "gondul-front-test:gondul-front", "gondul-graphite-test:gondul-graphite", "gondul-templating-test:gondul-templating", "gondul-grafana-test:gondul-grafana" ]
+      ports: "{{ varnish_ports }}"
+    - name: "gondul-snmp-test"
+      volumes: [ "{{ pwd.stdout }}/:/opt/gondul" ]
+      links: [ "gondul-graphite-test:graphite" ]
+      ports: [ ]
diff --git a/web/api/read/distro-tree b/web/api/public/distro-tree
index 5d93e1b..5d93e1b 100755
--- a/web/api/read/distro-tree
+++ b/web/api/public/distro-tree
diff --git a/web/api/public/switch-state b/web/api/public/switch-state
index 8e98205..36dfeaf 100755
--- a/web/api/public/switch-state
+++ b/web/api/public/switch-state
@@ -40,7 +40,7 @@ while ( my $ref = $q->fetchrow_hashref() ) {
 			$json{'switches'}{$sysname}{ifs}{$smallport}{'ifAlias'} = $port{'ifAlias'};
 		}
 		if ($data{'ports'}{$porti}{'ifType'} ne "propVirtual" and
-		    $data{'ports'}{$porti}{'ifAlias'} =~ m/trunk/i) {
+		    $data{'ports'}{$porti}{'ifAlias'} =~ m/LAG member/i) {
 			$json{'switches'}{$sysname}{'uplinks'}{'ifHCInOctets'} += $port{'ifHCInOctets'};
 			$json{'switches'}{$sysname}{'uplinks'}{'ifHCOutOctets'} += $port{'ifHCOutOctets'};
 			if ($port{'ifOperStatus'} eq "up") {
@@ -48,6 +48,15 @@ while ( my $ref = $q->fetchrow_hashref() ) {
 			}
 			$json{'switches'}{$sysname}{'uplinks'}{'total'} += 1;
 		}
+		if ($data{'ports'}{$porti}{'ifType'} ne "propVirtual" and
+		    $data{'ports'}{$porti}{'ifAlias'} =~ m/Clients/i) {
+			$json{'switches'}{$sysname}{'clients'}{'ifHCInOctets'} += $port{'ifHCInOctets'};
+			$json{'switches'}{$sysname}{'clients'}{'ifHCOutOctets'} += $port{'ifHCOutOctets'};
+			if ($port{'ifOperStatus'} eq "up") {
+				$json{'switches'}{$sysname}{'clients'}{'live'} += 1;
+			}
+			$json{'switches'}{$sysname}{'clients'}{'total'} += 1;
+		}
 		$json{'switches'}{$sysname}{ifs}{$smallport}{'ifHCInOctets'} += $port{'ifHCInOctets'} || 0;
 		$json{'switches'}{$sysname}{ifs}{$smallport}{'ifHCOutOctets'} += $port{'ifHCOutOctets'} || 0;
 		$json{'switches'}{$sysname}{totals}{'ifHCInOctets'} += $port{'ifHCInOctets'} || 0;
@@ -91,7 +100,7 @@ while ( my $ref = $q2->fetchrow_hashref() ) {
 			$json{'then'}{$sysname}{ifs}{$smallport}{'ifAlias'} = $port{'ifAlias'};
 		}
 		if ($data{'ports'}{$porti}{'ifType'} ne "propVirtual" and
-		    $data{'ports'}{$porti}{'ifAlias'} =~ m/trunk/i) {
+		    $data{'ports'}{$porti}{'ifAlias'} =~ m/LAG member/i) {
 			$json{'then'}{$sysname}{'uplinks'}{'ifHCInOctets'} += $port{'ifHCInOctets'};
 			$json{'then'}{$sysname}{'uplinks'}{'ifHCOutOctets'} += $port{'ifHCOutOctets'};
 			if ($port{'ifOperStatus'} eq "up") {
@@ -99,6 +108,15 @@ while ( my $ref = $q2->fetchrow_hashref() ) {
 			}
 			$json{'then'}{$sysname}{'uplinks'}{'total'} += 1;
 		}
+		if ($data{'ports'}{$porti}{'ifType'} ne "propVirtual" and
+		    $data{'ports'}{$porti}{'ifAlias'} =~ m/Clients/i) {
+			$json{'then'}{$sysname}{'clients'}{'ifHCInOctets'} += $port{'ifHCInOctets'};
+			$json{'then'}{$sysname}{'clients'}{'ifHCOutOctets'} += $port{'ifHCOutOctets'};
+			if ($port{'ifOperStatus'} eq "up") {
+				$json{'then'}{$sysname}{'clients'}{'live'} += 1;
+			}
+			$json{'then'}{$sysname}{'clients'}{'total'} += 1;
+		}
 
 		$json{'then'}{$sysname}{ifs}{$smallport}{'ifHCInOctets'} += $port{'ifHCInOctets'} || 0;
 		$json{'then'}{$sysname}{ifs}{$smallport}{'ifHCOutOctets'} += $port{'ifHCOutOctets'} || 0;
diff --git a/web/api/read/switches-management b/web/api/read/switches-management
index 425262e..9d8bc53 100755
--- a/web/api/read/switches-management
+++ b/web/api/read/switches-management
@@ -20,9 +20,9 @@ $nms::web::cc{'max-age'} = "5";
 $nms::web::cc{'stale-while-revalidate'} = "30";
 
 if (!defined($switch)) {
-	$q2 = $nms::web::dbh->prepare('select sysname,mgmt_v4_addr,subnet4,subnet6,mgmt_v6_addr,mgmt_v4_gw,mgmt_v6_gw,mgmt_vlan,traffic_vlan,poll_frequency,last_updated,distro_name,distro_phy_port,community from switches where placement is not null');
+	$q2 = $nms::web::dbh->prepare('select sysname,host(mgmt_v4_addr) as mgmt_v4_addr,subnet4,subnet6,host(mgmt_v6_addr) as mgmt_v6_addr,host(mgmt_v4_gw) as mgmt_v4_gw,host(mgmt_v6_gw) as mgmt_v6_gw,mgmt_vlan,traffic_vlan,poll_frequency,last_updated,distro_name,distro_phy_port,community from switches where placement is not null');
 } else {
-	$q2 = $nms::web::dbh->prepare('select sysname,mgmt_v4_addr,subnet4,subnet6,mgmt_v6_addr,mgmt_v4_gw,mgmt_v6_gw,mgmt_vlan,traffic_vlan,poll_frequency,last_updated,distro_name,distro_phy_port,community from switches where placement is not null and sysname = ' . $nms::web::dbh->quote($switch) . ';');
+	$q2 = $nms::web::dbh->prepare('select sysname,host(mgmt_v4_addr) as mgmt_v4_addr,subnet4,subnet6,host(mgmt_v6_addr) as mgmt_v6_addr,host(mgmt_v4_gw) as mgmt_v4_gw,host(mgmt_v6_gw) as mgmt_v6_gw,mgmt_vlan,traffic_vlan,poll_frequency,last_updated,distro_name,distro_phy_port,community from switches where placement is not null and sysname = ' . $nms::web::dbh->quote($switch) . ';');
 }
 
 $q2->execute();
diff --git a/web/css/nms.css b/web/css/nms.css
index 17f4ff9..9c783af 100644
--- a/web/css/nms.css
+++ b/web/css/nms.css
@@ -127,4 +127,5 @@ div.map-mode-legend button {
 .graph {
 	max-width: 100%;
 	width: 100%;
+	height: 240px;
 }
diff --git a/web/index.html b/web/index.html
index 9c66a41..7ea26e3 100644
--- a/web/index.html
+++ b/web/index.html
@@ -92,7 +92,7 @@
 	    </div>
 	    <p class="navbar-text" id="dhcp-summary">(...)</p>
 	    <div class="navbar-text" id="navbar-graph-div">
-		    <img id="navbar-graph" title="Blue: Average latency. Green: Total traffic. Time frame: last 60 minutes."/>
+		    <img id="navbar-graph" style="display:none" title="Blue: Average latency. Green: Total traffic. Time frame: last 60 minutes."/>
 	    </div>
 	    </li>
 	  </ul>
@@ -317,7 +317,7 @@
 	</div>
 	</div>
 
-	<div style="display:none;"><img id="source" src="img/tg16-salkart-clean-big.png" ></div>
+	<div style="display:none;"><img id="source" src="img/tg15-salkart-clean-big.png" ></div>
       </div>
     </div><!--/.fluid-container-->
     <script src="js/jquery.min.js" type="text/javascript"></script>
diff --git a/web/js/nms-map-handlers.js b/web/js/nms-map-handlers.js
index 42785ff..2214e2d 100644
--- a/web/js/nms-map-handlers.js
+++ b/web/js/nms-map-handlers.js
@@ -136,31 +136,53 @@ function uplinkInfo(sw)
 	ret.score = 0;
 	var u = 0;
 	var t  = 0;
+        var known_t = 0;
 	if (testTree(nmsData,['switchstate','switches',sw,'uplinks','live'])) {
 		u = parseInt(nmsData.switchstate.switches[sw].uplinks.live);
 		t = parseInt(nmsData.switchstate.switches[sw].uplinks.total);
+		known_t = t;
 		ret.data[0].value = u + " / " + t;
 		ret.data[0].description = "Uplinks (live/configured)";
 		if (nmsData.switches.switches[sw].subnet4 == undefined ||
 		    nmsData.switches.switches[sw].subnet4 == null) {
-			if (u == t) {
-				ret.score = 0
-				ret.why = "All uplinks up";
-			} else if (u == 1) {
-				ret.score = 800;
-				ret.why = "Only 1 of " + t + " uplinks alive";
-			} else {
-				ret.score = 650;
-				ret.why = u + " of " + t + " uplinks alive";
-			}
+		    if (tagged(sw,'3up')) {
+			    known_t = 3;
+		    } else if (tagged(sw,'2up')) {
+			    known_t = 2;
+		    } else if (tagged(sw, '1up')) {
+			    known_t = 1;
+		    } else if (tagged(sw,'4up')) {
+			    known_t = 4;
+		    }
+		    if (known_t != t) {
+			    ret.data[0].value += "(Overridden: " + known_t + ")";
+		    }
+
+		    if (u == known_t) {
+			    ret.score = 0
+				    ret.why = "All uplinks up";
+		    } else if (u == 1) {
+			    ret.score = 800;
+			    ret.why = "Only 1 of " + known_t + " uplinks alive";
+		    } else {
+			    ret.score = 650;
+			    ret.why = u + " of " + known_t + " uplinks alive";
+		    }
 		}
 	}
+	if (testTree(nmsData,['switchstate','switches',sw,'clients','live'])) {
+		var tu = parseInt(nmsData.switchstate.switches[sw].clients.live);
+		var tt = parseInt(nmsData.switchstate.switches[sw].clients.total);
+		ret.data[1] = {};
+		ret.data[1].value = (tu) + " / " + (tt);
+		ret.data[1].description = "Client ports (live/total)";
+	}
 	if (testTree(nmsData,['switchstate','switches',sw,'totals','live'])) {
 		var tu = parseInt(nmsData.switchstate.switches[sw].totals.live);
 		var tt = parseInt(nmsData.switchstate.switches[sw].totals.total);
-		ret.data[1] = {};
-		ret.data[1].value = (tu-u) + " / " + (tt-t);
-		ret.data[1].description = "Non-uplink ports (live/total)";
+		ret.data[2] = {};
+		ret.data[2].value = (tu-u) + " / " + (tt-t);
+		ret.data[2].description = "Non-uplink ports (live/total)";
 	}
 	return ret;
 }
@@ -422,7 +444,7 @@ function pingInfo(sw)
 			ret.why = "Latency";
 			ret.score = parseInt(v4 > v6 ? v4 : v6);
 		}
-		if (nmsData.ping.switches[sw].age4 > 5 || nmsData.ping.switches[sw].age6 > 5) {
+		if (nmsData.ping.switches[sw].age4 > 10 || nmsData.ping.switches[sw].age6 > 10) {
 			ret.why = "Old ping";
 			ret.score = 900;
 		}
@@ -483,9 +505,14 @@ function dhcpInfo(sw) {
 		var now = nmsData.dhcp.time;
 		var then = nmsData.dhcp.dhcp[sw];
 		var diff = now - then;
+		var divider = 4;
+		if(tagged(sw,'slowdhcp')) {
+			divider = 10;
+		}
+
 		ret.data[0].value = diff;
 		ret.why = "DHCP freshness";
-		ret.score = diff/4> 500 ? 500 : parseInt(diff/4);
+		ret.score = diff/divider> 500 ? 500 : parseInt(diff/divider);
 	} else {
 		ret.data[0].value = "No DHCP data";
 		if (testTree(nmsData,['smanagement','switches',sw])) {
@@ -509,9 +536,10 @@ function dhcpInfo(sw) {
 		ret.data[1].description = "DHCP clients";
 	}
 	if (testTree(nmsData,['switches','switches',sw, 'tags'])) {
-		if (nmsData.switches.switches[sw].tags.includes('ignoredhcp')) {
+		if (tagged(sw,'ignoredhcp')) {
 			ret.score = 0;
 			ret.why += "(Ignored)";
+			ret.data[0].value += "(Ignored)";
 		}
 	}
 	return ret;
diff --git a/web/js/nms.js b/web/js/nms.js
index 362aebc..8f98c75 100644
--- a/web/js/nms.js
+++ b/web/js/nms.js
@@ -395,8 +395,8 @@ function initNMS() {
 		nmsData.registerSource("snmp","/api/read/snmp");
 		nmsData.registerSource("smanagement","/api/read/switches-management");
 		nmsData.registerSource("oplog", "/api/read/oplog");
-		setInterval(nmsUpdateNavbarGraph, 10000);
-		nmsUpdateNavbarGraph();
+	//	setInterval(nmsUpdateNavbarGraph, 30000);
+	//	nmsUpdateNavbarGraph();
 		nmsOplog.init();
 	}
 
@@ -624,11 +624,9 @@ function restoreSettings()
  */
 function nmsUpdateNavbarGraph() {
 	var img = document.getElementById("navbar-graph");
-	var w = Math.floor(window.innerWidth / 4);
-	if (window.innerWidth > 2300)
-		w += 400;
+	var w = 200;
 	
-	img.src = "/render/?target=movingAverage(averageSeries(ping.*.ipv4),%225min%22)&target=secondYAxis(averageSeries(perSecond(snmp.*.ports.*.{ifHCInOctets,ifHCOutOctets})))&bgcolor=%23ffffff00&width=" + w + "&height=20&format=svg&from=-60min&until=now&graphOnly=true&somerandomthing=" + Math.floor(new Date().getTime() / 10000);
+	img.src = "/render/?target=movingAverage(averageSeries(ping.*.*.ipv4),%225min%22)&target=secondYAxis(averageSeries(perSecond(snmp.*.*.ports.*.{ifHCInOctets,ifHCOutOctets})))&bgcolor=%23ffffff00&width=" + w + "&height=20&format=svg&from=-30min&until=now&graphOnly=true";
 }
 /*
  * Test if the entire path specified in the arrary "ar" exists under the