Add three Nagios checks to detect bugs in /etc/resolv.conf, /etc/hosts

and a shutdown in progress.

7 files changed, 440 insertions, 0 deletions

diff --git a/debian/changelog b/debian/changelog
index 549d65f..a328ccc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -24,6 +24,8 @@ sitesummary (0.0.59) UNRELEASED; urgency=low
     found at <URL: http://www.monitoringexchange.org >.
   * Do not generate Nagios check for file systems with the fuse.ltspfs
     type, to avoid checking users usb disks on a thin client server.
+  * Add three Nagios checks to detect bugs in /etc/resolv.conf, /etc/hosts
+    and a shutdown in progress.
 
  -- Petter Reinholdtsen <pere@debian.org>  Tue, 26 Jan 2010 08:56:53 +0100
 
diff --git a/nagios-nrpe-commands.cfg b/nagios-nrpe-commands.cfg
index 7ce030e..11e5ccf 100644
--- a/nagios-nrpe-commands.cfg
+++ b/nagios-nrpe-commands.cfg
@@ -4,6 +4,8 @@
 command[check_apt]=/usr/lib/nagios/plugins/check_apt
 command[check_dhcp]=/usr/lib/nagios/plugins/check_dhcp
 command[check_disk]=/usr/lib/nagios/plugins/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$
+command[check_etc_hosts]=/usr/lib/sitesummary/nagios-plugins/check_etc_hosts
+command[check_etc_resolv]=/usr/lib/sitesummary/nagios-plugins/check_etc_resolv
 command[check_ftp]=/usr/lib/nagios/plugins/check_ftp -H localhost
 command[check_kernel_status]=/usr/lib/sitesummary/nagios-plugins/check_kernel_status
 command[check_linux_raid]=/usr/lib/nagios/plugins/check_linux_raid
@@ -12,5 +14,6 @@ command[check_ntp]=/usr/lib/nagios/plugins/check_ntp -H localhost
 command[check_procs]=/usr/lib/nagios/plugins/check_procs -w $ARG1$ -c $ARG2$
 command[check_procs_cron]=/usr/lib/nagios/plugins/check_procs -w $ARG1$ -c $ARG2$ -C cron
 command[check_procs_zombie]=/usr/lib/nagios/plugins/check_procs -w $ARG1$ -c $ARG2$ -s Z
+command[check_shutdown]=/usr/lib/sitesummary/nagios-plugins/check_shutdown
 command[check_smtp]=/usr/lib/nagios/plugins/check_smtp -H localhost
 command[check_swap]=/usr/lib/nagios/plugins/check_swap -w $ARG1 -c $ARG2$
diff --git a/nagios-plugins/check_etc_hosts b/nagios-plugins/check_etc_hosts
new file mode 100755
index 0000000..8704965
--- /dev/null
+++ b/nagios-plugins/check_etc_hosts
@@ -0,0 +1,247 @@
+#!/usr/bin/perl -w
+#
+# Author: Petter Reinholdtsen <pere@hungry.com>
+# Date: 2002-07-08
+#
+# Check /etc/hosts, and make sure the content matches the information
+# in DNS.  Lookup IP, and check if the names listed in /etc/hosts
+# maches the one in DNS.
+
+use vars qw($use_perl_resolver $debug $returnvalue $nagiosmsg);
+
+$debug = 0;
+$returnvalue = 0; # all ok
+$nagiosmsg = "";
+
+# Report missing reverse lookup.  This will ignore CNAME entries
+# pointing to the IP address, and report error in these cases.
+$use_reverse = 0;
+
+# Report missing forward-lookup.  This will complain on private
+# network IP addresses using the same name as a DNS entry.
+$use_forward = 1;
+
+eval 'use Net::DNS;';
+if ($@) {
+    print "Using /etc/hosts\n" if $debug;
+    $use_perl_resolver = 0;
+} else {
+    print "Using Net::DNS\n" if $debug;
+    $use_perl_resolver = 1;
+}
+
+$host = '/local/bin/host';
+
+# Look up ip address, and return ($error status, @DNS names, @DNS addresses)
+sub dns_lookup_ext {
+    local $address = shift;
+
+    # Stupid Tru64 Unix give me two copies of the error messages from
+    # host.  Throw away one of them.
+    close(STDERR);
+
+    print "Looking up $address using $host\n" if $debug;
+
+    my @names = ();
+    my @addresses = ();
+
+    my $lookup = "";
+    # Some versions need -i to make sure it uses reverse DNS lookup,
+    # and not /etc/hosts.
+    # This option will confuse 'host' on Irix and HP/UX, and make the
+    # program loop forever.  Avoiding it for now [pere 2002-08-06]
+    for $options ("") {
+        open(HOST, "$host $options $address 2>&1 |")
+            || die "Unable to execute host";
+        while (<HOST>) {
+            $lookup .= $_;
+            chomp;
+            print "host: $_\n" if $debug;
+
+            push(@names, lc($1)) if (/^Name: (.+)$/);
+            push(@names, lc($1)) if (/^Aliases: (.+)$/);
+
+            # 10.6.240.129.in-addr.arpa       PTR     perleporten.uio.no
+            push(@names, lc($1)) if (/\s+PTR\s+(.+)$/);
+
+            # spheniscus.uio.no has address 129.240.148.19
+            if (/^\S+ has address (\S+)$/) {
+                print "Match addr $1\n" if $debug;
+                push(@addresses, lc($1))
+            }
+
+            # 10.6.240.129.IN-ADDR.ARPA domain name pointer perleporten.uio.no
+            if (/IN-ADDR.ARPA domain name pointer\s+(.+)$/) {
+                print "Match name $1\n" if $debug;
+                push(@names, lc($1))
+            }
+
+            push(@addresses, $1) if (/^Address: (.+)$/);
+            push(@addresses, $1) if (/\s+A\s+(\d+.+)$/)
+            }
+        close(HOST);
+        if ($lookup =~ /Usage: /) {
+            # Probably unknown parameter, try again without -i
+            $lookup = "";
+        } else {
+            last;
+        }
+    }
+    return ("no/bad reply from DNS server", undef, undef)
+        if ($lookup !~ /domain name pointer/
+            && $lookup !~ /\shas address\s/
+            && $lookup !~ /\sPTR\s/
+            && $lookup !~ /Name:/);
+
+    if ( $address =~ m/^\d+\.\d+\.\d+\.\d+/
+         && ! grep /$address/, @addresses ) {
+        print "Adding $address to list of addresses\n" if $debug;
+        unshift(@addresses, $address) ;
+    }
+
+    return (undef, \@names, \@addresses);
+}
+
+sub dns_lookup_int {
+    my $address = shift;
+
+    print "Looking up $address using Net::DNS\n" if $debug;
+
+    my @names = ();
+    my @addresses = ();
+
+    my $res = new Net::DNS::Resolver;
+    my $query;
+    if ($address =~ m/\d+\.\d+\.\d+\.\d+/) {
+        $query = $res->query($address);
+    } else {
+        $query = $res->search($address);
+    }
+    if ($query) {
+        foreach $rr ($query->answer) {
+            print "Type: $rr->type\n" if $debug;
+            if ($rr->type eq "A") {
+                print $rr->address, " - A\n" if $debug;
+                push(@addresses, $rr->address);
+            }
+            if ($rr->type eq "CNAME") {
+                print $rr->cname, " - CNAME\n" if $debug;
+                push(@addresses, $rr->cname);
+            }
+            if ($rr->type eq "PTR") {
+                print $rr->ptrdname, " - PTR\n" if $debug;
+                push(@names, lc($rr->ptrdname));
+            }
+        }
+        return (undef, \@names, \@addresses);
+    }
+    else {
+        print "query failed: ", $res->errorstring, "\n" if $debug;
+        return ($res->errorstring, (), ());
+    }
+}
+
+sub dns_lookup {
+    my $entry = shift;
+
+    if ($use_perl_resolver) {
+        return dns_lookup_int($entry);
+    } else {
+        return dns_lookup_ext($entry);
+    }
+}
+
+sub error {
+    local ($level, $error) = @_;
+
+    $returnvalue = 1 if ($level =~ /^W$/ && $returnvalue <= 1);
+    $returnvalue = 2 if ($level =~ /^C$/);
+	
+    $nagiosmsg = $nagiosmsg . "<br>" unless ($nagiosmsg =~ /^$/);
+    $nagiosmsg = $nagiosmsg . "$error";
+}
+
+sub is_ip_private {
+    my $ip = shift;
+
+    return 1 if ($ip =~ m/^10\./);
+    return 1 if ($ip =~ m/^192\.168\./);
+
+    return 0;
+}
+
+sub is_names_ip_matching {
+    local ($ip, @names) = @_;
+
+    my $level = "W";
+
+    # Ignore IPv6 addresses for now.
+    return if ($ip =~ m/:/);
+
+    # Ignore private network
+    return if (is_ip_private($ip));
+
+    my $name;
+    for $name (sort @names) {
+        if ($use_reverse) {
+            # Check reverse
+            my ($retval, $revnames) = dns_lookup($ip);
+
+            return if ( $retval ); # Ignore unknown IP addresses
+
+            if ( ! $retval && ! grep /$name/, @{$revnames} ) {
+                error $level, "Incorrect /etc/hosts for $ip: ".
+                    "$name not in reverse DNS list";
+            }
+        }
+
+        if ($use_forward) {
+            # Check forward
+            my ($retval, $revnames, $forwip);
+
+            ($retval, $revnames) = dns_lookup($ip);
+            ($retval, undef, $forwip) = dns_lookup($name);
+
+            print "Forward DNS $name/$ip: ", join(" ", @{$forwip}), "\n"
+                if $debug;
+
+            # Ignore entry if both IP and hostname fail to resolve in DNS
+            return if ( ! defined $revnames && ! defined $forwip );
+
+            if ( ! grep /$ip/, @{$forwip} ) {
+                error $level, "Incorrect /etc/hosts for $ip: ".
+                    "IP not in forward DNS list for '$name'";
+            }
+        }
+    }
+}
+
+sub check_etc_hosts {
+    open(HOSTS, "< /etc/hosts") || die "Unable to open /etc/hosts";
+    while (<HOSTS>) {
+        chomp;
+        s/\#.+//;          # Skip comments
+        next if (/^\s*$/); # Skip empty lines
+
+        print "Testing $_\n" if $debug;
+
+        $_ = lc($_);
+
+        local ($ip, @names) = split(/\s+/);
+
+        # Skip localhost, it is different on some platforms.
+        next if ($ip eq '127.0.0.1');
+
+        is_names_ip_matching($ip, @names);
+    }
+    close(HOSTS);
+}
+
+check_etc_hosts() if ( -f "/etc/hosts" );
+
+if ($nagiosmsg =~ /^$/) {
+    print "/etc/hosts OK\n";
+} else {
+    print $nagiosmsg . "\n";
+}
+exit $returnvalue;
diff --git a/nagios-plugins/check_etc_resolv b/nagios-plugins/check_etc_resolv
new file mode 100755
index 0000000..b032111
--- /dev/null
+++ b/nagios-plugins/check_etc_resolv
@@ -0,0 +1,134 @@
+#!/usr/bin/perl -w
+#
+# Author: Petter Reinholdtsen <pere@hungry.com>
+# Date: 2001-11-09
+#
+# Check /etc/resolv.conf, and make sure the name servers listed are working.
+
+use Socket;
+
+use vars qw($host $debug $mincount $trycount $testhost $retval $nagiosmsg);
+
+$retval = 0;
+$nagiosmsg = "";
+
+$host = '/usr/bin/host';
+
+$debug = 0;
+
+# There should be at least this many servers in the list
+$mincount = 2;
+
+# Try to call host this many times before reporting any bugs
+$trycount = 3;
+
+# Which DNS name to look up
+$testhost = "www.uio.no";
+
+# Stolen from Logwatch.pm
+sub canonical_ipv6_address {
+    my @a = split /:/, shift;
+    my @b = qw(0 0 0 0 0 0 0 0);
+    my $i = 0;
+    # comparison is numeric, so we use hex function
+    while (defined $a[0] and $a[0] ne '') {$b[$i++] = hex(shift @a);}
+    @a = reverse @a;
+    $i = 7;
+    while (defined $a[0] and $a[0] ne '') {$b[$i--] = hex(shift @a);}
+    @b;
+}
+
+sub error_with_dns {
+    local ($count, $ip, $error) = @_;
+
+    # Count 1   = Error
+    # Count 2   = Problem
+    # Count 3-> = Warning
+    if (1 == $count) {
+	$retval = 2;
+    } elsif (2 == $count) {
+	$retval = 1 unless ($retval > 0);
+    }
+    
+    $nagiosmsg .= "<br>" unless ($nagiosmsg =~ /^$/);
+    $nagiosmsg .= "/etc/resolv.conf: nameserver #$count $ip: $error";
+}
+
+# Check if there is a DNS server running on the given IP address
+sub test_dns_server {
+    local ($ip) = @_;
+    local ($name) = "";
+
+    print "Checking $1\n" if $debug;
+
+    # there are other module functions that do this more gracefully
+    # (such as inet_pton), but we can't guarantee that they are available
+    # in every system, so we use the built-in gethostbyaddr.
+    if ($ip =~ /^[\d\.]*$/) {
+	$PackedAddr = pack('C4', split /\./,$ip);
+	$name = gethostbyaddr($PackedAddr,AF_INET());
+    } elsif ($ip =~ /^[0-9a-zA-Z:]*/) {
+	$PackedAddr = pack('n8', canonical_ipv6_address($ip));
+	$name = gethostbyaddr($PackedAddr, AF_INET6());
+    }
+
+    return "missing in DNS" if ( ! defined $name );
+
+    my $try = $trycount;
+    my $delay = 1; # Exponensial backoff
+    for ($try = $trycount; $try; --$try) {
+        print "Running '$host $testhost $ip 2>/dev/null'\n" if $debug;
+        local $lookup = `$host $testhost $ip 2>/dev/null`;
+
+        print "Reply from host (try=$try):\n  $lookup\n" if $debug;
+
+        if ($lookup =~ /\sA\s+\d/ || $lookup =~ /has address/ ||
+            $lookup =~ /domain name pointer/ || $lookup =~ /Name: /) {
+            return undef; # true
+        }
+        print "Sleeping $delay\n" if $debug;
+        sleep $delay;
+        $delay += $delay;
+    }
+
+    return "no/bad reply from DNS server";
+}
+
+sub check_etc_resolv_conf {
+    open(RESOLV, "< /etc/resolv.conf") || die "Unable to open /etc/resolv.conf";
+    local $count = 0;
+    while (<RESOLV>) {
+        chomp;
+        s/\#.+//;          # Skip comments
+        next if (/^\s*$/); # Skip empty lines
+        if (/^nameserver\s+(\S+)/) {
+            $count++;
+            local ($error) = test_dns_server($1);
+            if ( defined $error ) {
+                error_with_dns($count, $1, $error);
+	    }
+        }
+    }
+    close(RESOLV);
+    
+    if ($count < $mincount) {
+	$retval = 1 unless $retval > 0;
+	$nagiosmsg .= "<br>" unless ($nagiosmsg =~ /^$/);
+	$nagiosmsg .= "/etc/resolv.conf: Only $count nameservers in " .
+	    "/etc/resolv.conf (low-limit is $mincount)";
+    }
+}
+
+check_etc_resolv_conf() if ( -f "/etc/resolv.conf" && -x $host );
+
+unless ( -x $host ) {
+    $nagiosmsg .= "$host is missing or not executable, please fix..";
+    $retval = 1;
+}
+
+if ($nagiosmsg =~ /^$/) {
+    print "/etc/resolv.conf OK\n";
+} else {
+    print $nagiosmsg . "\n";
+}
+exit $retval;
diff --git a/nagios-plugins/check_shutdown b/nagios-plugins/check_shutdown
new file mode 100755
index 0000000..756e6fb
--- /dev/null
+++ b/nagios-plugins/check_shutdown
@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# Report when a reboot is in progress.  Useful to detect if
+# reboot-when-idle have been used on a server.
+
+set -e
+
+PATH=/bin:/sbin:/usr/sbin:/usr/bin:/local/bin:/local/sbin
+
+shutdownpid="`pgrep '^shutdown$'|head -1`"
+
+if [ "$shutdownpid" ] ; then
+    case "`uname -s`" in
+	Linux)
+	    cmd="`ps --no-headers --pid $shutdownpid -o command`"
+	    echo "REBOOT IN PROGRESS: $cmd"
+	    ;;
+	*)
+	    echo "REBOOT IN PROGRESS"
+	    ;;
+    esac
+    exit 1
+fi
+
+echo "OK - no shutdown running"
+exit 0
diff --git a/nagios-templates.cfg b/nagios-templates.cfg
index d751345..5b783ac 100644
--- a/nagios-templates.cfg
+++ b/nagios-templates.cfg
@@ -108,6 +108,16 @@ define command{
     command_line	/usr/lib/nagios/plugins/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$
 }
 
+define command {
+    command_name	check_etc_hosts
+    comamnd_line	/usr/lib/sitesummary/nagios-plugins/check_etc_hosts
+}
+
+define command {
+    command_name	check_etc_resolv
+    command_line	/usr/lib/sitesummary/nagios-plugins/check_etc_resolv
+}
+
 define command{
     command_name	check_http
     command_line	/usr/lib/nagios/plugins/check_http -H $HOSTADDRESS$ -I $HOSTADDRESS$
@@ -123,6 +133,11 @@ define command{
     command_line	/usr/lib/nagios/plugins/check_ping -H $HOSTADDRESS$ -w $ARG1$ -c $ARG2$
 }
 
+define command {
+    command_name	check_shutdown
+    command_line	/usr/lib/sitesummary/nagios-plugins/check_shutdown
+}
+
 define command{
     command_name	check_ssh
     command_line	/usr/lib/nagios/plugins/check_ssh $HOSTADDRESS$
diff --git a/sitesummary-nodes b/sitesummary-nodes
index 7260c0b..e313388 100755
--- a/sitesummary-nodes
+++ b/sitesummary-nodes
@@ -360,6 +360,19 @@ sub generate_nagios_config {
         print_nagios_service_check($remote, $hostname, "kernel status",
                                    "check_kernel_status");
 
+	# Detect bad DNS servers
+        print_nagios_service_check($remote, $hostname, "/etc/resolv.conf",
+                                   "check_etc_resolv");
+
+	# Detect hosts entries not matching DNS entries
+        print_nagios_service_check($remote, $hostname, "/etc/hosts",
+                                   "check_etc_hosts");
+
+	# Detect a shutdown in progress
+        print_nagios_service_check($remote, $hostname, "shutdown status",
+                                   "check_shutdown");
+
+
 #        print_nagios_service_check($remote, $hostname, "dhcp",
 #                                   "check_dhcp")
 #            if is_pkg_installed($hostid, "dhcp3-server");