From ddc497e0fba6334362847fdf698318b5549ead16 Mon Sep 17 00:00:00 2001
From: slinderud <Simen@linux.com>
Date: Sun, 10 Feb 2019 15:34:31 +0100
Subject: fixed templateing install and service file, updated install
 documentation and included new varnish vcl file

---
 ansible/roles/web/files/auth.vcl         |   1 +
 ansible/roles/web/files/template.service |  15 ++++
 ansible/roles/web/files/varnish.vcl      | 140 +++++++++++++++++++++++++++----
 ansible/roles/web/handlers/main.yml      |   8 +-
 ansible/roles/web/tasks/main.yml         |  19 ++++-
 5 files changed, 166 insertions(+), 17 deletions(-)
 create mode 100644 ansible/roles/web/files/auth.vcl
 create mode 100644 ansible/roles/web/files/template.service

(limited to 'ansible')

diff --git a/ansible/roles/web/files/auth.vcl b/ansible/roles/web/files/auth.vcl
new file mode 100644
index 0000000..81c21eb
--- /dev/null
+++ b/ansible/roles/web/files/auth.vcl
@@ -0,0 +1 @@
+req.http.Authorization != "Basic dGVjaDpydWxlcw==" 
diff --git a/ansible/roles/web/files/template.service b/ansible/roles/web/files/template.service
new file mode 100644
index 0000000..b33b8b8
--- /dev/null
+++ b/ansible/roles/web/files/template.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Gondul Templating
+Documentation=http://google.com
+After=network.target
+
+[Service]
+ExecStart=/opt/gondul/templating/templating.py -h ::1 -p 8081
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/ansible/roles/web/files/varnish.vcl b/ansible/roles/web/files/varnish.vcl
index 2cdbbfe..c0dc8b6 100644
--- a/ansible/roles/web/files/varnish.vcl
+++ b/ansible/roles/web/files/varnish.vcl
@@ -1,21 +1,97 @@
 # vim: ts=8:expandtab:sw=4:softtabstop=4
 
+# VCL for Gondul - also requires auth.vcl (see further down)
+# Also uses hitch and acmetool for ssl
 vcl 4.0;
 
+import std;
+
+# API - apache
 backend default {
     .host = "::1";
     .port = "8080";
 }
 
+# Templating engine
+backend templating {
+    .host = "::1";
+    .port = "8081";
+}
+
+# Definitely not influx
 backend influx {
     .host = "::1";
     .port = "8086";
 }
 
+# For certbot
+# WTF... isn't this apache? Apparently acmetool listens on port 402
+backend acmetool {
+   .host = "::1";
+   .port = "402";
+}
+
+# White-list localhost - PLEASE make sure this is actually smart
+acl white {
+    "::1";
+    "127.0.0.0"/8;
+    #"172.16.0.0"/12;
+    #"192.168.0.0"/16;
+    #"10.0.0.0"/8;
+}
+
+# vcl_recv is "prep-processing of requests
 sub vcl_recv {
+    # Handle certbot by passing /.well-known to acmetool
+    if (req.url ~ "^/.well-known/acme-challenge/") {
+       set req.backend_hint = acmetool;
+       return(pass);
+    }
+
+    # Redirect to https - note that this does NOT happen for 
+    # "whitelisted" stuff - e.g., templating engine.
+    #disabled as we haven't fixd hitch for ssl termination
+    #if (std.port(local.ip) == 80 && client.ip !~ white) {
+    #    set req.http.x-redir = "https://" + req.http.host + req.url;
+    #    return(synth(301));
+    #}
+
+    # Basic authentication ....
+    # We include the following from /etc/varnish/auth.vcl, to keep passwords
+    # out of default vcl:
+    # req.http.Authorization != "Basic AAAA"
+    #
+    # where AAAA is the result of:
+    # echo -n user:password | base64.
+    # Example:
+    # kly@jade:~$ echo -n tech:rules | base64 
+    # dGVjaDpydWxlcw==
+    # # cat /etc/varnish/auth.vcl 
+    # req.http.Authorization != "Basic dGVjaDpydWxlcw=="
+    if (client.ip !~ white && 
+            include "/etc/varnish/auth.vcl";) {
+        return(synth(401));
+    } else {
+        unset req.http.Authorization;
+        set req.http.X-Webauth-User = "admin";
+    }
+
+    if (req.url ~ "^/api/templates") {
+        set req.url = regsub(req.url,"^/api/templates","");
+        set req.backend_hint = templating;
+    }
+    
+    if (req.url ~ "^/query") {
+        set req.backend_hint = influx;
+    }
+
+    # More human-typable URL
     if (req.url ~ "^/where" || req.url ~ "^/location") {
         set req.url = "/api/public/location";
     }
+    
+    # Fairly standard filtering. Default VCL will do "pipe", which is
+    # pointless for us.
     if (req.method != "GET" &&
         req.method != "HEAD" &&
         req.method != "PUT" &&
@@ -27,50 +103,84 @@ sub vcl_recv {
         return (synth(418,"LOLOLOL"));
     }
 
-    if (req.url ~ "^/query") {
-        set req.backend_hint = influx;
-    }
-
+    # We can only cache GET/HEAD requests.
     if (req.method != "GET" && req.method != "HEAD") {
-        /* We only deal with GET and HEAD by default */
         return (pass);
     }
 
-    # Brukes ikke. Cookies er for nubs.
+    # We don't use cookies - so get rid of them so we don't mess up the cache
+    # by accident.
     unset req.http.Cookie;
 
-    # Tvinges gjennom for å cache med authorization-skrot.
+    # Force hash, since we want to cache with Authorization headers
     return (hash);
 }
 
-
-# Rosa magi
+# vcl_hash runs right after vcl_recv, and determines what
+# is "unique", e.g., what's part of the hash key. We simply
+# add the Authorization header, allowing caching of authenticated
+# content.
+# NOTE: We do NOT run "return" so it will fall back to the default
+# vcl builtin, which will add ip/host and URL as you'd expect.
 sub vcl_hash {
-    # Wheee. Legg til authorization-headeren i hashen.
     hash_data(req.http.authorization);
 }
 
-# Mauve magi. Hva nå enn det er.
-# Dette er WIP - Skal flyttes til backend
+# vcl_synth is run for "synthetic messages": responses generated internally
+# from Varnish, typically error messages or "return (synth...)"
+sub vcl_synth {
+    if (resp.status == 401) {
+        set resp.http.WWW-Authenticate = {"Basic realm="WHAT .... is your favorite color?""};
+    }
+
+    # Second part of redirect-logic
+    if (resp.status == 301) {
+        set resp.http.Location = req.http.x-redir;
+        return (deliver);
+    }
+}
+
+# vcl_backend_response is run when we have a reply from a backend,
+# allowing us to massage the backend response. We wish to do as little
+# as possible here to keep things transparent.
 sub vcl_backend_response {
+    # Expose the URL used for debug purposes and future
+    # cache invalidation.
     set beresp.http.x-url = bereq.url;
+
+    # If the backend response supplies the "x-ban" HTTP response
+    # header, then invalidate based on it. This is used for for
+    # invalidating e.g. switch-management if a switch is added, or the oplog.
     if (beresp.http.x-ban) {
         ban("obj.http.x-url ~ " + beresp.http.x-ban);
     }
+    
+    # Force gzip on text-based content so we don't have to
+    # rely on Apache. 
+    if (beresp.http.content-type ~ "text") {
+        set beresp.do_gzip = true;
+    }
+
+    # Do some hand-crafting for influx. Should probably be
+    # improved... e.g.: with checking error codes.
     if (bereq.url ~ "/query") {
-        # Let's blindly cache influx requests for 5+10s
         set beresp.http.Cache-Control = "max-age=5";
         unset beresp.http.Pragma;
         set beresp.uncacheable = false;
         set beresp.grace = 10s;
         set beresp.ttl = 5s;
     }
+    
+    # Wait, nvm, we catch non-200 here and make them actually cacheable for 5
+    # seconds - we don't want to nuke a backend just because it has ...issues. 
     if (beresp.status != 200) {
         set beresp.uncacheable = false;
         set beresp.ttl = 5s;
     }
-    if (bereq.url ~ "\.(html|css|js)") {
-        # Mainly for ease of development
+    # So for html/css/js there really is no sensible blackend to set
+    # smart TTL, so we hard-code it to 10s. 10s can be a bit annoying
+    # for development, but works.
+    if (bereq.url ~ "\.(html|css|js)" || bereq.url ~ "^/[^/.]*") {
         set beresp.ttl = 10s;
     }
 }
diff --git a/ansible/roles/web/handlers/main.yml b/ansible/roles/web/handlers/main.yml
index 5d36f51..4550f9d 100644
--- a/ansible/roles/web/handlers/main.yml
+++ b/ansible/roles/web/handlers/main.yml
@@ -11,6 +11,12 @@
     daemon_reload: yes
     name: varnish
 
-- name: reload systemd
+- name: restart gondul-template
   systemd:
+    state: restarted
     daemon_reload: yes
+    name: gondul-template
+
+- name: reload systemd
+  systemd:
+    daemon_reload: yes
\ No newline at end of file
diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml
index 1c89947..466dec8 100644
--- a/ansible/roles/web/tasks/main.yml
+++ b/ansible/roles/web/tasks/main.yml
@@ -31,7 +31,9 @@
           'libxml2-dev',
           'build-essential',
           'cpanminus',
-          'apt-transport-https']
+          'apt-transport-https',
+          'python-setuptools',
+          'python3-pip']
       state: present
 
 - name: Add packagecloud.io Varnish apt key.
@@ -59,6 +61,12 @@
   notify:
     - reload systemd
 
+- name: Copy varnish-acl config
+  copy:
+      dest: /etc/varnish/auth.vcl
+      src: auth.vcl
+  notify: restart varnish
+
 - name: Copy varnish config
   copy:
       dest: /etc/varnish/default.vcl
@@ -87,3 +95,12 @@
 - name: Install InfluxDB module
   cpanm:
     name: AnyEvent::InfluxDB
+
+- name: Install python dependencies
+  command: /usr/bin/pip3 install -r /opt/gondul/templating/requirements.txt
+
+- name: add systemd service file for gondul-template
+  copy:
+    dest: /etc/systemd/system/gondul-template.service
+    src: template.service
+  notify: restart gondul-template
\ No newline at end of file
-- 
cgit v1.2.3


From f1526d1b25cd04f3857d754f7e98d0729fe1cc49 Mon Sep 17 00:00:00 2001
From: Lasse Haugen <haugen.lasse@gmail.com>
Date: Tue, 12 Feb 2019 09:49:21 +0100
Subject: added newlines

---
 ansible/roles/web/handlers/main.yml | 3 ++-
 ansible/roles/web/tasks/main.yml    | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

(limited to 'ansible')

diff --git a/ansible/roles/web/handlers/main.yml b/ansible/roles/web/handlers/main.yml
index 4550f9d..77d887b 100644
--- a/ansible/roles/web/handlers/main.yml
+++ b/ansible/roles/web/handlers/main.yml
@@ -19,4 +19,5 @@
 
 - name: reload systemd
   systemd:
-    daemon_reload: yes
\ No newline at end of file
+    daemon_reload: yes
+
diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml
index 466dec8..1ee2862 100644
--- a/ansible/roles/web/tasks/main.yml
+++ b/ansible/roles/web/tasks/main.yml
@@ -103,4 +103,5 @@
   copy:
     dest: /etc/systemd/system/gondul-template.service
     src: template.service
-  notify: restart gondul-template
\ No newline at end of file
+  notify: restart gondul-template
+
-- 
cgit v1.2.3


From f20c48db1b51d2c44eb62756326edfb9ec1cfbd0 Mon Sep 17 00:00:00 2001
From: Lasse Haugen <haugen.lasse@gmail.com>
Date: Tue, 12 Feb 2019 09:58:27 +0100
Subject: Added path to templates

---
 ansible/roles/web/files/gondul-template.service | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 ansible/roles/web/files/gondul-template.service

(limited to 'ansible')

diff --git a/ansible/roles/web/files/gondul-template.service b/ansible/roles/web/files/gondul-template.service
new file mode 100644
index 0000000..7a2a231
--- /dev/null
+++ b/ansible/roles/web/files/gondul-template.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Gondul Templating
+Documentation=http://google.com
+After=network-online.target
+
+[Service]
+ExecStart=/opt/gondul/templating/templating.py -h ::1 -p 8081 -t /opt/gondul/web/templates/
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target
+
-- 
cgit v1.2.3


From dac472463220592672f69f8f589caad5f425f9c3 Mon Sep 17 00:00:00 2001
From: Lasse Haugen <haugen.lasse@gmail.com>
Date: Tue, 12 Feb 2019 09:58:42 +0100
Subject: renamed service

---
 ansible/roles/web/files/template.service | 15 ---------------
 ansible/roles/web/tasks/main.yml         |  2 +-
 2 files changed, 1 insertion(+), 16 deletions(-)
 delete mode 100644 ansible/roles/web/files/template.service

(limited to 'ansible')

diff --git a/ansible/roles/web/files/template.service b/ansible/roles/web/files/template.service
deleted file mode 100644
index b33b8b8..0000000
--- a/ansible/roles/web/files/template.service
+++ /dev/null
@@ -1,15 +0,0 @@
-[Unit]
-Description=Gondul Templating
-Documentation=http://google.com
-After=network.target
-
-[Service]
-ExecStart=/opt/gondul/templating/templating.py -h ::1 -p 8081
-MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-
-[Install]
-WantedBy=multi-user.target
-
diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml
index 1ee2862..9d2ff2b 100644
--- a/ansible/roles/web/tasks/main.yml
+++ b/ansible/roles/web/tasks/main.yml
@@ -102,6 +102,6 @@
 - name: add systemd service file for gondul-template
   copy:
     dest: /etc/systemd/system/gondul-template.service
-    src: template.service
+    src: gondul-template.service
   notify: restart gondul-template
 
-- 
cgit v1.2.3


From 31e7ffd29f6bc74d742b24d5a9a1c463507857c0 Mon Sep 17 00:00:00 2001
From: slinderud <Simen@linux.com>
Date: Tue, 12 Feb 2019 10:30:03 +0100
Subject: moved from pip to apt

---
 ansible/roles/web/tasks/main.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

(limited to 'ansible')

diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml
index 9d2ff2b..ddbf74d 100644
--- a/ansible/roles/web/tasks/main.yml
+++ b/ansible/roles/web/tasks/main.yml
@@ -32,8 +32,8 @@
           'build-essential',
           'cpanminus',
           'apt-transport-https',
-          'python-setuptools',
-          'python3-pip']
+          'python3-netaddr',
+          'python3-flask']
       state: present
 
 - name: Add packagecloud.io Varnish apt key.
@@ -96,9 +96,6 @@
   cpanm:
     name: AnyEvent::InfluxDB
 
-- name: Install python dependencies
-  command: /usr/bin/pip3 install -r /opt/gondul/templating/requirements.txt
-
 - name: add systemd service file for gondul-template
   copy:
     dest: /etc/systemd/system/gondul-template.service
-- 
cgit v1.2.3


From fd6a6a180a234f569b2977ae44adbc264eab4807 Mon Sep 17 00:00:00 2001
From: slinderud <Simen@linux.com>
Date: Tue, 12 Feb 2019 10:32:01 +0100
Subject: removed copy-paste stuff from service file

---
 ansible/roles/web/files/gondul-template.service | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'ansible')

diff --git a/ansible/roles/web/files/gondul-template.service b/ansible/roles/web/files/gondul-template.service
index 7a2a231..edde9a3 100644
--- a/ansible/roles/web/files/gondul-template.service
+++ b/ansible/roles/web/files/gondul-template.service
@@ -6,9 +6,6 @@ After=network-online.target
 [Service]
 ExecStart=/opt/gondul/templating/templating.py -h ::1 -p 8081 -t /opt/gondul/web/templates/
 MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
 
 [Install]
 WantedBy=multi-user.target
-- 
cgit v1.2.3


From e945540c2f55b67119db19ed1a60ca4dbec59ace Mon Sep 17 00:00:00 2001
From: Lasse Haugen <haugen.lasse@gmail.com>
Date: Tue, 12 Feb 2019 11:02:48 +0100
Subject: Update service file

---
 ansible/roles/web/files/gondul-template.service |  5 +++--
 ansible/roles/web/files/template.service        | 15 ---------------
 2 files changed, 3 insertions(+), 17 deletions(-)
 delete mode 100644 ansible/roles/web/files/template.service

(limited to 'ansible')

diff --git a/ansible/roles/web/files/gondul-template.service b/ansible/roles/web/files/gondul-template.service
index edde9a3..148e374 100644
--- a/ansible/roles/web/files/gondul-template.service
+++ b/ansible/roles/web/files/gondul-template.service
@@ -1,11 +1,12 @@
 [Unit]
 Description=Gondul Templating
-Documentation=http://google.com
+Documentation=https://github.com/gathering/gondul
 After=network-online.target
 
 [Service]
 ExecStart=/opt/gondul/templating/templating.py -h ::1 -p 8081 -t /opt/gondul/web/templates/
-MountFlags=slave
+ExecReload=/usr/bin/kill -HUP $MAINPID
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target
diff --git a/ansible/roles/web/files/template.service b/ansible/roles/web/files/template.service
deleted file mode 100644
index b33b8b8..0000000
--- a/ansible/roles/web/files/template.service
+++ /dev/null
@@ -1,15 +0,0 @@
-[Unit]
-Description=Gondul Templating
-Documentation=http://google.com
-After=network.target
-
-[Service]
-ExecStart=/opt/gondul/templating/templating.py -h ::1 -p 8081
-MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-
-[Install]
-WantedBy=multi-user.target
-
-- 
cgit v1.2.3


From b018cde6bdd90d219eabb29d4b5bb200e50191c9 Mon Sep 17 00:00:00 2001
From: Lasse Haugen <haugen.lasse@gmail.com>
Date: Tue, 12 Feb 2019 11:10:16 +0100
Subject: service file cleanup

---
 ansible/roles/ping/files/gondul-pinger.service | 10 ++++------
 ansible/roles/snmp/files/gondul-snmp.service   | 10 ++++------
 ansible/roles/web/handlers/main.yml            |  1 -
 ansible/roles/web/tasks/main.yml               |  1 -
 4 files changed, 8 insertions(+), 14 deletions(-)

(limited to 'ansible')

diff --git a/ansible/roles/ping/files/gondul-pinger.service b/ansible/roles/ping/files/gondul-pinger.service
index fc9cabd..5f7bbce 100644
--- a/ansible/roles/ping/files/gondul-pinger.service
+++ b/ansible/roles/ping/files/gondul-pinger.service
@@ -1,14 +1,12 @@
 [Unit]
 Description=Gondul ping collector
-Documentation=http://google.com
-After=network.target
+Documentation=https://github.com/gathering/gondul
+After=network-online.target
 
 [Service]
 ExecStart=/opt/gondul/collectors/ping.pl
-MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
+ExecReload=/usr/bin/kill -HUP $MAINPID
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target
diff --git a/ansible/roles/snmp/files/gondul-snmp.service b/ansible/roles/snmp/files/gondul-snmp.service
index e5f2179..d7bd134 100644
--- a/ansible/roles/snmp/files/gondul-snmp.service
+++ b/ansible/roles/snmp/files/gondul-snmp.service
@@ -1,14 +1,12 @@
 [Unit]
 Description=Gondul snmp collector
-Documentation=http://google.com
-After=network.target
+Documentation=https://github.com/gathering/gondul
+After=network-online.target
 
 [Service]
 ExecStart=/opt/gondul/collectors/snmpfetchng.pl
-MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
+ExecReload=/usr/bin/kill -HUP $MAINPID
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target
diff --git a/ansible/roles/web/handlers/main.yml b/ansible/roles/web/handlers/main.yml
index 77d887b..624574d 100644
--- a/ansible/roles/web/handlers/main.yml
+++ b/ansible/roles/web/handlers/main.yml
@@ -20,4 +20,3 @@
 - name: reload systemd
   systemd:
     daemon_reload: yes
-
diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml
index ddbf74d..b10593e 100644
--- a/ansible/roles/web/tasks/main.yml
+++ b/ansible/roles/web/tasks/main.yml
@@ -101,4 +101,3 @@
     dest: /etc/systemd/system/gondul-template.service
     src: gondul-template.service
   notify: restart gondul-template
-
-- 
cgit v1.2.3


From 65be318e53ceb6f5e6022e24fd585930f4477dda Mon Sep 17 00:00:00 2001
From: Lasse Haugen <haugen.lasse@gmail.com>
Date: Tue, 12 Feb 2019 11:19:00 +0100
Subject: newline fuckups

---
 ansible/roles/web/handlers/main.yml | 1 +
 ansible/roles/web/tasks/main.yml    | 1 +
 2 files changed, 2 insertions(+)

(limited to 'ansible')

diff --git a/ansible/roles/web/handlers/main.yml b/ansible/roles/web/handlers/main.yml
index 624574d..77d887b 100644
--- a/ansible/roles/web/handlers/main.yml
+++ b/ansible/roles/web/handlers/main.yml
@@ -20,3 +20,4 @@
 - name: reload systemd
   systemd:
     daemon_reload: yes
+
diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml
index b10593e..ddbf74d 100644
--- a/ansible/roles/web/tasks/main.yml
+++ b/ansible/roles/web/tasks/main.yml
@@ -101,3 +101,4 @@
     dest: /etc/systemd/system/gondul-template.service
     src: gondul-template.service
   notify: restart gondul-template
+
-- 
cgit v1.2.3