From 1cb5d90536de7fbbc812149c3c5f57842766342e Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Sun, 13 Jan 2019 21:21:51 +0100 Subject: Update to debian9 --- INSTALLING.rst | 8 ++++---- ansible/roles/postgres/tasks/main.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/INSTALLING.rst b/INSTALLING.rst index 2a4bcb8..b04e226 100644 --- a/INSTALLING.rst +++ b/INSTALLING.rst @@ -4,8 +4,8 @@ Installing Gondul Requirements ------------ -- Debian Stable (jessie) with backports (possibly newer) -- Ansible v2.1 or newer (recommended: from backports) +- Debian Stable (stretch) with backports (possibly newer) +- Ansible v2.7 or newer (recommended: from backports) - A harddrive of some size. Recommended: SSD. 200GB should be sufficient for almost any party. - CPU: Depends on client-load. Most semi-modern cpu's will be more than @@ -27,9 +27,9 @@ As root: # YOURUSER=kly # apt-get install sudo git # echo ${YOURUSER} ALL=NOPASSWD: ALL >> /etc/sudoers - # echo deb http://http.debian.net/debian jessie-backports main non-free contrib > /etc/apt/sources.list.d/bp.list + # echo deb http://http.debian.net/debian stretch-backports main non-free contrib > /etc/apt/sources.list.d/bp.list # apt-get update - # apt-get install ansible/jessie-backports + # apt-get install ansible/stretch-backports As ``$YOURUSER``:: diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 9adeafe..3073046 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -3,12 +3,12 @@ name: "{{ item }}" state: present with_items: - - postgresql-9.4 + - postgresql - python-psycopg2 - sudo - name: Drop postgresql-config copy: - dest: /etc/postgresql/9.4/main/postgresql.conf + dest: /etc/postgresql/9.6/main/postgresql.conf src: postgresql.conf - name: Add db to hosts lineinfile: -- cgit v1.2.3 From b25e34d67168ac2bb1d74943940cfa72f464a137 Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Sun, 13 Jan 2019 21:53:43 +0100 Subject: changed to 9.6 --- ansible/roles/postgres/files/postgresql.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/roles/postgres/files/postgresql.conf b/ansible/roles/postgres/files/postgresql.conf index cf6e67e..7c45ea4 100644 --- a/ansible/roles/postgres/files/postgresql.conf +++ b/ansible/roles/postgres/files/postgresql.conf @@ -38,15 +38,15 @@ # The default values of these variables are driven from the -D command-line # option or PGDATA environment variable, represented here as ConfigDir. -data_directory = '/var/lib/postgresql/9.4/main' # use data in another directory +data_directory = '/var/lib/postgresql/9.6/main' # use data in another directory # (change requires restart) -hba_file = '/etc/postgresql/9.4/main/pg_hba.conf' # host-based authentication file +hba_file = '/etc/postgresql/9.6/main/pg_hba.conf' # host-based authentication file # (change requires restart) -ident_file = '/etc/postgresql/9.4/main/pg_ident.conf' # ident configuration file +ident_file = '/etc/postgresql/9.6/main/pg_ident.conf' # ident configuration file # (change requires restart) # If external_pid_file is not explicitly set, no extra PID file is written. -external_pid_file = '/var/run/postgresql/9.4-main.pid' # write an extra PID file +external_pid_file = '/var/run/postgresql/9.6-main.pid' # write an extra PID file # (change requires restart) @@ -450,7 +450,7 @@ log_timezone = 'UTC' #track_functions = none # none, pl, all #track_activity_query_size = 1024 # (change requires restart) #update_process_title = on -stats_temp_directory = '/var/run/postgresql/9.4-main.pg_stat_tmp' +stats_temp_directory = '/var/run/postgresql/9.6-main.pg_stat_tmp' # - Statistics Monitoring - -- cgit v1.2.3 From f5e3ab81ac2d6810b6df0be1b798f5c9100032ac Mon Sep 17 00:00:00 2001 From: slinderud Date: Sun, 13 Jan 2019 23:19:22 +0100 Subject: Fixed ansible postgres bug --- ansible/roles/postgres/tasks/main.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 9adeafe..6283a75 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -1,3 +1,6 @@ +# Postgres +# https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 +# - name: Install db-packages apt: name: "{{ item }}" @@ -28,12 +31,27 @@ become_user: postgres postgresql_db: name: nms + # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 + become: true + become_user: postgres + vars: + ansible_ssh_pipelining: true - name: Ensure a valid postgres-user become_user: postgres postgresql_user: db: nms name: nms password: risbrod + # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 + become: true + become_user: postgres + vars: + ansible_ssh_pipelining: true - name: Import SQL become_user: postgres shell: psql nms < /opt/gondul/ansible/roles/postgres/files/schema.sql + # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 + become: true + become_user: postgres + vars: + ansible_ssh_pipelining: true -- cgit v1.2.3 From 7caeb2c763270636261dba584ba489d9e66aca4f Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Mon, 14 Jan 2019 15:54:37 +0100 Subject: Change to new ansible style --- ansible/roles/postgres/tasks/main.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 3073046..61befa5 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -1,11 +1,7 @@ - name: Install db-packages apt: - name: "{{ item }}" + name: ['postgresql', 'python-psycopg2', 'sudo'] state: present - with_items: - - postgresql - - python-psycopg2 - - sudo - name: Drop postgresql-config copy: dest: /etc/postgresql/9.6/main/postgresql.conf -- cgit v1.2.3 From 4bb5325285119407fc60034c52ce91615698f36f Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Mon, 14 Jan 2019 15:57:05 +0100 Subject: Removed dupicates --- ansible/roles/postgres/tasks/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 7aef2a6..96f5a0b 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -24,7 +24,6 @@ state: present line: "{{ whoami.stdout }} ALL=(postgres) NOPASSWD: ALL" - name: Make postgres-db - become_user: postgres postgresql_db: name: nms # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 @@ -33,7 +32,6 @@ vars: ansible_ssh_pipelining: true - name: Ensure a valid postgres-user - become_user: postgres postgresql_user: db: nms name: nms @@ -44,7 +42,6 @@ vars: ansible_ssh_pipelining: true - name: Import SQL - become_user: postgres shell: psql nms < /opt/gondul/ansible/roles/postgres/files/schema.sql # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 become: true -- cgit v1.2.3 From ca5bf27b0824b691af9895e4ab8d6de378866d12 Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Mon, 14 Jan 2019 16:03:56 +0100 Subject: Change to new ansible style --- ansible/roles/influx/tasks/main.yml | 4 +-- ansible/roles/web/tasks/main.yml | 65 ++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/ansible/roles/influx/tasks/main.yml b/ansible/roles/influx/tasks/main.yml index 236c2a9..cb3a784 100644 --- a/ansible/roles/influx/tasks/main.yml +++ b/ansible/roles/influx/tasks/main.yml @@ -1,9 +1,7 @@ - name: Install apt-packages apt: - name: "{{ item }}" + name: 'apt-transport-https' state: present - with_items: - apt-transport-https - name: Import InfluxDB GPG signing key apt_key: url=https://repos.influxdata.com/influxdb.key state=present diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml index dce3f4c..e687d97 100644 --- a/ansible/roles/web/tasks/main.yml +++ b/ansible/roles/web/tasks/main.yml @@ -1,39 +1,38 @@ - name: Install front-packages apt: - name: "{{ item }}" + name: [ + 'libcapture-tiny-perl', + 'libcommon-sense-perl', + 'libdata-dumper-simple-perl', + 'libdbd-pg-perl', + 'libdbi-perl', + 'libdigest-perl', + 'libgd-perl', + 'libgeo-ip-perl', + 'libhtml-parser-perl', + 'libhtml-template-perl', + 'libjson-perl', + 'libjson-xs-perl', + 'libnetaddr-ip-perl', + 'libnet-cidr-perl', + 'libnet-ip-perl', + 'libnet-oping-perl', + 'libnet-rawip-perl', + 'libsnmp-perl', + 'libsocket6-perl', + 'libsocket-perl', + 'libswitch-perl', + 'libtimedate-perl', + 'perl', + 'perl-base', + 'perl-modules', + 'libfreezethaw-perl', + 'apache2', + 'libxml2-dev', + 'build-essential', + 'cpanminus', + 'apt-transport-https'] state: present - with_items: - - libcapture-tiny-perl - - libcommon-sense-perl - - libdata-dumper-simple-perl - - libdbd-pg-perl - - libdbi-perl - - libdigest-perl - - libgd-perl - - libgeo-ip-perl - - libhtml-parser-perl - - libhtml-template-perl - - libjson-perl - - libjson-xs-perl - - libnetaddr-ip-perl - - libnet-cidr-perl - - libnet-ip-perl - - libnet-oping-perl - - libnet-rawip-perl - - libsnmp-perl - - libsocket6-perl - - libsocket-perl - - libswitch-perl - - libtimedate-perl - - perl - - perl-base - - perl-modules - - libfreezethaw-perl - - apache2 - - libxml2-dev - - build-essential - - cpanminus - - apt-transport-https - name: Add packagecloud.io Varnish apt key. apt_key: -- cgit v1.2.3 From 5fc667a40fad97d289d45aa919a66222341ed44b Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Mon, 14 Jan 2019 22:27:38 +0100 Subject: some ansible-lint fixes --- ansible/roles/postgres/tasks/main.yml | 5 +---- ansible/roles/snmp/tasks/main.yml | 6 ++++-- ansible/roles/web/handlers/main.yml | 15 ++++++++++++--- ansible/roles/web/tasks/main.yml | 3 ++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml index 96f5a0b..902eeb9 100644 --- a/ansible/roles/postgres/tasks/main.yml +++ b/ansible/roles/postgres/tasks/main.yml @@ -1,4 +1,4 @@ -# Postgres +# Postgres # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 # - name: Install db-packages @@ -26,7 +26,6 @@ - name: Make postgres-db postgresql_db: name: nms - # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 become: true become_user: postgres vars: @@ -36,14 +35,12 @@ db: nms name: nms password: risbrod - # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 become: true become_user: postgres vars: ansible_ssh_pipelining: true - name: Import SQL shell: psql nms < /opt/gondul/ansible/roles/postgres/files/schema.sql - # https://github.com/ansible/ansible/issues/16048#issuecomment-229012509 become: true become_user: postgres vars: diff --git a/ansible/roles/snmp/tasks/main.yml b/ansible/roles/snmp/tasks/main.yml index 2cb7165..6056a25 100644 --- a/ansible/roles/snmp/tasks/main.yml +++ b/ansible/roles/snmp/tasks/main.yml @@ -1,8 +1,10 @@ -- file: +- name: Create data directory + file: path: /opt/gondul/data state: directory mode: 0755 -- stat: +- name: register mibdir + stat: path: /opt/gondul/data/mibs register: mibdir - name: Get mibs diff --git a/ansible/roles/web/handlers/main.yml b/ansible/roles/web/handlers/main.yml index 3f71f4c..6c644ca 100644 --- a/ansible/roles/web/handlers/main.yml +++ b/ansible/roles/web/handlers/main.yml @@ -1,7 +1,16 @@ --- - name: restart apache - service: name=apache2 state=restarted + systemd: + state: restarted + daemon_reload: yes + name: apache2 + - name: restart varnish - service: name=varnish state=restarted + systemd: + state: restarted + daemon_reload: yes + name: varnish + - name: reload systemd - command: systemctl daemon-reload + systemd: + daemon_reload: yes \ No newline at end of file diff --git a/ansible/roles/web/tasks/main.yml b/ansible/roles/web/tasks/main.yml index e687d97..1c89947 100644 --- a/ansible/roles/web/tasks/main.yml +++ b/ansible/roles/web/tasks/main.yml @@ -84,5 +84,6 @@ ignore_errors: true notify: restart apache -- cpanm: +- name: Install InfluxDB module + cpanm: name: AnyEvent::InfluxDB -- cgit v1.2.3 From 647b31f8ea2bb341fe989e8beafa79abd8e8d57c Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Mon, 14 Jan 2019 23:02:05 +0100 Subject: newline fix --- ansible/roles/web/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/web/handlers/main.yml b/ansible/roles/web/handlers/main.yml index 6c644ca..5d36f51 100644 --- a/ansible/roles/web/handlers/main.yml +++ b/ansible/roles/web/handlers/main.yml @@ -13,4 +13,4 @@ - name: reload systemd systemd: - daemon_reload: yes \ No newline at end of file + daemon_reload: yes -- cgit v1.2.3 From aedafb4fb169be914782e4b72503e655b7e1bc41 Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Mon, 14 Jan 2019 23:02:25 +0100 Subject: Install basic packages --- ansible/roles/common/tasks/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index 562af85..7b402c3 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -4,3 +4,12 @@ - git-all - git-gondul git: repo={{ git_repo }} dest=/opt/gondul update=no accept_hostkey=yes version={{ git_branch }} + +- name: Install basic packages + action: > + {{ ansible_pkg_mgr }} name={{ item }} state=present update_cache=yes + with_items: + - curl + - vim + - git + - iptables-persistent \ No newline at end of file -- cgit v1.2.3 From ffd8a012c81534fe39eae2a8033679d15ca92abc Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Mon, 14 Jan 2019 23:06:41 +0100 Subject: change to apt module --- ansible/roles/common/tasks/main.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index 7b402c3..e573383 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -6,10 +6,10 @@ git: repo={{ git_repo }} dest=/opt/gondul update=no accept_hostkey=yes version={{ git_branch }} - name: Install basic packages - action: > - {{ ansible_pkg_mgr }} name={{ item }} state=present update_cache=yes - with_items: - - curl - - vim - - git - - iptables-persistent \ No newline at end of file + apt: + name: [ + 'curl', + 'vim', + 'git', + 'iptables-persistent'] + state: present -- cgit v1.2.3 From bb1cc93d51efcd24d77b51187638e9616c061ed6 Mon Sep 17 00:00:00 2001 From: Lasse Haugen Date: Sat, 19 Jan 2019 19:23:16 +0100 Subject: Moved git install before git pull --- ansible/roles/common/tasks/main.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index e573383..4c27c32 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -1,10 +1,3 @@ -- name: Gondul-repo - become: true - tags: - - git-all - - git-gondul - git: repo={{ git_repo }} dest=/opt/gondul update=no accept_hostkey=yes version={{ git_branch }} - - name: Install basic packages apt: name: [ @@ -13,3 +6,10 @@ 'git', 'iptables-persistent'] state: present + +- name: Gondul-repo + become: true + tags: + - git-all + - git-gondul + git: repo={{ git_repo }} dest=/opt/gondul update=no accept_hostkey=yes version={{ git_branch }} -- cgit v1.2.3 From 4f9cd6706692eb6b1a000efb37cfe76e202e6d07 Mon Sep 17 00:00:00 2001 From: Kristian Lyngstol Date: Mon, 21 Jan 2019 19:23:52 +0100 Subject: Use ::1 instead of localhost to avoid multiple IPs Fixes #193 --- ansible/roles/web/files/varnish.vcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/web/files/varnish.vcl b/ansible/roles/web/files/varnish.vcl index b082971..2cdbbfe 100644 --- a/ansible/roles/web/files/varnish.vcl +++ b/ansible/roles/web/files/varnish.vcl @@ -3,12 +3,12 @@ vcl 4.0; backend default { - .host = "localhost"; + .host = "::1"; .port = "8080"; } backend influx { - .host = "localhost"; + .host = "::1"; .port = "8086"; } -- cgit v1.2.3 From d4cd5cfac3c3c4028575114708eb551d25e1933e Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Mon, 21 Jan 2019 23:11:51 +0100 Subject: Added slight rewrite of template webserver Moved to flask for more "free" things. Possibly easier to extend in the future. Added argparser so we are able to define settings from the command line. --- build/test/gondul-templating-test.Dockerfile | 6 +- templating/requirements.txt | 4 + templating/templating.py | 180 ++++++++++++--------------- 3 files changed, 85 insertions(+), 105 deletions(-) create mode 100644 templating/requirements.txt diff --git a/build/test/gondul-templating-test.Dockerfile b/build/test/gondul-templating-test.Dockerfile index 80c09c0..6dabc9c 100644 --- a/build/test/gondul-templating-test.Dockerfile +++ b/build/test/gondul-templating-test.Dockerfile @@ -1,7 +1,9 @@ FROM debian:jessie RUN apt-get update -RUN apt-get -y install \ - python3-jinja2 \ +RUN apt-get -y install \ + python3-jinja2 \ + python3-netaddr \ + python3-flask \ python3-requests RUN mkdir -p /opt/gondul diff --git a/templating/requirements.txt b/templating/requirements.txt new file mode 100644 index 0000000..20e96a0 --- /dev/null +++ b/templating/requirements.txt @@ -0,0 +1,4 @@ +jinja2 +requests +flask +netaddr diff --git a/templating/templating.py b/templating/templating.py index fff5d42..9584fe3 100755 --- a/templating/templating.py +++ b/templating/templating.py @@ -1,115 +1,89 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 + +import argparse +import traceback +import sys -import requests,traceback -from jinja2 import Template,Environment,FileSystemLoader,TemplateNotFound -import json import netaddr -import http.server -from enum import Enum +import requests + +from flask import Flask, request +from jinja2 import Environment, FileSystemLoader, TemplateNotFound endpoints = "read/networks read/oplog read/snmp read/switches-management public/distro-tree public/config public/dhcp public/dhcp-summary public/ping public/switches public/switch-state".split() -objects = dict() + +objects = {} + def getEndpoint(endpoint): - r = requests.get("http://localhost:80/api/%s" % endpoint) - if (r.status_code != 200): - raise Exception("Bad status code for endpoint %s: %s" % (endpoint, r.status_code)) + r = requests.get(f"http://localhost:80/api/{endpoint}") + if r.status_code != 200: + raise Exception(f"Bad status code for endpoint {endpoint}: {r.status_code}") return r.json() + def updateData(): for a in endpoints: objects[a] = getEndpoint(a) -def netmask(ip): - return netaddr.IPNetwork(ip).netmask -def cidr(ip): - return netaddr.IPNetwork(ip).prefixlen -def networkId(ip): - return netaddr.IPNetwork(ip).ip -def getFirstDhcpIp(ip): - return netaddr.IPNetwork(ip)[3] -def getLastDhcpIp(ip): - return netaddr.IPNetwork(ip)[-1] -def getDistro(src): - return src.split(":")[0] -def getPort(src): - return src.split(":")[1] -def getFirstFapIp(ip): - return netaddr.IPNetwork(ip)[netaddr.IPNetwork(ip).size/2] - -env = Environment(loader=FileSystemLoader(['templates/','/opt/gondul/data/templates', '/opt/gondul/web/templates']), trim_blocks=True) - -env.filters["netmask"] = netmask -env.filters["cidr"] = cidr -env.filters["networkId"] = networkId -env.filters["getFirstDhcpIp"] = getFirstDhcpIp -env.filters["getLastDhcpIp"] = getLastDhcpIp -env.filters["agentDistro"] = getDistro -env.filters["agentPort"] = getPort -env.filters["getFirstFapIP"] = getFirstFapIp - -class Mode(Enum): - Get = 1 - Post = 2 - -class MyHandler(http.server.BaseHTTPRequestHandler): - - options = dict() - - def parse_options(self): - self.url = self.path[1:] - self.options = dict() - if self.url.find("?") != -1: - (self.url, tmpoptions) = self.url.split("?") - tmptuples = tmpoptions.split("&") - for a in tmptuples: - (x,y) = a.split("=") - self.options[x] = y - - def generic(self, mode): - updateData() - self.parse_options() - body = "" - try: - if mode == Mode.Get: - template = env.get_template(self.url) - elif mode == Mode.Post: - length = self.headers.get('content-length') - if not length: - length = 0 - content = self.rfile.read(int(length)).decode('UTF-8') - template = env.from_string(content) - else: - raise Exception("Invalid mode") - - body = template.render(objects=objects, options=self.options) - self.send_response(200) - except TemplateNotFound as err: - body = "Template \"%s\" not found\n" % self.url - self.send_response(404) - except Exception as err: - body = ("Templating of \"%s\" failed to render. Most likely due to an error in the template. Error transcript:\n\n%s\n----\n\n%s\n" % (self.url, err, traceback.format_exc())) - if mode == Mode.Get: - self.send_response(400) - else: - self.send_response(500) - finally: - self.send_header('Cache-Control','max-age=5, s-maxage=1') - body = body.encode('UTF-8') - self.send_header('Content-Length', int(len(body))) - self.end_headers() - self.wfile.write(body) - self.wfile.flush() - - def do_GET(self): - self.generic(Mode.Get) - - def do_POST(self): - self.generic(Mode.Post) - -def run(server_class=http.server.HTTPServer, handler_class=http.server.BaseHTTPRequestHandler): - server_address = ('localhost', 8081) - httpd = server_class(server_address, handler_class) - httpd.serve_forever() - -run(handler_class=MyHandler) + +env = Environment(loader=FileSystemLoader([]), trim_blocks=True) + +env.filters["netmask"] = lambda ip: netaddr.IPNetwork(ip).netmask +env.filters["cidr"] = lambda ip: netaddr.IPNetwork(ip).prefixlen +env.filters["networkId"] = lambda ip: netaddr.IPNetwork(ip).ip +env.filters["getFirstDhcpIp"] = lambda ip: netaddr.IPNetwork(ip)[3] +env.filters["getLastDhcpIp"] = lambda ip: netaddr.IPNetwork(ip)[-1] +env.filters["agentDistro"] = lambda src: src.split(":")[0] +env.filters["agentPort"] = lambda src: src.split(":")[1] +env.filters["getFirstFapIP"] = lambda ip: netaddr.IPNetwork(ip)[netaddr.IPNetwork(ip).size / 2] + +app = Flask(__name__) + + +@app.after_request +def add_header(response): + if response.status_code == 200: + response.cache_control.max_age = 5 + response.cache_control.s_maxage = 1 + return response + + +@app.route("/", methods=["GET"]) +def root_get(path): + updateData() + try: + template = env.get_template(path) + body = template.render(objects=objects, options=request.args) + except TemplateNotFound: + return f'Template "{path}" not found\n', 404 + except Exception as err: + return f'Templating of "{path}" failed to render. Most likely due to an error in the template. Error transcript:\n\n{err}\n----\n\n{traceback.format_exe()}\n', 400 + return body, 200 + + +@app.route("/", methods=["POST", "PUT"]) +def root_post(path): + updateData() + try: + content = request.stream.read(int(request.headers["Content-Length"])) + template = env.from_string(content.decode("utf-8")) + body = template.render(objects=objects, options=request.args) + except Exception as err: + return f'Templating of "{path}" failed to render. Most likely due to an error in the template. Error transcript:\n\n{err}\n----\n\n{traceback.format_exe()}\n', 400 + return body, 200 + + +parser = argparse.ArgumentParser(description="Process templates for gondul.", add_help=False) +parser.add_argument("-t", "--templates", type=str, nargs="+", help="location of templates") +parser.add_argument("-h", "--host", type=str, default="127.0.0.1", help="host address") +parser.add_argument("-p", "--port", type=int, default=8080, help="host port") +parser.add_argument("-d", "--debug", action="store_true", help="enable debug mode") + +args = parser.parse_args() +env.loader.searchpath = args.templates + +if not sys.argv[1:]: + parser.print_help() + +app.run(host=args.host, port=args.port, debug=args.debug) -- cgit v1.2.3 From 115fe3455e6c561723cf3ae0de1856a2dd661d8e Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 22 Jan 2019 17:04:47 +0100 Subject: Added python 3.5 compatible string formatting --- templating/templating.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templating/templating.py b/templating/templating.py index 9584fe3..03ad180 100755 --- a/templating/templating.py +++ b/templating/templating.py @@ -16,9 +16,9 @@ objects = {} def getEndpoint(endpoint): - r = requests.get(f"http://localhost:80/api/{endpoint}") + r = requests.get("http://localhost:80/api/{}".format(endpoint)) if r.status_code != 200: - raise Exception(f"Bad status code for endpoint {endpoint}: {r.status_code}") + raise Exception("Bad status code for endpoint {}: {}".format(endpoint, r.status_code)) return r.json() @@ -56,9 +56,9 @@ def root_get(path): template = env.get_template(path) body = template.render(objects=objects, options=request.args) except TemplateNotFound: - return f'Template "{path}" not found\n', 404 + return 'Template "{}" not found\n'.format(path), 404 except Exception as err: - return f'Templating of "{path}" failed to render. Most likely due to an error in the template. Error transcript:\n\n{err}\n----\n\n{traceback.format_exe()}\n', 400 + return 'Templating of "{}" failed to render. Most likely due to an error in the template. Error transcript:\n\n{}\n----\n\n{}\n'.format(path, err, traceback.format_exc()), 400 return body, 200 @@ -70,7 +70,7 @@ def root_post(path): template = env.from_string(content.decode("utf-8")) body = template.render(objects=objects, options=request.args) except Exception as err: - return f'Templating of "{path}" failed to render. Most likely due to an error in the template. Error transcript:\n\n{err}\n----\n\n{traceback.format_exe()}\n', 400 + return 'Templating of "{}" failed to render. Most likely due to an error in the template. Error transcript:\n\n{}\n----\n\n{}\n'.format(path, err, traceback.format_exc()), 400 return body, 200 -- cgit v1.2.3 From aa28da05ce5514e1336f105124cc63cd67c22511 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 22 Jan 2019 17:05:31 +0100 Subject: Removed `PUT` from `POST` method Although nice to have, this isn't compatible with the implementation of the web service. --- templating/templating.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templating/templating.py b/templating/templating.py index 03ad180..03b7dc9 100755 --- a/templating/templating.py +++ b/templating/templating.py @@ -62,7 +62,7 @@ def root_get(path): return body, 200 -@app.route("/", methods=["POST", "PUT"]) +@app.route("/", methods=["POST"]) def root_post(path): updateData() try: -- cgit v1.2.3 From 0d3b2b5a27512d0a4d3d238a322f8a416cbee018 Mon Sep 17 00:00:00 2001 From: Kristian Lyngstol Date: Wed, 23 Jan 2019 14:59:32 +0100 Subject: Add document for git-split Bah, forgot to use English. Use google translate if you so require. --- doc/gondul-git-split.rst | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 doc/gondul-git-split.rst diff --git a/doc/gondul-git-split.rst b/doc/gondul-git-split.rst new file mode 100644 index 0000000..4911626 --- /dev/null +++ b/doc/gondul-git-split.rst @@ -0,0 +1,60 @@ +================ +Ny repo-struktur +================ + +Motivasjon +========== + +Vi ønsker å dele opp gondul-repoet i mindre blokker, hensikten bak splitten +er delt: + +- Klarere skille mellom ellers uavhengige komponenter +- Enklere utvikling for alle +- Deployment forenkles ved å ha det i et eget repo, og at hvert repo kan + levere en faktisk pakke som installeres om ønskelig. +- Lettere å fryse enkelt-komponenter i forkant av arrangement. + + + +Nye repo +======== + +- Templating +- Front - inkluderer web/{js,img,fonts,css} og web/index.html +- lib - inkluderer include/ - Målet er nok å endre denne, da det ikke + egentlig er voldsomt overlapp mellom API og collectors, men inntill + videre er det eget repo. +- api - inkluderer web/api +- collectors - inkluderer collectors/ +- gondul/ - inkluderer ansible, dokumentasjon, default config. + + +På sikt er målet at API er det eneste som snakker med postgres, men inntill +videre vil fortsatt collectors snakke direkte. Collectors kan i prinsippet +deles yttligere opp om det ønskes, men det blir mye små-repoer. + +Navn: + +gondul-templating +gondul-frontend +gondul-api +gondul-collectors +gondul + +Repoet som da heter "gondul" blir "master-repo" og et slags +integrasjonsrepo. Det kan potensielt bli delt mer på sikt for å skille +ansible-saker fra dokumentasjon mm. Dette venter vi med for å unngå +usedvanlig mye fragmentering. + +Installasjon +============ + +Alt installeres default i /opt/$gondul-repo - Det vil være opp til +master-repoet å binde ting sammen. Det vil typisk bety at apache/nginx +settes opp for å levere statisk innhold for front og i dag levere CGI for +API, templating settes opp på egen port - Varnish vil da sørge for faktisk +ruting. + +Enkelt-repoer vil det være naturlig at leverer sin egen installasjonsrutine. +På sikt: Debian-pakker. Her er det viktigere for noen komponenter enn andre. + -- cgit v1.2.3 From 48ea18352bbf9dbfabe6fbce09ff6e6c9fa6d8f2 Mon Sep 17 00:00:00 2001 From: Simen Linderud Date: Wed, 23 Jan 2019 16:18:05 +0100 Subject: Markdown edits --- doc/gondul-git-split.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/gondul-git-split.rst b/doc/gondul-git-split.rst index 4911626..bded55a 100644 --- a/doc/gondul-git-split.rst +++ b/doc/gondul-git-split.rst @@ -36,9 +36,13 @@ deles yttligere opp om det ønskes, men det blir mye små-repoer. Navn: gondul-templating + gondul-frontend + gondul-api + gondul-collectors + gondul Repoet som da heter "gondul" blir "master-repo" og et slags -- cgit v1.2.3 From 272cab8f75ac7aceb73d8e58b155e1c8b8499c9b Mon Sep 17 00:00:00 2001 From: Kristian Lyngstol Date: Thu, 24 Jan 2019 15:34:37 +0100 Subject: Add gondul-receiver.rst doc --- doc/gondul-receiver.rst | 183 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 doc/gondul-receiver.rst diff --git a/doc/gondul-receiver.rst b/doc/gondul-receiver.rst new file mode 100644 index 0000000..f83ce7c --- /dev/null +++ b/doc/gondul-receiver.rst @@ -0,0 +1,183 @@ + +================================ +API for receiving time-base data +================================ + +Background +========== + +Toda, Gondul has three different "collectors". The ping-collector, the +snmp-collector and the dhcp log-tailer. + +They all write data directly to the postgres backend. + +Over the years we've tried different methods of storing time series data for +actual graphs. To support this, we've stored some data in two sources. Most +recently, we've stored stuff in postgres and influxdb. + +In addition to actually storing this data in different locations, we some +times need to "massage" data, or change the database schema. A prime example +for SNMP is to actually establish a tree-structure for port-data by picking +up ifTable and ifXTable and building a "ports"-tree using ifIndex. An other +example is normalization of MAC addresses for example. + +We also need to do something with Virtual Chassis MIBs. + +While we've been able to do all this, the fact that these collectors all +write directly to the postgres database creates a strong cross-dependency +between the collectors, the database schema and the API. It has also created +a strong depenendency to time series database tools. + +This has made it difficult to safely experiment with enriching input-data +without introducing critical bugs in collectors or breaking the north-bound +API. + +This document outlines a way to reduce this problem. + +Concept +======= + +The concept is to create a generic "time-based" API for all time-oriented +data. The API will cover high-frequency producers like the ping collector, +but also low-frequency producers like the operations log, or the DHCP +tailer. + +While the API will be generic, it will provide just enough data to allow the +receiver to identify the type of data and apply enrichments to it and thus +treat it diffrently. By default, the data posted will just be written to +postgres, but through enrichment add-ons, we can also chose to split a +single SNMP poll into multiple entries (e.g.: individual entries for virtual +chassis nodes), or re-arrange the data to produce interface-mapping. + +The enrichment will also be able to do basically anything with the data, +including sending it to multiple other APIs - e.g. influx. + +While the first version does not deal with authentication, future versions +should. + +Core API +======== + +The core API accepts N or more metrics in a single post. + +The core of the API will accept 3 fields: + +- "source" - a text-string identifying the source of the data, e.g. "dhcp", + "ping", "snmp". This should be sent either as a json text field, or as + part of the url. E.g., allow posting to + https://user:pass@gondul/api/write/gtimes/dhcp . The benefit of linking + this with the URL is that it will simplify authentication in the future, + allowing "write-only" accounts. +- "metadata" - this is a generic JSON object that contain a number of fields + that will be indexed upon or used by enrichment. Example: { "server": + "dhcpserver1", "time": "2019-01-05T15:00:10Z" }. + "time": "2019-01-05T12:00:10Z", " +- "data" - an array of json-objects. Each object in the array must either + have a "time" field or the "metadata"-field must have a time field. + + +Examples +======== + +Example 1, dhcp:: + + { + src: "dhcp", + metadata: { + server: "dhcpserver1" + }, + data: [ + { + type: "assignment", + time: "2001-01-01T15:12:01Z", + ip: "2001:db8::1", + circuit: "vlan123:e3-1:ge-0/0/1", + msg: "blatti foo" + }, + { + type: "renew", + time: "2001-01-01T15:32:01Z", + ip: "2001:db8::1", + circuit: "vlan123:e3-1:ge-0/0/1", + msg: "blatti foo something" + } + ] + } + +Example 2, ping:: + + { + "src": "ping", + "metadata": { + "time": "2019-05-01T15:01:12Z" + }, + "data": [ + { "s": "e1-3", "l": 0.91211 }, + { "s": "e1-2", "l": 0.12211 }, + { "s": "e1-1", "l": 0.12311 }, + { "s": "e3-1", "l": 1.12111 }, + { "s": "e3-2", "l": null }, + { "s": "e3-3", "l": 0.91211 }, + { "s": "e3-4", "l": 0.91211 } + ] + } + +Example 3, oplog:: + + { + "src": "oplog", + "data": [ + { + "system": "floor", + "user": "kristian", + "message": "lol", + "time": "2019-04-19T15:00:10Z" + } + ] + } + +Note that "metadata" is optional. + +Implementation plan +=================== + +The plan would be to start small. The first candidate is the dhcp log +tailer, which needs to support IPv6 and thus needs a change. + +The first implementation would be a "hard-coded" perl API since that is what +we already have. There is no current plan to migrate other producers to the +new API at this time. + +Since particularly the ping collector can produce quite a lot of data, some +care might be needed to support it. This will most likely require a +different apporach than the old CGI-based perl way of doing things. + +To allow a flexible enrichment-scheme, it might be necessarry to implement a +separate service in a more modern language. There are currently three worthy +alternatives: + +Node.js has the benefit of using JavaScript which is already heavily used in +Gondul, and is fairly fault-tolerant. There are also already plans to +utilize node.js to do server-side parsing of health data. However, I'm +unsure if it offers the speed or integration we need. + +Python is an other alternative, which is also already used. It is slightly +more mature than Node.js, but also doesn't really offer much else. + +The third alternative is Go, which will certainly provide us with the speed +we need, but might not allow the development pace we require during an +event. + +No conclusion is offered and at any rate, no plans to actually implement +such a service exist nor should one be planned until we have more experience +from the DHCP-collector implementation. + +Storage +======= + +Storage is deliberately left OUT of the API definition, but for +implementation-purposes we should assume postgres as the primary target with +influx as a sencodary target. Details of how this is done is intentionally +left out of this document as this should not be relevant to any user of the +API. + -- cgit v1.2.3 From af9857d2e9c8c0506afe7a3afc1125fedae9c84a Mon Sep 17 00:00:00 2001 From: Kristian Lyngstol Date: Thu, 24 Jan 2019 15:34:53 +0100 Subject: Minor tweaks on gondul-git-split --- doc/gondul-git-split.rst | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/gondul-git-split.rst b/doc/gondul-git-split.rst index bded55a..e4fd155 100644 --- a/doc/gondul-git-split.rst +++ b/doc/gondul-git-split.rst @@ -28,22 +28,17 @@ Nye repo - collectors - inkluderer collectors/ - gondul/ - inkluderer ansible, dokumentasjon, default config. - På sikt er målet at API er det eneste som snakker med postgres, men inntill videre vil fortsatt collectors snakke direkte. Collectors kan i prinsippet deles yttligere opp om det ønskes, men det blir mye små-repoer. Navn: -gondul-templating - -gondul-frontend - -gondul-api - -gondul-collectors - -gondul +- gondul-templating +- gondul-frontend +- gondul-api +- gondul-collectors +- gondul Repoet som da heter "gondul" blir "master-repo" og et slags integrasjonsrepo. Det kan potensielt bli delt mer på sikt for å skille @@ -59,6 +54,9 @@ settes opp for å levere statisk innhold for front og i dag levere CGI for API, templating settes opp på egen port - Varnish vil da sørge for faktisk ruting. -Enkelt-repoer vil det være naturlig at leverer sin egen installasjonsrutine. -På sikt: Debian-pakker. Her er det viktigere for noen komponenter enn andre. +Alt av "deployment" legges i "gondul"-repoet, men hver enkelt repo kan også +ønske å levere rutiner for isolert installasjon av typen som hører hjemme i +for eksempel en python-pakke eller debian-pakke. + + -- cgit v1.2.3 From 2b165d6d035f368bbdc0372e41cf71731c23f5bc Mon Sep 17 00:00:00 2001 From: Kristian Lyngstol Date: Thu, 24 Jan 2019 15:37:19 +0100 Subject: docs/gondul-receiver.rst: Minor style fixes --- doc/gondul-receiver.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/doc/gondul-receiver.rst b/doc/gondul-receiver.rst index f83ce7c..6b2ea5f 100644 --- a/doc/gondul-receiver.rst +++ b/doc/gondul-receiver.rst @@ -62,17 +62,16 @@ The core API accepts N or more metrics in a single post. The core of the API will accept 3 fields: -- "source" - a text-string identifying the source of the data, e.g. "dhcp", +- `source` - a text-string identifying the source of the data, e.g. "dhcp", "ping", "snmp". This should be sent either as a json text field, or as part of the url. E.g., allow posting to - https://user:pass@gondul/api/write/gtimes/dhcp . The benefit of linking - this with the URL is that it will simplify authentication in the future, - allowing "write-only" accounts. -- "metadata" - this is a generic JSON object that contain a number of fields - that will be indexed upon or used by enrichment. Example: { "server": - "dhcpserver1", "time": "2019-01-05T15:00:10Z" }. - "time": "2019-01-05T12:00:10Z", " -- "data" - an array of json-objects. Each object in the array must either + ``https://user:pass@gondul/api/write/gtimes/dhcp`` . The benefit of + linking this with the URL is that it will simplify authentication in the + future, allowing "write-only" accounts. +- `metadata` - this is a generic JSON object that contain a number of fields + that will be indexed upon or used by enrichment. Example: ``{ "server": + "dhcpserver1", "time": "2019-01-05T15:00:10Z" }``. +- `data` - an array of json-objects. Each object in the array must either have a "time" field or the "metadata"-field must have a time field. -- cgit v1.2.3 From 4306bc4f9c5ff40a5d56f700a2d753345188605f Mon Sep 17 00:00:00 2001 From: Kristian Lyngstol Date: Thu, 24 Jan 2019 15:38:41 +0100 Subject: gondul-receiver: Emphasise the limitations of the first implementation --- doc/gondul-receiver.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/gondul-receiver.rst b/doc/gondul-receiver.rst index 6b2ea5f..e663e19 100644 --- a/doc/gondul-receiver.rst +++ b/doc/gondul-receiver.rst @@ -147,6 +147,9 @@ The first implementation would be a "hard-coded" perl API since that is what we already have. There is no current plan to migrate other producers to the new API at this time. +The first implementation would not offer much in the way of generic storage +for other users than the dhcp collector. + Since particularly the ping collector can produce quite a lot of data, some care might be needed to support it. This will most likely require a different apporach than the old CGI-based perl way of doing things. -- cgit v1.2.3