]> git.immae.eu Git - perso/Immae/Config/Nix.git/commitdiff
Add status page for monitoring host
authorIsmaël Bouya <ismael.bouya@normalesup.org>
Sun, 5 Jan 2020 16:09:33 +0000 (17:09 +0100)
committerIsmaël Bouya <ismael.bouya@normalesup.org>
Sun, 5 Jan 2020 16:09:33 +0000 (17:09 +0100)
modules/private/certificates.nix
modules/private/default.nix
modules/private/monitoring/status.nix [new file with mode: 0644]
modules/private/monitoring/status/app.py [new file with mode: 0755]

index 337a7fcf4e5e05361efd093f8b3388e307c5df65..9e60a093d1170fcbafa68709bdbccd83c175551a 100644 (file)
@@ -1,4 +1,4 @@
-{ lib, pkgs, config,  ... }:
+{ lib, pkgs, config, name, ... }:
 {
   options.myServices.certificates = {
     enable = lib.mkEnableOption "enable certificates";
@@ -6,9 +6,12 @@
       default = {
         webroot = "${config.security.acme.directory}/acme-challenge";
         email = "ismael@bouya.org";
-        postRun = ''
-          systemctl reload httpdTools.service httpdInte.service httpdProd.service
-        '';
+        postRun = builtins.concatStringsSep "\n" [
+          (lib.optionalString config.services.httpd.Prod.enable "systemctl reload httpdProd.service")
+          (lib.optionalString config.services.httpd.Tools.enable "systemctl reload httpdTools.service")
+          (lib.optionalString config.services.httpd.Inte.enable "systemctl reload httpdInte.service")
+          (lib.optionalString config.services.nginx.enable "systemctl reload nginx.service")
+        ];
         plugins = [ "cert.pem" "chain.pem" "fullchain.pem" "full.pem" "key.pem" "account_key.json" ];
       };
       description = "Default configuration for certificates";
     services.duplyBackup.profiles.system.excludeFile = ''
       + ${config.security.acme.directory}
       '';
+    services.nginx = {
+      recommendedTlsSettings = true;
+      virtualHosts = { "${config.hostEnv.FQDN}" = { useACMEHost = name; forceSSL = true; }; };
+    };
     services.websites.certs = config.myServices.certificates.certConfig;
     myServices.databasesCerts = config.myServices.certificates.certConfig;
     myServices.ircCerts = config.myServices.certificates.certConfig;
@@ -26,8 +33,8 @@
     security.acme.preliminarySelfsigned = true;
 
     security.acme.certs = {
-      "eldiron" = config.myServices.certificates.certConfig // {
-        domain = "eldiron.immae.eu";
+      "${name}" = config.myServices.certificates.certConfig // {
+        domain = config.hostEnv.FQDN;
       };
     };
 
         '')
       ; })
     ) config.security.acme.certs // {
-      httpdProd.after = [ "acme-selfsigned-certificates.target" ];
-      httpdProd.wants = [ "acme-selfsigned-certificates.target" ];
-      httpdTools.after = [ "acme-selfsigned-certificates.target" ];
-      httpdTools.wants = [ "acme-selfsigned-certificates.target" ];
-      httpdInte.after = [ "acme-selfsigned-certificates.target" ];
-      httpdInte.wants = [ "acme-selfsigned-certificates.target" ];
+      httpdProd = lib.mkIf config.services.httpd.Prod.enable
+        { after = [ "acme-selfsigned-certificates.target" ]; wants = [ "acme-selfsigned-certificates.target" ]; };
+      httpdTools = lib.mkIf config.services.httpd.Tools.enable
+        { after = [ "acme-selfsigned-certificates.target" ]; wants = [ "acme-selfsigned-certificates.target" ]; };
+      httpdInte = lib.mkIf config.services.httpd.Inte.enable
+        { after = [ "acme-selfsigned-certificates.target" ]; wants = [ "acme-selfsigned-certificates.target" ]; };
     };
   };
 }
index 30b9df637830209533dbedc8b20aed55fa6dbfb6..47abec84952d753a1b1b20a21ed8f8e61e886d63 100644 (file)
@@ -71,6 +71,7 @@ set = {
   ejabberd = ./ejabberd;
   ssh = ./ssh;
   monitoring = ./monitoring;
+  status = ./monitoring/status.nix;
 
   environment = ./environment.nix;
   system = ./system.nix;
diff --git a/modules/private/monitoring/status.nix b/modules/private/monitoring/status.nix
new file mode 100644 (file)
index 0000000..ed4d681
--- /dev/null
@@ -0,0 +1,61 @@
+{ config, pkgs, lib, name, ... }:
+{
+  options = {
+    myServices.status = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to enable status app.
+        '';
+      };
+    };
+  };
+  config = lib.mkIf config.myServices.status.enable {
+    secrets.keys = [
+      {
+        dest = "naemon-status/environment";
+        user = "naemon";
+        group = "naemon";
+        permission = "0400";
+        text = ''
+          TOKENS=${builtins.concatStringsSep " " config.myEnv.monitoring.nrdp_tokens}
+          '';
+      }
+    ];
+    services.nginx = {
+      enable = true;
+      recommendedOptimisation = true;
+      recommendedGzipSettings = true;
+      recommendedProxySettings = true;
+      virtualHosts."status.immae.eu" = {
+        useACMEHost = name;
+        forceSSL = true;
+        locations."/".proxyPass = "http://unix:/run/naemon-status/socket.sock:/";
+      };
+    };
+    security.acme.certs."${name}".extraDomains."status.immae.eu" = null;
+
+    myServices.certificates.enable = true;
+    networking.firewall.allowedTCPPorts = [ 80 443 18000 ];
+    systemd.services.naemon-status = {
+      description = "Naemon status";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        EnvironmentFile = config.secrets.fullPaths."naemon-status/environment";
+        Type = "simple";
+        WorkingDirectory = "${./status}";
+        ExecStart = let
+          python = pkgs.python3.withPackages (p: [ p.gunicorn p.flask p.flask_login ]);
+        in
+          "${python}/bin/gunicorn -w4 --bind unix:/run/naemon-status/socket.sock app:app";
+        User = "naemon";
+        RuntimeDirectory = "naemon-status";
+        StandardOutput = "journal";
+        StandardError = "inherit";
+      };
+    };
+  };
+}
diff --git a/modules/private/monitoring/status/app.py b/modules/private/monitoring/status/app.py
new file mode 100755 (executable)
index 0000000..b1d419c
--- /dev/null
@@ -0,0 +1,409 @@
+from flask import Flask, request, render_template_string, jsonify, make_response
+from flask_login import LoginManager, UserMixin, login_required
+import socket
+import json
+import time
+import os
+
+login_manager = LoginManager()
+app = Flask(__name__)
+login_manager.init_app(app)
+
+STATUS = [
+        "ok",
+        "warning",
+        "error",
+        "unknown"
+        ]
+
+HOST_STATUS = [
+        "up",
+        "down",
+        "unreachable",
+        ]
+
+#### Push
+AUTHORIZED_KEYS = os.environ.get("TOKENS", "").split()
+COMMAND_FILE = "/var/run/naemon/naemon.cmd"
+
+ERROR_NO_REQUEST_HANDLER="NO REQUEST HANDLER"
+ERROR_NO_TOKEN_SUPPLIED="NO TOKEN"
+ERROR_BAD_TOKEN_SUPPLIED="BAD TOKEN"
+
+ERROR_BAD_COMMAND_FILE="BAD COMMAND FILE"
+ERROR_COMMAND_FILE_OPEN_WRITE="COMMAND FILE UNWRITEABLE"
+ERROR_COMMAND_FILE_OPEN="CANNOT OPEN COMMAND FILE"
+ERROR_BAD_WRITE="WRITE ERROR"
+
+ERROR_BAD_DATA="BAD DATA"
+ERROR_BAD_JSON="BAD JSON"
+
+ERROR_NO_CORRECT_STATUS="NO STATUS WAS CORRECT"
+#### /Push
+
+def get_lq(request):
+    # https://mathias-kettner.de/checkmk_livestatus.html
+    socket_path="/var/run/naemon/live"
+    s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    s.connect(socket_path)
+    s.send(request.encode())
+    s.shutdown(socket.SHUT_WR)
+    chunks = []
+    while len(chunks) == 0 or len(chunks[-1]) > 0:
+        chunks.append(s.recv(4096))
+    s.close()
+    return b"".join(chunks).decode()
+
+class Host:
+    def __init__(self, name, alias, status, webname):
+        self.name = name
+        self.alias = alias
+        self.webname = webname or alias
+        self.status = status
+        self.services = []
+
+    @classmethod
+    def parse_hosts(cls, payload):
+        parsed = [cls.parse(p) for p in json.loads(payload)]
+        return {p.name: p for p in parsed}
+
+    @classmethod
+    def parse(cls, payload):
+        return cls(payload[0], payload[1], HOST_STATUS[payload[2]], payload[3].get("WEBSTATUS_NAME"))
+
+    def __repr__(self):
+        return "Host {}: {} ({})".format(self.name, self.alias, self.webname)
+
+    @classmethod
+    def query(cls):
+        answer = get_lq("""GET hosts
+Filter: groups >= webstatus-hosts
+Columns: name alias state custom_variables
+OutputFormat: json
+""")
+        return cls.parse_hosts(answer)
+
+    def fill_services(self, services):
+        self.services = [service for service in services if service.host == self.name]
+
+class ServiceGroup:
+    def __init__(self, name, alias):
+        self.name = name
+        self.alias = alias
+        self.services = []
+
+    @classmethod
+    def parse_groups(cls, payload):
+        parsed = [cls.parse(p) for p in json.loads(payload)]
+        return {p.name: p for p in parsed}
+
+    @classmethod
+    def parse(cls, payload):
+        return cls(payload[0], payload[1])
+
+    @classmethod
+    def query(cls):
+        answer = get_lq("""GET servicegroups
+Filter: name ~ ^webstatus-
+Columns: name alias custom_variables
+OutputFormat: json
+""")
+        return cls.parse_groups(answer)
+
+    def fill_services(self, services):
+        self.services = [service for service in services if any([group == self.name for group in service.groups])]
+
+    def __repr__(self):
+        return "ServiceGroup {}: {}".format(self.name, self.alias)
+
+class Service:
+    def __init__(self, name, host, groups, status, webname, url, description, infos):
+        self.name = name
+        self.host = host
+        self.groups = groups
+        self.status = status
+        self.webname = webname
+        self.url = url
+        self.description = description
+        self.infos = infos
+
+    @classmethod
+    def parse_services(cls, payload):
+        parsed = json.loads(payload)
+        return [cls.parse(p) for p in parsed if cls.valid(p[2])]
+
+    @staticmethod
+    def valid(groups):
+        return any([b.startswith("webstatus-") for b in groups])
+
+    @classmethod
+    def parse(cls, payload):
+        return cls(payload[0],
+                payload[1],
+                payload[2],
+                STATUS[payload[3]],
+                payload[4].get("WEBSTATUS_NAME"),
+                payload[4].get("WEBSTATUS_URL"),
+                payload[5],
+                payload[6])
+
+    @classmethod
+    def query(cls):
+        answer = get_lq("""GET services
+Columns: display_name host_name groups state custom_variables description plugin_output
+OutputFormat: json
+""")
+        return cls.parse_services(answer)
+
+    def __repr__(self):
+        return "Service {}: {}".format(self.name, self.webname)
+
+def get_infos():
+    hosts = Host.query()
+    servicegroups = ServiceGroup.query()
+    services = Service.query()
+
+    for host in hosts:
+        hosts[host].fill_services(services)
+    for group in servicegroups:
+        servicegroups[group].fill_services(services)
+    return (hosts, servicegroups, services)
+
+TEMPLATE='''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html lang="en" xmlns="http://www.w3.org/1999/xhtml">
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <title>Status</title>
+        <meta name="referrer" content="no-referrer" />
+        <style type="text/css">
+        ul {
+            list-style: none;
+            margin: 0px;
+        }
+        ul li:nth-child(2n) {
+            background-color: rgb(240, 240, 240);
+        }
+        li.resource, li.service {
+            margin: 1px 0px;
+        }
+        span.status {
+            display: inline-block;
+            width: 150px;
+            text-align: center;
+            margin-right: 5px;
+            font-variant: small-caps;
+            font-size: 1.2em;
+        }
+        .status_ok,.status_up {
+            background-color: rgba(0, 255, 0, 0.5);;
+        }
+        .status_warning {
+            background-color: rgba(255, 255, 0, 0.5);;
+        }
+        .status_error,.status_down {
+            background-color: rgba(255, 0, 0, 0.5);;
+        }
+        .status_unknown,.status_unreachable {
+            background-color: rgba(0, 0, 255, 0.5);;
+        }
+        .infos {
+            margin-left: 40px;
+            color: rgb(100, 100, 100);
+        }
+        div#services {
+            column-count: auto;
+            column-width: 36em;
+        }
+        div.servicegroup {
+            -webkit-column-break-inside: avoid;
+            break-inside: avoid;
+        }
+        h3.servicegroup_title, h3.host_title {
+            margin: 1px 0px;
+        }
+        span.service_host, span.infos {
+            float: right;
+            display: inline-block;
+            color: rgb(100, 100, 100);
+        }
+        </style>
+    </head>
+    <body>
+        <h2>Hosts</h2>
+        {%- for host in hosts.values() %}
+            <h3 class="host_title">
+                <span class="status status_{{ host.status }}">{{ host.status }}</span>
+                <span class="host">{{ host.webname }}</span>
+            </h3>
+            {%- for service in servicegroups["webstatus-resources"].services if service.host == host.name -%}
+                {%- if loop.first %}
+                <ul class="resources">
+                {% endif %}
+
+                <li class="resource">
+                    <span class="status status_{{ service.status }}">{{ service.status }}</span>
+                    <span class="description">{{ service.description }}</span>
+                    <span class="infos">{{ service.infos }}</span>
+                </li>
+
+                {%- if loop.last %}
+                </ul>
+                {% endif %}
+            {% endfor %}
+        {%- endfor %}
+
+        <h2>Services</h2>
+        <div id="services">
+        {%- for group in servicegroups.values() if group.services and group.name != "webstatus-resources" %}
+            <div class="servicegroup">
+            <h3 class="servicegroup_title">{{ group.alias }}</h3>
+            {%- for service in group.services -%}
+                {%- if loop.first %}
+                <ul class="services">
+                {% endif %}
+
+                <li class="service" title="{{ service.infos }}">
+                    <span class="status status_{{ service.status }}">{{ service.status }}</span>
+                    <span class="description">
+                        {% if service.url and service.url.startswith("https://") %}
+                        <a href="{{ service.url }}">{{ service.webname or service.description }}</a>
+                        {% else %}
+                        {{ service.webname or service.description }}
+                        {% endif %}
+                    </span>
+                    <span class="service_host">{{ hosts[service.host].webname }}</span>
+                </li>
+
+                {%- if loop.last %}
+                </ul>
+                {% endif %}
+            {%- endfor -%}
+            </div>
+        {%- endfor %}
+        </div>
+    </body>
+</html>
+'''
+
+@login_manager.request_loader
+def load_user_from_request(request):
+    api_key = request.headers.get('Token')
+    if api_key in AUTHORIZED_KEYS:
+        return UserMixin()
+    content = request.get_json(force=True, silent=True)
+    if content is not None and content.get("token") in AUTHORIZED_KEYS:
+        return UserMixin()
+
+@app.route("/live", methods=["POST"])
+@login_required
+def live():
+    query = request.get_data()
+    result = get_lq(query.decode() + "\n")
+    resp = make_response(result)
+    resp.content_type = "text/plain"
+    return resp
+
+@app.route("/", methods=["GET"])
+def get():
+    (hosts, servicegroups, services) = get_infos()
+    resp = make_response(render_template_string(TEMPLATE, hosts=hosts, servicegroups=servicegroups))
+    resp.content_type = "text/html"
+    return resp
+
+@app.route("/", methods=["POST"])
+@login_required
+def push():
+    content = request.get_json(force=True, silent=True)
+    if content is None:
+        return ERROR_BAD_JSON
+    if content.get("cmd") != "submitcheck":
+        return render_error(ERROR_NO_REQUEST_HANDLER)
+    if "checkresult" not in content or not isinstance(content["checkresult"], list):
+        return render_error(ERROR_BAD_DATA)
+
+    checks = 0
+    errors = 0
+    for check in map(lambda x: CheckResult.from_json(x), content["checkresult"]):
+        if check is None:
+            errors += 1
+            continue
+        try:
+            write_check_output(check)
+        except Exception as e:
+            return render_error(str(e))
+        checks += 1
+    return render_response(checks, errors)
+
+def write_check_output(check):
+    if check.type== "service":
+        command = "[{time}] PROCESS_SERVICE_CHECK_RESULT;{hostname};{servicename};{state};{output}";
+    else:
+        command = "[{time}] PROCESS_HOST_CHECK_RESULT;{hostname};{state};{output}";
+    formatted = command.format(
+            time=int(time.time()),
+            hostname=check.hostname,
+            state=check.state,
+            output=check.output,
+            servicename=check.servicename,
+        )
+
+    if not os.path.exists(COMMAND_FILE):
+        raise Exception(ERROR_BAD_COMMAND_FILE)
+    if not os.access(COMMAND_FILE, os.W_OK):
+        raise Exception(ERROR_COMMAND_FILE_OPEN_WRITE)
+    if not os.access(COMMAND_FILE, os.W_OK):
+        raise Exception(ERROR_COMMAND_FILE_OPEN_WRITE)
+    try:
+        with open(COMMAND_FILE, "w") as c:
+            c.write(formatted + "\n")
+    except Exception as e:
+        raise Exception(ERROR_BAD_WRITE)
+
+def render_error(error):
+    return jsonify({
+            "status": "error",
+            "message": error,
+            })
+
+def render_response(checks, errors):
+    if checks > 0:
+        return jsonify({
+            "status": "ok",
+            "result": {
+                "checks": checks,
+                "errors": errors,
+                }
+            })
+    else:
+        return jsonify({
+            "status": "error",
+            "message": ERROR_NO_CORRECT_STATUS,
+            })
+
+class CheckResult:
+    def __init__(self, hostname, state, output, servicename, checktype):
+        self.hostname = hostname
+        self.state = state
+        self.output = output
+        self.servicename = servicename
+        self.type = checktype
+
+    @classmethod
+    def from_json(klass, j):
+        if not isinstance(j, dict):
+            return None
+        for key in ["hostname", "state", "output"]:
+            if key not in j or not isinstance(j[key], str):
+                return None
+        for key in ["servicename", "type"]:
+            if key in j and not isinstance(j[key], str):
+                return None
+        return klass(
+                j["hostname"],
+                j["state"],
+                j["output"],
+                j.get("servicename", ""),
+                j.get("type", "host"))
+