From: Ismaël Bouya <ismael.bouya@normalesup.org>
Date: Tue, 29 Dec 2020 21:06:59 +0000 (+0100)
Subject: Add Eban monitoring
X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=2edbb2d889bd9d1787bc1745a75c1b6969d148ab;p=perso%2FImmae%2FConfig%2FNix.git

Add Eban monitoring
---

diff --git a/modules/private/environment.nix b/modules/private/environment.nix
index e79feec..490a405 100644
--- a/modules/private/environment.nix
+++ b/modules/private/environment.nix
@@ -558,6 +558,14 @@ in
               };
             };
           };
+          eban = mkOption {
+            description = "Eban credentials for webhook";
+            type = submodule {
+              options = {
+                password = mkOption { type = str; description = "Password"; };
+              };
+            };
+          };
           nrdp_tokens = mkOption { type = listOf str; description = "Tokens allowed to push status update"; };
           slack_url = mkOption { type = str; description = "Slack webhook url to push status update"; };
           slack_channel = mkOption { type = str; description = "Slack channel to push status update"; };
diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index a298f92..73e4275 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -78,6 +78,9 @@ let
     wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
       pkgs.curl pkgs.jq
     ]}
+    wrapProgram $out/notify_eban_url --prefix PATH : ${lib.makeBinPath [
+      pkgs.curl
+    ]}
     wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
       (pkgs.python3.withPackages (ps: [ps.ovh]))
     ]}
@@ -157,7 +160,7 @@ let
     };
   otherObjects = map
     (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
-    [ "ulminfo-fr" "phare" ];
+    [ "ulminfo-fr" "phare" "eban" ];
   masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
   commonObjects = pkgs.callPackage ./objects_common.nix ({
     master = cfg.master;
@@ -326,6 +329,7 @@ in
           config.myEnv.monitoring.ovh_sms.consumer_key
           config.myEnv.monitoring.ovh_sms.account
         ]}
+        $USER210$=${config.myEnv.monitoring.eban.password}
       '';
       objectDefs = toObjects commonObjects
         + toObjects hostObjects
diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix
index 7f553a0..df378b5 100644
--- a/modules/private/monitoring/objects_common.nix
+++ b/modules/private/monitoring/objects_common.nix
@@ -34,6 +34,7 @@ in
       use = "linux-server";
       hostgroups = "webstatus-hosts";
       _webstatus_name = hostName;
+      _webstatus_vhost = "status.immae.eu";
     };
   };
   service = [
@@ -155,6 +156,8 @@ in
     # $OVE is to force naemon to run via shell instead of execve which fails here
     notify-service-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE";
     notify-by-slack = "HOST=\"$HOSTALIAS$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_slack \"$ARG1$\" \"$ARG2$\"";
+    notify-host-eban-url = "STATUS_NAME=\"Server\" PASSWORD=\"$USER210$\" HOSTSTATE=\"$HOSTSTATE$\" $USER2$/notify_eban_url";
+    notify-service-eban-url = "STATUS_NAME=\"$_SERVICEWEBSTATUS_NAME$\" PASSWORD=\"$USER210$\" SERVICESTATE=\"$SERVICESTATE$\" $USER2$/notify_eban_url";
 
     notify-master = "$USER2$/send_nrdp.sh -u \"$USER200$\" -t \"$USER201$\" -H \"$HOSTADDRESS$\" -s \"$SERVICEDESC$\" -S \"$SERVICESTATEID$\" -o \"$SERVICEOUTPUT$ | $SERVICEPERFDATA$\"";
   };
diff --git a/modules/private/monitoring/objects_eban.nix b/modules/private/monitoring/objects_eban.nix
new file mode 100644
index 0000000..659b0ec
--- /dev/null
+++ b/modules/private/monitoring/objects_eban.nix
@@ -0,0 +1,60 @@
+{ ... }:
+let
+  serviceTemplate = rest: {
+    host_name = "eban.bzh";
+    use = "external-web-service";
+    contacts = "eban";
+    contact_groups = "null";
+    check_interval = "15";
+
+    servicegroups = "webstatus-resources";
+  } // rest;
+in
+{
+  contact = {
+    eban = {
+      use = "generic-contact";
+      host_notification_commands = "notify-host-eban-url";
+      service_notification_commands = "notify-service-eban-url";
+    };
+  };
+  host = {
+    "eban.bzh" = {
+      alias = "eban.bzh";
+      address = "eban.bzh";
+      use = "linux-server";
+      hostgroups = "webstatus-hosts";
+      contacts = "eban";
+      contact_groups = "null";
+      _webstatus_name = "Eban";
+      _webstatus_vhost = "status.eban.bzh";
+    };
+  };
+  service = [
+    (serviceTemplate {
+      service_description = "Eban website is up and running";
+      check_command = ["check_https" "eban.bzh" "/" "<title>"];
+      _webstatus_name = "Main Website";
+      _webstatus_url = "https://eban.bzh/";
+    })
+    (serviceTemplate {
+      service_description = "Eban blog is up and running";
+      check_command = ["check_https" "blog.eban.bzh" "/" "<title>"];
+      _webstatus_name = "Blog";
+      _webstatus_url = "https://blog.eban.bzh/";
+    })
+    (serviceTemplate {
+      service_description = "Eban gitea is up and running";
+      check_command = ["check_https" "git.eban.bzh" "/" "<title>"];
+      _webstatus_name = "Git";
+      _webstatus_url = "https://git.eban.bzh/";
+    })
+    (serviceTemplate {
+      service_description = "Eban Cloud is up and running";
+      check_command = ["check_https" "cloud.eban.bzh" "/" "<title>"];
+
+      _webstatus_name = "Cloud";
+      _webstatus_url = "https://cloud.eban.bzh/";
+    })
+  ];
+}
diff --git a/modules/private/monitoring/objects_phare.nix b/modules/private/monitoring/objects_phare.nix
index a00e5e8..a61b46e 100644
--- a/modules/private/monitoring/objects_phare.nix
+++ b/modules/private/monitoring/objects_phare.nix
@@ -7,6 +7,7 @@
       use = "linux-server";
       hostgroups = "webstatus-hosts";
       _webstatus_name = "phare";
+      _webstatus_vhost = "status.immae.eu";
     };
   };
   service = [
diff --git a/modules/private/monitoring/objects_tiboqorl-fr.nix b/modules/private/monitoring/objects_tiboqorl-fr.nix
index c3851b5..77403ba 100644
--- a/modules/private/monitoring/objects_tiboqorl-fr.nix
+++ b/modules/private/monitoring/objects_tiboqorl-fr.nix
@@ -22,6 +22,7 @@ in
       contact_groups = "tiboqorl";
       hostgroups = "webstatus-hosts";
       _webstatus_name = "tiboqorl";
+      _webstatus_vhost = "status.immae.eu";
     };
   };
   service = [
diff --git a/modules/private/monitoring/objects_ulminfo-fr.nix b/modules/private/monitoring/objects_ulminfo-fr.nix
index b970ecd..574e0e3 100644
--- a/modules/private/monitoring/objects_ulminfo-fr.nix
+++ b/modules/private/monitoring/objects_ulminfo-fr.nix
@@ -7,6 +7,7 @@
       use = "linux-server";
       hostgroups = "webstatus-hosts";
       _webstatus_name = "ulminfo";
+      _webstatus_vhost = "status.immae.eu";
     };
   };
   service = [
diff --git a/modules/private/monitoring/plugins/notify_eban_url b/modules/private/monitoring/plugins/notify_eban_url
new file mode 100755
index 0000000..431e320
--- /dev/null
+++ b/modules/private/monitoring/plugins/notify_eban_url
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+PASS=$(echo "$PASSWORD" | base64 -d)
+
+if [ "$SERVICESTATE" = "CRITICAL" -o "$SERVICESTATE" = "UNKNOWN" -o "$HOSTSTATE" = "DOWN" -o "$HOSTSTATE" = "UNREACHABLE" ]; then
+  action=downAlert
+elif [ "$SERVICESTATE" = "OK" -o "$HOSTSTATE" = "UP" ]; then
+  action=upAlert
+fi
+
+if [ -n "$action" ]; then
+  curl -X GET -G --data-urlencode "service=$STATUS_NAME" --data-urlencode "mdp=$PASS" https://infra.eban.bzh/$action
+fi
diff --git a/modules/private/monitoring/status.nix b/modules/private/monitoring/status.nix
index 4ca0327..73f4749 100644
--- a/modules/private/monitoring/status.nix
+++ b/modules/private/monitoring/status.nix
@@ -32,6 +32,12 @@
       upstreams."netdata".extraConfig = ''
         keepalive 64;
         '';
+      virtualHosts."status.eban.bzh" = {
+        acmeRoot = config.myServices.certificates.webroot;
+        useACMEHost = name;
+        forceSSL = true;
+        locations."/".proxyPass = "http://unix:/run/naemon-status/socket.sock:/";
+      };
       virtualHosts."status.immae.eu" = {
         acmeRoot = config.myServices.certificates.webroot;
         useACMEHost = name;
@@ -60,6 +66,7 @@
     };
     security.acme.certs."${name}" = {
       extraDomains."status.immae.eu" = null;
+      extraDomains."status.eban.bzh" = null;
       user = config.services.nginx.user;
       group = config.services.nginx.group;
     };
diff --git a/modules/private/monitoring/status/app.py b/modules/private/monitoring/status/app.py
index b1d419c..ff92891 100755
--- a/modules/private/monitoring/status/app.py
+++ b/modules/private/monitoring/status/app.py
@@ -55,33 +55,34 @@ def get_lq(request):
     return b"".join(chunks).decode()
 
 class Host:
-    def __init__(self, name, alias, status, webname):
+    def __init__(self, name, alias, status, webname, vhost):
         self.name = name
         self.alias = alias
         self.webname = webname or alias
+        self.vhost = vhost
         self.status = status
         self.services = []
 
     @classmethod
-    def parse_hosts(cls, payload):
-        parsed = [cls.parse(p) for p in json.loads(payload)]
+    def parse_hosts(cls, payload, vhost):
+        parsed = filter(lambda x: x.vhost == vhost, [cls.parse(p) for p in json.loads(payload)])
         return {p.name: p for p in parsed}
 
     @classmethod
     def parse(cls, payload):
-        return cls(payload[0], payload[1], HOST_STATUS[payload[2]], payload[3].get("WEBSTATUS_NAME"))
+        return cls(payload[0], payload[1], HOST_STATUS[payload[2]], payload[3].get("WEBSTATUS_NAME"), payload[3].get("WEBSTATUS_VHOST"))
 
     def __repr__(self):
         return "Host {}: {} ({})".format(self.name, self.alias, self.webname)
 
     @classmethod
-    def query(cls):
+    def query(cls, vhost):
         answer = get_lq("""GET hosts
 Filter: groups >= webstatus-hosts
 Columns: name alias state custom_variables
 OutputFormat: json
 """)
-        return cls.parse_hosts(answer)
+        return cls.parse_hosts(answer, vhost)
 
     def fill_services(self, services):
         self.services = [service for service in services if service.host == self.name]
@@ -110,8 +111,8 @@ OutputFormat: json
 """)
         return cls.parse_groups(answer)
 
-    def fill_services(self, services):
-        self.services = [service for service in services if any([group == self.name for group in service.groups])]
+    def fill_services(self, services, hosts):
+        self.services = [service for service in services if any([group == self.name for group in service.groups]) and service.host in hosts]
 
     def __repr__(self):
         return "ServiceGroup {}: {}".format(self.name, self.alias)
@@ -158,15 +159,15 @@ OutputFormat: json
     def __repr__(self):
         return "Service {}: {}".format(self.name, self.webname)
 
-def get_infos():
-    hosts = Host.query()
+def get_infos(vhost):
+    hosts = Host.query(vhost)
     servicegroups = ServiceGroup.query()
     services = Service.query()
 
     for host in hosts:
         hosts[host].fill_services(services)
     for group in servicegroups:
-        servicegroups[group].fill_services(services)
+        servicegroups[group].fill_services(services, hosts)
     return (hosts, servicegroups, services)
 
 TEMPLATE='''<?xml version="1.0" encoding="UTF-8"?>
@@ -254,12 +255,14 @@ TEMPLATE='''<?xml version="1.0" encoding="UTF-8"?>
             {% endfor %}
         {%- endfor %}
 
+        {%- for group in servicegroups.values() if group.services and group.name != "webstatus-resources" %}
+        {%- if loop.first %}
         <h2>Services</h2>
         <div id="services">
-        {%- for group in servicegroups.values() if group.services and group.name != "webstatus-resources" %}
+        {%- endif %}
             <div class="servicegroup">
             <h3 class="servicegroup_title">{{ group.alias }}</h3>
-            {%- for service in group.services -%}
+            {%- for service in group.services if service.host in hosts -%}
                 {%- if loop.first %}
                 <ul class="services">
                 {% endif %}
@@ -281,8 +284,10 @@ TEMPLATE='''<?xml version="1.0" encoding="UTF-8"?>
                 {% endif %}
             {%- endfor -%}
             </div>
-        {%- endfor %}
+        {%- if loop.last %}
         </div>
+        {% endif %}
+        {%- endfor %}
     </body>
 </html>
 '''
@@ -307,7 +312,7 @@ def live():
 
 @app.route("/", methods=["GET"])
 def get():
-    (hosts, servicegroups, services) = get_infos()
+    (hosts, servicegroups, services) = get_infos(request.host)
     resp = make_response(render_template_string(TEMPLATE, hosts=hosts, servicegroups=servicegroups))
     resp.content_type = "text/html"
     return resp
diff --git a/modules/private/system.nix b/modules/private/system.nix
index 71a49e3..0e72d99 100644
--- a/modules/private/system.nix
+++ b/modules/private/system.nix
@@ -26,7 +26,8 @@
     ];
 
     services.journald.extraConfig = ''
-      MaxLevelStore=warning
+      #Should be "warning" but disabled for now, it prevents anything from being stored
+      MaxLevelStore=info
       MaxRetentionSec=1year
       '';
 
diff --git a/nixops/secrets b/nixops/secrets
index 3d83a12..00f59f4 160000
--- a/nixops/secrets
+++ b/nixops/secrets
@@ -1 +1 @@
-Subproject commit 3d83a120a6247493e7cdac8d670ea85ebbfb711c
+Subproject commit 00f59f49098d79f0839541fd379e8933e3ddf9d7