Add monitoring for dilion and quatresaisons

8 files changed, 77 insertions, 10 deletions

diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index 344e369..a298f92 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -16,13 +16,21 @@ let
     else if config.myServices.databases.enable
     then config.myServices.databases.postgresql.package
     else pkgs.postgresql;
+  zfsPlugin = pkgs.fetchurl {
+    url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh";
+    sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf";
+  };
   myplugins = pkgs.runCommand "buildplugins" {
     buildInputs = [ pkgs.makeWrapper pkgs.perl ];
   } ''
     mkdir $out
+    cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh
     cp ${./plugins}/* $out/
     patchShebangs $out
     wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
+    wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [
+      pkgs.which pkgs.zfs pkgs.gawk
+    ]}
     wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
       pkgs.curl pkgs.jq
     ]}
@@ -79,6 +87,11 @@ let
     '';
   toObjects = pkgs.callPackage ./to_objects.nix {};
   commonConfig = {
+    dilion = {
+      processWarn = "250"; processAlert = "400";
+      loadWarn = "8.0"; loadAlert = "10.0";
+      interface = "eth0";
+    };
     eldiron = {
       processWarn = "250"; processAlert = "400";
       loadWarn = "8.0"; loadAlert = "10.0";
@@ -95,11 +108,16 @@ let
       load15Warn = "1.0"; load15Alert = "2.0";
       interface = "ens3";
     };
+    quatresaisons = {
+      processWarn = "250"; processAlert = "400";
+      loadWarn = "8.0"; loadAlert = "10.0";
+      interface = "eth0";
+    };
   };
   externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
     (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
   masterPassiveObjects = let
-    passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron"]) nodes;
+    passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
     toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
     passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
       (_: n: toPassiveServices n.config.myServices.monitoring.services)
diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix
index 82043eb..7f553a0 100644
--- a/modules/private/monitoring/objects_common.nix
+++ b/modules/private/monitoring/objects_common.nix
@@ -143,6 +143,7 @@ in
     check_smtp = "$USER1$/check_smtp -H $HOSTADDRESS$ -p 25 -S -D 21,15";
     check_tcp = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -e \"$ARG2$\" -Mcrit";
     check_tcp_ssl = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -S -D 21,15";
+    check_zfs = "$USER2$/check_zpool.sh -p ALL";
 
     check_host_alive = "$USER1$/check_ping -H $HOSTADDRESS$ -w 3000.0,80% -c 5000.0,100% -p 5";
     check_last_file_date = "${sudo} -u \"$ARG3$\" $USER2$/check_last_file_date \"$ARG1$\" \"$ARG2$\"";
diff --git a/modules/private/monitoring/objects_dilion.nix b/modules/private/monitoring/objects_dilion.nix
new file mode 100644
index 0000000..ea4ec37
--- /dev/null
+++ b/modules/private/monitoring/objects_dilion.nix
@@ -0,0 +1,22 @@
+{ lib, hostFQDN, emailCheck, ... }:
+let
+  defaultPassiveInfo = {
+    filter = lib.attrsets.filterAttrs
+      (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_");
+    use = "external-passive-service";
+    freshness_threshold = "450";
+    retry_interval = "1";
+    servicegroups = "webstatus-resources";
+    host_name = hostFQDN;
+  };
+in
+{
+  service = [
+    {
+      passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-resources"; };
+      service_description = "No ZFS pool is degraded";
+      use = "local-service";
+      check_command = ["check_zfs"];
+    }
+  ];
+}
diff --git a/modules/private/monitoring/objects_eldiron.nix b/modules/private/monitoring/objects_eldiron.nix
index 92f997f..2c15dd6 100644
--- a/modules/private/monitoring/objects_eldiron.nix
+++ b/modules/private/monitoring/objects_eldiron.nix
@@ -19,6 +19,12 @@ in
       check_command = ["check_postgresql_replication" "backup-2" "/run/postgresql" "5432"];
     }
     {
+      passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-resources"; };
+      service_description = "No ZFS pool is degraded";
+      use = "local-service";
+      check_command = ["check_zfs"];
+    }
+    {
       passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-email"; };
       service_description = "mailq is empty";
       use = "local-service";
diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix
index 94b676b..4b784ed 100644
--- a/modules/private/monitoring/objects_monitoring-1.nix
+++ b/modules/private/monitoring/objects_monitoring-1.nix
@@ -113,17 +113,17 @@ in
     }
 
     # Backup services
-    {
-      service_description = "eriomem backup is up and not full";
-      host_name = "eldiron.immae.eu";
-      use = "external-service";
-      check_command = "check_backup_eriomem";
+    # {
+    #   service_description = "eriomem backup is up and not full";
+    #   host_name = "eldiron.immae.eu";
+    #   use = "external-service";
+    #   check_command = "check_backup_eriomem";
 
-      check_interval = "120";
-      notification_interval = "1440";
+    #   check_interval = "120";
+    #   notification_interval = "1440";
 
-      servicegroups = "webstatus-backup";
-    }
+    #   servicegroups = "webstatus-backup";
+    # }
     {
       service_description = "ovh backup is up and not full";
       host_name = "eldiron.immae.eu";
diff --git a/modules/private/monitoring/objects_quatresaisons.nix b/modules/private/monitoring/objects_quatresaisons.nix
new file mode 100644
index 0000000..3927ef5
--- /dev/null
+++ b/modules/private/monitoring/objects_quatresaisons.nix
@@ -0,0 +1,16 @@
+{ lib, hostFQDN, emailCheck, ... }:
+let
+  defaultPassiveInfo = {
+    filter = lib.attrsets.filterAttrs
+      (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_");
+    use = "external-passive-service";
+    freshness_threshold = "450";
+    retry_interval = "1";
+    servicegroups = "webstatus-resources";
+    host_name = hostFQDN;
+  };
+in
+{
+  service = [
+  ];
+}
diff --git a/modules/private/system/dilion.nix b/modules/private/system/dilion.nix
index 4e2a11f..b89b364 100644
--- a/modules/private/system/dilion.nix
+++ b/modules/private/system/dilion.nix
@@ -24,6 +24,8 @@
       "/boot/pass.key" = "/boot/pass.key";
     };
     kernel.sysctl."vm.nr_hugepages" = 256; # for xmr-stak
+    # available in nixos-20.09
+    #zfs.requestEncryptionCredentials = [ "zpool/root" ];
   };
   nix.maxJobs = 8;
   powerManagement.cpuFreqGovernor = "powersave";
@@ -147,6 +149,7 @@
   nix.binaryCaches = [ "https://hydra.iohk.io" "https://cache.nixos.org" ];
   nix.binaryCachePublicKeys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" ];
 
+  myServices.monitoring.enable = true;
   myServices.certificates.enable = true;
   security.acme.certs."${name}" = {
     user = config.services.nginx.user;
diff --git a/modules/private/system/quatresaisons.nix b/modules/private/system/quatresaisons.nix
index 353323f..e905460 100644
--- a/modules/private/system/quatresaisons.nix
+++ b/modules/private/system/quatresaisons.nix
@@ -274,6 +274,7 @@ in
     }
   ];
 
+  myServices.monitoring.enable = true;
   myServices.certificates.enable = true;
   users.mutableUsers = true;
   system.stateVersion = "21.03";