Add monitoring for backup-2

7 files changed, 193 insertions, 50 deletions

diff --git a/modules/private/monitoring/conf/specific_backup-2.cfg b/modules/private/monitoring/conf/specific_backup-2.cfg
new file mode 100644
index 0000000..ff91322
--- /dev/null
+++ b/modules/private/monitoring/conf/specific_backup-2.cfg
@@ -0,0 +1,36 @@
+# vim: filetype=nagios
+
+define service {
+  service_description    Size on /backup2 partition
+  check_command          check_local_disk!10%!5%!/backup2
+  use                    local-service
+}
+
+define command {
+  command_line           /run/wrappers/bin/sudo -u "$ARG3$" $USER2$/check_last_file_date "$ARG1$" "$ARG2$"
+  command_name           check_last_file_date
+}
+
+define service {
+  service_description    Last backup in /backup2/phare is not too old
+  check_command          check_last_file_date!/backup2/phare!14!backup
+  use                    local-service
+}
+
+define service {
+  service_description    Last backup in /backup2/immae_eu is not too old
+  check_command          check_last_file_date!/backup2/immae_eu!14!backup
+  use                    local-service
+}
+
+define service {
+  service_description    Last backup in /backup2/immae_fr is not too old
+  check_command          check_last_file_date!/backup2/immae_fr!14!backup
+  use                    local-service
+}
+
+define service {
+  service_description    Last postgresql dump in /backup2/eldiron/postgresql_backup is not too old
+  check_command          check_last_file_date!/backup2/eldiron/postgresql_backup!7!postgres
+  use                    local-service
+}
diff --git a/modules/private/monitoring/conf/specific_eldiron.cfg b/modules/private/monitoring/conf/specific_eldiron.cfg
new file mode 100644
index 0000000..fd5a43d
--- /dev/null
+++ b/modules/private/monitoring/conf/specific_eldiron.cfg
@@ -0,0 +1,29 @@
+# vim: filetype=nagios
+#
+define command {
+  command_line  /run/wrappers/bin/sudo -u postgres $USER2$/check_postgres_replication "$ARG1$" "$ARG2$" "$ARG3$"
+  command_name  check_postgresql_replication
+}
+
+define service {
+  service_description   Postgresql replication for backup-1 is up to date
+  check_command         check_postgresql_replication!backup-1!/run/postgresql!5432
+  use                   local-service
+}
+
+define service {
+  service_description   Postgresql replication for backup-2 is up to date
+  check_command         check_postgresql_replication!backup-2!/run/postgresql!5432
+  use                   local-service
+}
+
+define service {
+  service_description  mailq is empty
+  use                  local-service
+  check_command        check_mailq
+}
+
+define command {
+  command_name        check_mailq
+  command_line        $USER1$/check_mailq -s -w 1 -c 2
+}
diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index c5acd40..6062aba 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -1,4 +1,4 @@
-{ config, myconfig, pkgs, lib, ... }:
+{ config, myconfig, pkgs, lib, name, hostFQDN, ... }:
 let
   myplugins = pkgs.runCommand "buildplugins" {
     buildInputs = [ pkgs.makeWrapper pkgs.perl ];
@@ -13,16 +13,57 @@ let
     wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
       pkgs.gnugrep pkgs.gawk pkgs.procps-ng
     ]}
+    wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
+      pkgs.postgresql
+    ]}
     '';
+  defaultObjects =
+    let specific_file = ./conf + "/specific_" + name + ".cfg";
+    in
+        builtins.readFile ./conf/local_services.cfg
+      + builtins.readFile ./conf/timeperiods.cfg
+      + builtins.readFile ./conf/services.cfg
+      + builtins.readFile ./conf/contacts.cfg
+      + builtins.readFile ./conf/hosts.cfg
+      + ''
+        define command {
+          command_line       ${myplugins}/send_nrdp.sh -u "$USER200$" -t "$USER201$" -H "$HOSTADDRESS$" -s "$SERVICEDESC$" -S "$SERVICESTATEID$" -o "$SERVICEOUTPUT$"
+          command_name       notify-master
+        }
+        define service {
+          service_description  No mdadm array is degraded
+          use                  local-service
+          check_command        check_command_output!${pkgs.mdadm}/bin/mdadm --monitor --scan -1!^$!-s 0 -r root
+        }
+
+        define service {
+          name                local-service
+          use                 generic-service
+          host_name           ${hostFQDN}
+          check_interval      5
+          max_check_attempts  4
+          register            0
+          retry_interval      1
+        }
+        define host {
+          host_name           ${hostFQDN}
+          alias               ${hostFQDN}
+          address             ${hostFQDN}
+          use                 linux-server
+        }
+        ''
+      + lib.strings.optionalString (builtins.pathExists specific_file) (builtins.readFile specific_file);
 in
 {
   options = {
-    myServices.monitoring.enable = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      description = ''
-        Whether to enable monitoring.
-      '';
+    myServices.monitoring = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to enable monitoring.
+        '';
+      };
     };
   };
 
@@ -39,6 +80,21 @@ in
         users = [ "naemon" ];
         runAs = "root";
       }
+      {
+        commands = [
+          { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
+          { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
+        ];
+        users = [ "naemon" ];
+        runAs = "postgres";
+      }
+      {
+        commands = [
+          { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
+        ];
+        users = [ "naemon" ];
+        runAs = "backup";
+      }
     ];
     environment.etc."mdadm.conf" = {
       enable = true;
@@ -66,49 +122,7 @@ in
         $USER200$=${myconfig.env.monitoring.status_url}
         $USER201$=${myconfig.env.monitoring.status_token}
       '';
-      objectDefs = builtins.readFile ./conf/local_services.cfg
-        + builtins.readFile ./conf/timeperiods.cfg
-        + builtins.readFile ./conf/services.cfg
-        + builtins.readFile ./conf/contacts.cfg
-        + builtins.readFile ./conf/hosts.cfg
-        + ''
-          define command {
-            command_line       ${myplugins}/send_nrdp.sh -u "$USER200$" -t "$USER201$" -H "$HOSTADDRESS$" -s "$SERVICEDESC$" -S "$SERVICESTATEID$" -o "$SERVICEOUTPUT$"
-            command_name       notify-master
-          }
-          define service {
-            service_description  No mdadm array is degraded
-            use                  local-service
-            check_command        check_command_output!${pkgs.mdadm}/bin/mdadm --monitor --scan -1!^$!-s 0 -r root
-          }
-
-          define service {
-            service_description  mailq is empty
-            use                  local-service
-            check_command        check_mailq
-          }
-
-          define command {
-            command_name        check_mailq
-            command_line        $USER1$/check_mailq -s -w 1 -c 2
-          }
-
-          define service {
-            name                local-service
-            use                 generic-service
-            host_name           eldiron.immae.eu
-            check_interval      5
-            max_check_attempts  4
-            register            0
-            retry_interval      1
-          }
-          define host {
-            host_name           eldiron.immae.eu
-            alias               eldiron.immae.eu
-            address             eldiron.immae.eu
-            use                 linux-server
-          }
-          '';
+      objectDefs = defaultObjects;
     };
   };
 }
diff --git a/modules/private/monitoring/plugins/check_last_file_date b/modules/private/monitoring/plugins/check_last_file_date
new file mode 100755
index 0000000..df45bbc
--- /dev/null
+++ b/modules/private/monitoring/plugins/check_last_file_date
@@ -0,0 +1,26 @@
+#!/bin/bash
+ 
+STATE_OK=0
+STATE_WARNING=1
+STATE_CRITICAL=2
+STATE_UNKNOWN=3
+  
+base_path=$1
+hours=$2
+
+last_date=$(find $base_path -mindepth 1 -maxdepth 1 -printf "%T@\n" 2>/dev/null | sort | tail -n 1)
+
+if [ -z "$last_date" ]; then
+  echo "UNKNOWN: Could not read folder"
+  exit $STATE_UNKNOWN
+else
+  LC_ALL=C last_date=$(printf "%.*f" 0 $last_date)
+  min_date=$(date -d "$hours hours ago" "+%s")
+  if [ "$min_date" -lt "$last_date" ]; then
+    echo "OK: Last file $(date -d @$last_date)"
+    exit $STATE_OK
+  else
+    echo "CRITICAL: Last file $(date -d @$last_date)"
+    exit $STATE_CRITICAL
+  fi
+fi
diff --git a/modules/private/monitoring/plugins/check_postgres_replication b/modules/private/monitoring/plugins/check_postgres_replication
new file mode 100755
index 0000000..009b4d5
--- /dev/null
+++ b/modules/private/monitoring/plugins/check_postgres_replication
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+STATE_OK=0
+STATE_WARNING=1
+STATE_CRITICAL=2
+STATE_UNKNOWN=3
+
+user=$1
+host=$2
+port=$3
+
+lag=$(psql -h $host -p $port -A -t -c "SELECT COALESCE(EXTRACT(EPOCH FROM replay_lag),0) FROM pg_stat_replication WHERE usename='$user'" 2>/dev/null)
+exit_code=$?
+
+if [[ $exit_code -ne 0 ]]; then
+  echo "UNKNOWN - Impossible to run psql command"
+  exit $STATE_UNKNOWN
+elif [[ -z "$lag" ]]; then
+  echo "UNKNOWN - No replication found for $user"
+  exit $STATE_UNKNOWN
+else
+  output="Replication lag for $user is ${lag}s"
+  LC_ALL=C lag=$(printf "%.*f" 0 $lag)
+
+  if [[ $lag -lt 5 ]]; then
+    echo "OK - $output"
+    exit $STATE_OK
+  elif [[ $lag -lt 10 ]]; then
+    echo "WARNING - $output"
+    exit $STATE_WARNING
+  else
+    echo "CRITICAL - $output"
+    exit $STATE_CRITICAL
+  fi
+fi
diff --git a/modules/private/system/backup-2.nix b/modules/private/system/backup-2.nix
index 1c5b7d8..3120a57 100644
--- a/modules/private/system/backup-2.nix
+++ b/modules/private/system/backup-2.nix
@@ -3,6 +3,7 @@
 {
   boot.kernelPackages = pkgs.linuxPackages_latest;
   _module.args.privateFiles = privateFiles;
+  _module.args.hostFQDN = "backup-2.v.immae.eu";
   imports = builtins.attrValues (import ../..);
 
   deployment = {
@@ -48,6 +49,7 @@
     ssh_key_private = myconfig.env.rsync_backup.ssh_key.private;
   };
 
+  myServices.monitoring.enable = true;
   myServices.databasesReplication = {
     postgresql = {
       enable = true;
diff --git a/modules/private/system/eldiron.nix b/modules/private/system/eldiron.nix
index 5384bc2..d79cf16 100644
--- a/modules/private/system/eldiron.nix
+++ b/modules/private/system/eldiron.nix
@@ -3,6 +3,7 @@
 {
   boot.kernelPackages = pkgs.linuxPackages_latest;
   _module.args.privateFiles = privateFiles;
+  _module.args.hostFQDN = "eldiron.immae.eu";
 
   networking = {
     firewall.enable = true;