Add e-mail checks monitoring

11 files changed, 240 insertions, 39 deletions

diff --git a/modules/private/environment.nix b/modules/private/environment.nix
index 81b5df5..7da2480 100644
--- a/modules/private/environment.nix
+++ b/modules/private/environment.nix
@@ -486,6 +486,19 @@ in
           slack_url = mkOption { type = str; description = "Slack webhook url to push status update"; };
           slack_channel = mkOption { type = str; description = "Slack channel to push status update"; };
           contacts = mkOption { type = attrsOf unspecified; description = "Contact dicts to fill naemon objects"; };
+          email_check = mkOption {
+            description = "Emails services to check";
+            type = attrsOf (submodule {
+              options = {
+                local = mkOption { type = bool; default = false; description = "Use local configuration"; };
+                port = mkOption { type = nullOr str; default = null; description = "Port to connect to ssh"; };
+                login = mkOption { type = nullOr str; default = null; description = "Login to connect to ssh"; };
+                targets = mkOption { type = listOf str; description = "Hosts to send E-mails to"; };
+                mail_address = mkOption { type = str; description = "E-mail recipient part to send e-mail to"; };
+                mail_domain = mkOption { type = str; description = "E-mail domain part to send e-mail to"; };
+              };
+            });
+          };
         };
       };
     };
diff --git a/modules/private/mail/postfix.nix b/modules/private/mail/postfix.nix
index 8fe06da..51f4de7 100644
--- a/modules/private/mail/postfix.nix
+++ b/modules/private/mail/postfix.nix
@@ -1,4 +1,4 @@
-{ lib, pkgs, config, nodes, name, ... }:
+{ lib, pkgs, config, nodes, ... }:
 {
   config = lib.mkIf config.myServices.mail.enable {
     services.duplyBackup.profiles.mail.excludeFile = ''
@@ -220,7 +220,13 @@
           '';
         scripts = lib.attrsets.mapAttrs (n: v:
           toScript n (pkgs.callPackage (builtins.fetchGit { url = v.src.url; ref = "master"; rev = v.src.rev; }) { scriptEnv = v.env; })
-        ) config.myEnv.mail.scripts;
+        ) config.myEnv.mail.scripts // {
+          testmail = pkgs.writeScript "testmail" ''
+            #! ${pkgs.stdenv.shell}
+            ${pkgs.coreutils}/bin/touch \
+              "/var/lib/naemon/checks/email/$(${pkgs.procmail}/bin/formail -x To: | ${pkgs.coreutils}/bin/tr -d ' <>')"
+            '';
+        };
       in builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList (n: v: ''${n}: "|${v}"'') scripts);
       mapFiles = let
         recipient_maps = let
@@ -246,8 +252,12 @@
           ) config.myEnv.mail.postfix.backup_domains
         );
         virtual_map = {
-          virtual = pkgs.writeText "postfix-virtual" (
+          virtual = let
+            cfg = config.myEnv.monitoring.email_check.eldiron;
+            address = "${cfg.mail_address}@${cfg.mail_domain}";
+          in pkgs.writeText "postfix-virtual" (
             builtins.concatStringsSep "\n" (
+              ["${address} testmail@localhost"] ++
               lib.attrsets.mapAttrsToList (
                 n: v: lib.optionalString v.external ''
                   script_${n}@mail.immae.eu ${n}@localhost, scripts@mail.immae.eu
@@ -425,5 +435,25 @@
         "smtp.immae.eu" = null;
       };
     };
+    system.activationScripts.testmail = {
+      deps = [ "users" ];
+      text = let
+        allCfg = config.myEnv.monitoring.email_check;
+        cfg = allCfg.eldiron;
+        reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem "eldiron" v.targets) allCfg);
+        to_email = cfg': host':
+          let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
+          in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
+        mails_to_receive = builtins.concatStringsSep " " (map (to_email cfg) reverseTargets);
+      in ''
+        install -m 0555 -o nobody -g nogroup -d /var/lib/naemon/checks/email
+        for f in ${mails_to_receive}; do
+          if [ ! -f /var/lib/naemon/checks/email/$f ]; then
+            install -m 0644 -o nobody -g nogroup /dev/null -T /var/lib/naemon/checks/email/$f
+            touch -m -d @0 /var/lib/naemon/checks/email/$f
+          fi
+        done
+        '';
+    };
   };
 }
diff --git a/modules/private/mail/relay.nix b/modules/private/mail/relay.nix
index e0aa387..6ac3df8 100644
--- a/modules/private/mail/relay.nix
+++ b/modules/private/mail/relay.nix
@@ -126,8 +126,12 @@
           ) config.myEnv.mail.postfix.backup_domains
         );
         virtual_map = {
-          virtual = pkgs.writeText "postfix-virtual" (
+          virtual = let
+            cfg = config.myEnv.monitoring.email_check.eldiron;
+            address = "${cfg.mail_address}@${cfg.mail_domain}";
+          in pkgs.writeText "postfix-virtual" (
             builtins.concatStringsSep "\n" (
+              ["${address} 1"] ++
               lib.attrsets.mapAttrsToList (
                 n: v: lib.optionalString v.external ''
                   script_${n}@mail.immae.eu 1
diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index e1357a7..a7013af 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -1,6 +1,16 @@
 { config, pkgs, lib, name, nodes, ... }:
 let
   cfg = config.myServices.monitoring;
+  send_mails = pkgs.runCommand "send_mails" {
+    buildInputs = [ pkgs.makeWrapper ];
+  } ''
+    mkdir -p $out/bin
+    cp ${./send_mails} $out/bin/send_mails
+    patchShebangs $out
+    wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
+      pkgs.mailutils
+    ]}
+    '';
   myplugins = pkgs.runCommand "buildplugins" {
     buildInputs = [ pkgs.makeWrapper pkgs.perl ];
   } ''
@@ -29,6 +39,11 @@ let
     wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
       pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
     ]}
+    wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
+      pkgs.openssh send_mails
+    ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
+      pkgs.perlPackages.TimeDate
+    ]}
     wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
       pkgs.lftp
     ]}
@@ -67,11 +82,6 @@ let
     };
   };
   masterPassiveObjects = let
-    otherPassiveObjects = map
-      (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") {}))
-      [ "caldance-1" "ulminfo-fr" "immae-eu" "phare" "tiboqorl-fr" ];
-    otherPassiveServices = lib.flatten (map (h: h.service or []) otherPassiveObjects);
-    otherPassiveHosts = (map (h: h.host)) otherPassiveObjects;
     passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron"]) nodes;
     toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
     passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
@@ -79,12 +89,37 @@ let
       passiveNodes
       );
   in {
-    service = passiveServices ++ otherPassiveServices;
+    service = passiveServices;
     host = lib.lists.foldr
       (a: b: a//b)
       {}
-      (otherPassiveHosts ++ lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes);
+      (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes);
   };
+  emailCheck = host: hostFQDN: let
+    allCfg = config.myEnv.monitoring.email_check;
+    cfg = allCfg."${host}";
+    reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
+    to_email = cfg': host':
+      let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
+      in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
+    mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
+    mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
+    command = if cfg.local
+    then
+      [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
+    else
+      [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
+  in
+    {
+      service_description = "${hostFQDN} email service is active";
+      use = "mail-service";
+      host_name = hostFQDN;
+      servicegroups = "webstatus-email";
+      check_command = command;
+    };
+  otherObjects = map
+    (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
+    [ "caldance-1" "ulminfo-fr" "immae-eu" "phare" "tiboqorl-fr" ];
   masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
   commonObjects = pkgs.callPackage ./objects_common.nix ({
     master = cfg.master;
@@ -99,7 +134,7 @@ let
       lib.attrsets.optionalAttrs
         (builtins.pathExists specific_file)
         (pkgs.callPackage specific_file {
-          inherit config;
+          inherit config emailCheck;
           hostFQDN = config.hostEnv.fqdn;
           hostName = name;
         });
@@ -228,7 +263,8 @@ in
       objectDefs = toObjects commonObjects
         + toObjects hostObjects
         + lib.optionalString cfg.master (toObjects masterObjects)
-        + lib.optionalString cfg.master (toObjects masterPassiveObjects);
+        + lib.optionalString cfg.master (toObjects masterPassiveObjects)
+        + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
     };
   };
 }
diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix
index 15eee97..a9d6da4 100644
--- a/modules/private/monitoring/objects_common.nix
+++ b/modules/private/monitoring/objects_common.nix
@@ -91,6 +91,8 @@ in
   ];
   command = {
     check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$";
+    check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$";
+    check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$";
     check_eriomem = "$USER2$/check_eriomem $USER208$";
     check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$";
     check_ftp_database = "$USER2$/check_ftp_database";
diff --git a/modules/private/monitoring/objects_eldiron.nix b/modules/private/monitoring/objects_eldiron.nix
index bee4645..92f997f 100644
--- a/modules/private/monitoring/objects_eldiron.nix
+++ b/modules/private/monitoring/objects_eldiron.nix
@@ -1,4 +1,4 @@
-{ lib, hostFQDN, ... }:
+{ lib, hostFQDN, emailCheck, ... }:
 let
   defaultPassiveInfo = {
     filter = lib.attrsets.filterAttrs
@@ -24,5 +24,8 @@ in
       use = "local-service";
       check_command = ["check_mailq"];
     }
+    (emailCheck "eldiron" hostFQDN // {
+      passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-email"; freshness_threshold = "1350"; };
+    })
   ];
 }
diff --git a/modules/private/monitoring/objects_immae-eu.nix b/modules/private/monitoring/objects_immae-eu.nix
index a6337e9..cffb180 100644
--- a/modules/private/monitoring/objects_immae-eu.nix
+++ b/modules/private/monitoring/objects_immae-eu.nix
@@ -84,12 +84,5 @@ in
       service_description = "mailq is empty";
       servicegroups = "webstatus-email";
     }
-
-    ## Sending e-mail
-    {
-      service_description = "immae.eu email service is active";
-      servicegroups = "webstatus-email";
-      freshness_threshold = "1350";
-    }
   ];
 }
diff --git a/modules/private/monitoring/objects_phare.nix b/modules/private/monitoring/objects_phare.nix
index ab46436..a00e5e8 100644
--- a/modules/private/monitoring/objects_phare.nix
+++ b/modules/private/monitoring/objects_phare.nix
@@ -1,4 +1,4 @@
-{ ... }:
+{ emailCheck, ... }:
 {
   host = {
     "phare.normalesup.org" = {
@@ -10,13 +10,6 @@
     };
   };
   service = [
-    {
-      service_description = "phare.normalesup.org email service is active";
-      use = "external-passive-service";
-      host_name = "phare.normalesup.org";
-      freshness_threshold = "1350";
-      retry_interval = "1";
-      servicegroups = "webstatus-email";
-    }
+    (emailCheck "phare" "phare.normalesup.org")
   ];
 }
diff --git a/modules/private/monitoring/objects_ulminfo-fr.nix b/modules/private/monitoring/objects_ulminfo-fr.nix
index 87a3e05..b970ecd 100644
--- a/modules/private/monitoring/objects_ulminfo-fr.nix
+++ b/modules/private/monitoring/objects_ulminfo-fr.nix
@@ -1,4 +1,4 @@
-{ ... }:
+{ emailCheck, ... }:
 {
   host = {
     "ulminfo.fr" = {
@@ -10,13 +10,6 @@
     };
   };
   service = [
-    {
-      service_description = "ulminfo.fr email service is active";
-      use = "external-passive-service";
-      host_name = "ulminfo.fr";
-      freshness_threshold = "1350";
-      retry_interval = "1";
-      servicegroups = "webstatus-email";
-    }
+    (emailCheck "ulminfo" "ulminfo.fr")
   ];
 }
diff --git a/modules/private/monitoring/plugins/check_emails b/modules/private/monitoring/plugins/check_emails
new file mode 100755
index 0000000..0ee3e4e
--- /dev/null
+++ b/modules/private/monitoring/plugins/check_emails
@@ -0,0 +1,119 @@
+#!/usr/bin/env perl
+
+use strict;
+use Getopt::Std;
+use File::Basename;
+use Date::Parse;
+use POSIX qw(strftime);
+
+$| = 1;
+
+my %opts;
+getopts('hH:l:s:p:f:i:n:r:', \%opts);
+
+my $STATE_OK = 0;
+my $STATE_WARNING = 1;
+my $STATE_CRITICAL = 2;
+my $STATE_UNKNOWN = 3;
+
+if ($opts{'h'} || scalar(%opts) == 0) {
+  &print_help();
+  exit($STATE_OK);
+}
+
+my $port = $opts{'p'};
+my $host = $opts{'H'};
+my $login = $opts{'l'};
+if ($login ne '') {
+  $login = "$login@";
+}
+
+my $identity = $opts{'i'};
+my $local_directory = $opts{'n'};
+my $return_path = $opts{'r'};
+
+my @emails_to_send = split(/,/, $opts{'s'});
+my @emails_to_expect = split(/,/, $opts{'f'});
+
+my $cmd_result;
+if ($local_directory ne '') {
+  if (! -d $local_directory) {
+    print "Emails $host UNKNOWN - Could not find local directory";
+    exit($STATE_UNKNOWN);
+  }
+  $cmd_result = `send_mails $local_directory $return_path @emails_to_send 2>&1`;
+} else {
+  $cmd_result = `ssh -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -p $port -i $identity $login$host send_mails @emails_to_send 2>&1`;
+
+  if ($cmd_result =~ /Host key verification failed./) {
+    print "Emails $host UNKNOWN - Could not connect to host with ssh key\n";
+    exit($STATE_UNKNOWN);
+  }
+}
+
+my @lines = split(/\n/, $cmd_result);
+
+my %found_emails;
+
+foreach my $line (@lines) {
+  my @split_line = split(/;/, $line, 2);
+  $found_emails{$split_line[0]} = $split_line[1];
+}
+
+my $output = "";
+foreach my $email_from (@emails_to_expect) {
+  my @email_split = split(/:/, $email_from);
+  my $email = $email_split[0];
+  my $from = $email_split[1];
+
+  if ( exists $found_emails{$email} ) {
+    my $email_date = str2time($found_emails{$email});
+    my $current_date = strftime "%s", localtime;
+
+    if ($current_date - $email_date > 60*30) {
+      $output = "$output$email ($found_emails{$email} from $from) ";
+    }
+  } else {
+    $output = "$output$email (missing) "
+  }
+}
+
+if ($output ne '') {
+  print "Emails $host CRITICAL - expecting emails: $output\n";
+  exit($STATE_CRITICAL);
+} else {
+  print "Emails $host OK\n";
+  exit($STATE_OK);
+}
+
+sub print_help() {
+  print << "EOF";
+Check sent emails
+
+Options:
+-h
+    Print detailed help screen
+
+-H
+    Host to check
+
+-l
+    Login
+
+-i
+    Identity file
+
+-n
+    Don’t use ssh, pass that directory to script
+
+-r
+    Return path for local e-mails
+
+-s
+    Comma separated list of emails to send from the host.
+
+-f
+    Comma separated list of emails to expect on the host.
+EOF
+}
+
diff --git a/modules/private/monitoring/send_mails b/modules/private/monitoring/send_mails
new file mode 100755
index 0000000..105c505
--- /dev/null
+++ b/modules/private/monitoring/send_mails
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+CHECK_DIR=$1
+shift
+RETURN_PATH=$1
+shift
+
+for mail in "$@"; do
+  echo "Test Mail" | MAILRC=/dev/null mail -n -r "$RETURN_PATH" -s "TestMailImmae " "$mail"
+done
+
+if [ -d "$CHECK_DIR" ]; then
+  cd $CHECK_DIR
+  stat -c '%n;%y' *
+fi