From 71a2425ed95120a6de3a41bb233b1066779d4c26 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Fri, 17 Jan 2020 01:15:04 +0100 Subject: [PATCH] Add e-mail checks monitoring --- modules/private/environment.nix | 13 ++ modules/private/mail/postfix.nix | 36 +++++- modules/private/mail/relay.nix | 6 +- modules/private/monitoring/default.nix | 54 ++++++-- modules/private/monitoring/objects_common.nix | 2 + .../private/monitoring/objects_eldiron.nix | 5 +- .../private/monitoring/objects_immae-eu.nix | 7 -- modules/private/monitoring/objects_phare.nix | 11 +- .../private/monitoring/objects_ulminfo-fr.nix | 11 +- .../private/monitoring/plugins/check_emails | 119 ++++++++++++++++++ modules/private/monitoring/send_mails | 15 +++ 11 files changed, 240 insertions(+), 39 deletions(-) create mode 100755 modules/private/monitoring/plugins/check_emails create mode 100755 modules/private/monitoring/send_mails diff --git a/modules/private/environment.nix b/modules/private/environment.nix index 81b5df5..7da2480 100644 --- a/modules/private/environment.nix +++ b/modules/private/environment.nix @@ -486,6 +486,19 @@ in slack_url = mkOption { type = str; description = "Slack webhook url to push status update"; }; slack_channel = mkOption { type = str; description = "Slack channel to push status update"; }; contacts = mkOption { type = attrsOf unspecified; description = "Contact dicts to fill naemon objects"; }; + email_check = mkOption { + description = "Emails services to check"; + type = attrsOf (submodule { + options = { + local = mkOption { type = bool; default = false; description = "Use local configuration"; }; + port = mkOption { type = nullOr str; default = null; description = "Port to connect to ssh"; }; + login = mkOption { type = nullOr str; default = null; description = "Login to connect to ssh"; }; + targets = mkOption { type = listOf str; description = "Hosts to send E-mails to"; }; + mail_address = mkOption { type = str; description = "E-mail recipient part to send e-mail to"; }; + mail_domain = mkOption { type = str; description = "E-mail domain part to send e-mail to"; }; + }; + }); + }; }; }; }; diff --git a/modules/private/mail/postfix.nix b/modules/private/mail/postfix.nix index 8fe06da..51f4de7 100644 --- a/modules/private/mail/postfix.nix +++ b/modules/private/mail/postfix.nix @@ -1,4 +1,4 @@ -{ lib, pkgs, config, nodes, name, ... }: +{ lib, pkgs, config, nodes, ... }: { config = lib.mkIf config.myServices.mail.enable { services.duplyBackup.profiles.mail.excludeFile = '' @@ -220,7 +220,13 @@ ''; scripts = lib.attrsets.mapAttrs (n: v: toScript n (pkgs.callPackage (builtins.fetchGit { url = v.src.url; ref = "master"; rev = v.src.rev; }) { scriptEnv = v.env; }) - ) config.myEnv.mail.scripts; + ) config.myEnv.mail.scripts // { + testmail = pkgs.writeScript "testmail" '' + #! ${pkgs.stdenv.shell} + ${pkgs.coreutils}/bin/touch \ + "/var/lib/naemon/checks/email/$(${pkgs.procmail}/bin/formail -x To: | ${pkgs.coreutils}/bin/tr -d ' <>')" + ''; + }; in builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList (n: v: ''${n}: "|${v}"'') scripts); mapFiles = let recipient_maps = let @@ -246,8 +252,12 @@ ) config.myEnv.mail.postfix.backup_domains ); virtual_map = { - virtual = pkgs.writeText "postfix-virtual" ( + virtual = let + cfg = config.myEnv.monitoring.email_check.eldiron; + address = "${cfg.mail_address}@${cfg.mail_domain}"; + in pkgs.writeText "postfix-virtual" ( builtins.concatStringsSep "\n" ( + ["${address} testmail@localhost"] ++ lib.attrsets.mapAttrsToList ( n: v: lib.optionalString v.external '' script_${n}@mail.immae.eu ${n}@localhost, scripts@mail.immae.eu @@ -425,5 +435,25 @@ "smtp.immae.eu" = null; }; }; + system.activationScripts.testmail = { + deps = [ "users" ]; + text = let + allCfg = config.myEnv.monitoring.email_check; + cfg = allCfg.eldiron; + reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem "eldiron" v.targets) allCfg); + to_email = cfg': host': + let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+"; + in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}"; + mails_to_receive = builtins.concatStringsSep " " (map (to_email cfg) reverseTargets); + in '' + install -m 0555 -o nobody -g nogroup -d /var/lib/naemon/checks/email + for f in ${mails_to_receive}; do + if [ ! -f /var/lib/naemon/checks/email/$f ]; then + install -m 0644 -o nobody -g nogroup /dev/null -T /var/lib/naemon/checks/email/$f + touch -m -d @0 /var/lib/naemon/checks/email/$f + fi + done + ''; + }; }; } diff --git a/modules/private/mail/relay.nix b/modules/private/mail/relay.nix index e0aa387..6ac3df8 100644 --- a/modules/private/mail/relay.nix +++ b/modules/private/mail/relay.nix @@ -126,8 +126,12 @@ ) config.myEnv.mail.postfix.backup_domains ); virtual_map = { - virtual = pkgs.writeText "postfix-virtual" ( + virtual = let + cfg = config.myEnv.monitoring.email_check.eldiron; + address = "${cfg.mail_address}@${cfg.mail_domain}"; + in pkgs.writeText "postfix-virtual" ( builtins.concatStringsSep "\n" ( + ["${address} 1"] ++ lib.attrsets.mapAttrsToList ( n: v: lib.optionalString v.external '' script_${n}@mail.immae.eu 1 diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix index e1357a7..a7013af 100644 --- a/modules/private/monitoring/default.nix +++ b/modules/private/monitoring/default.nix @@ -1,6 +1,16 @@ { config, pkgs, lib, name, nodes, ... }: let cfg = config.myServices.monitoring; + send_mails = pkgs.runCommand "send_mails" { + buildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out/bin + cp ${./send_mails} $out/bin/send_mails + patchShebangs $out + wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [ + pkgs.mailutils + ]} + ''; myplugins = pkgs.runCommand "buildplugins" { buildInputs = [ pkgs.makeWrapper pkgs.perl ]; } '' @@ -29,6 +39,11 @@ let wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [ pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap ]} + wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [ + pkgs.openssh send_mails + ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [ + pkgs.perlPackages.TimeDate + ]} wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [ pkgs.lftp ]} @@ -67,11 +82,6 @@ let }; }; masterPassiveObjects = let - otherPassiveObjects = map - (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") {})) - [ "caldance-1" "ulminfo-fr" "immae-eu" "phare" "tiboqorl-fr" ]; - otherPassiveServices = lib.flatten (map (h: h.service or []) otherPassiveObjects); - otherPassiveHosts = (map (h: h.host)) otherPassiveObjects; passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron"]) nodes; toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo); passiveServices = lib.flatten (lib.attrsets.mapAttrsToList @@ -79,12 +89,37 @@ let passiveNodes ); in { - service = passiveServices ++ otherPassiveServices; + service = passiveServices; host = lib.lists.foldr (a: b: a//b) {} - (otherPassiveHosts ++ lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes); + (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes); }; + emailCheck = host: hostFQDN: let + allCfg = config.myEnv.monitoring.email_check; + cfg = allCfg."${host}"; + reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg); + to_email = cfg': host': + let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+"; + in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}"; + mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets); + mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets); + command = if cfg.local + then + [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ] + else + [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ]; + in + { + service_description = "${hostFQDN} email service is active"; + use = "mail-service"; + host_name = hostFQDN; + servicegroups = "webstatus-email"; + check_command = command; + }; + otherObjects = map + (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; })) + [ "caldance-1" "ulminfo-fr" "immae-eu" "phare" "tiboqorl-fr" ]; masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; }; commonObjects = pkgs.callPackage ./objects_common.nix ({ master = cfg.master; @@ -99,7 +134,7 @@ let lib.attrsets.optionalAttrs (builtins.pathExists specific_file) (pkgs.callPackage specific_file { - inherit config; + inherit config emailCheck; hostFQDN = config.hostEnv.fqdn; hostName = name; }); @@ -228,7 +263,8 @@ in objectDefs = toObjects commonObjects + toObjects hostObjects + lib.optionalString cfg.master (toObjects masterObjects) - + lib.optionalString cfg.master (toObjects masterPassiveObjects); + + lib.optionalString cfg.master (toObjects masterPassiveObjects) + + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects)); }; }; } diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix index 15eee97..a9d6da4 100644 --- a/modules/private/monitoring/objects_common.nix +++ b/modules/private/monitoring/objects_common.nix @@ -91,6 +91,8 @@ in ]; command = { check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$"; + check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$"; + check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$"; check_eriomem = "$USER2$/check_eriomem $USER208$"; check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$"; check_ftp_database = "$USER2$/check_ftp_database"; diff --git a/modules/private/monitoring/objects_eldiron.nix b/modules/private/monitoring/objects_eldiron.nix index bee4645..92f997f 100644 --- a/modules/private/monitoring/objects_eldiron.nix +++ b/modules/private/monitoring/objects_eldiron.nix @@ -1,4 +1,4 @@ -{ lib, hostFQDN, ... }: +{ lib, hostFQDN, emailCheck, ... }: let defaultPassiveInfo = { filter = lib.attrsets.filterAttrs @@ -24,5 +24,8 @@ in use = "local-service"; check_command = ["check_mailq"]; } + (emailCheck "eldiron" hostFQDN // { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-email"; freshness_threshold = "1350"; }; + }) ]; } diff --git a/modules/private/monitoring/objects_immae-eu.nix b/modules/private/monitoring/objects_immae-eu.nix index a6337e9..cffb180 100644 --- a/modules/private/monitoring/objects_immae-eu.nix +++ b/modules/private/monitoring/objects_immae-eu.nix @@ -84,12 +84,5 @@ in service_description = "mailq is empty"; servicegroups = "webstatus-email"; } - - ## Sending e-mail - { - service_description = "immae.eu email service is active"; - servicegroups = "webstatus-email"; - freshness_threshold = "1350"; - } ]; } diff --git a/modules/private/monitoring/objects_phare.nix b/modules/private/monitoring/objects_phare.nix index ab46436..a00e5e8 100644 --- a/modules/private/monitoring/objects_phare.nix +++ b/modules/private/monitoring/objects_phare.nix @@ -1,4 +1,4 @@ -{ ... }: +{ emailCheck, ... }: { host = { "phare.normalesup.org" = { @@ -10,13 +10,6 @@ }; }; service = [ - { - service_description = "phare.normalesup.org email service is active"; - use = "external-passive-service"; - host_name = "phare.normalesup.org"; - freshness_threshold = "1350"; - retry_interval = "1"; - servicegroups = "webstatus-email"; - } + (emailCheck "phare" "phare.normalesup.org") ]; } diff --git a/modules/private/monitoring/objects_ulminfo-fr.nix b/modules/private/monitoring/objects_ulminfo-fr.nix index 87a3e05..b970ecd 100644 --- a/modules/private/monitoring/objects_ulminfo-fr.nix +++ b/modules/private/monitoring/objects_ulminfo-fr.nix @@ -1,4 +1,4 @@ -{ ... }: +{ emailCheck, ... }: { host = { "ulminfo.fr" = { @@ -10,13 +10,6 @@ }; }; service = [ - { - service_description = "ulminfo.fr email service is active"; - use = "external-passive-service"; - host_name = "ulminfo.fr"; - freshness_threshold = "1350"; - retry_interval = "1"; - servicegroups = "webstatus-email"; - } + (emailCheck "ulminfo" "ulminfo.fr") ]; } diff --git a/modules/private/monitoring/plugins/check_emails b/modules/private/monitoring/plugins/check_emails new file mode 100755 index 0000000..0ee3e4e --- /dev/null +++ b/modules/private/monitoring/plugins/check_emails @@ -0,0 +1,119 @@ +#!/usr/bin/env perl + +use strict; +use Getopt::Std; +use File::Basename; +use Date::Parse; +use POSIX qw(strftime); + +$| = 1; + +my %opts; +getopts('hH:l:s:p:f:i:n:r:', \%opts); + +my $STATE_OK = 0; +my $STATE_WARNING = 1; +my $STATE_CRITICAL = 2; +my $STATE_UNKNOWN = 3; + +if ($opts{'h'} || scalar(%opts) == 0) { + &print_help(); + exit($STATE_OK); +} + +my $port = $opts{'p'}; +my $host = $opts{'H'}; +my $login = $opts{'l'}; +if ($login ne '') { + $login = "$login@"; +} + +my $identity = $opts{'i'}; +my $local_directory = $opts{'n'}; +my $return_path = $opts{'r'}; + +my @emails_to_send = split(/,/, $opts{'s'}); +my @emails_to_expect = split(/,/, $opts{'f'}); + +my $cmd_result; +if ($local_directory ne '') { + if (! -d $local_directory) { + print "Emails $host UNKNOWN - Could not find local directory"; + exit($STATE_UNKNOWN); + } + $cmd_result = `send_mails $local_directory $return_path @emails_to_send 2>&1`; +} else { + $cmd_result = `ssh -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -p $port -i $identity $login$host send_mails @emails_to_send 2>&1`; + + if ($cmd_result =~ /Host key verification failed./) { + print "Emails $host UNKNOWN - Could not connect to host with ssh key\n"; + exit($STATE_UNKNOWN); + } +} + +my @lines = split(/\n/, $cmd_result); + +my %found_emails; + +foreach my $line (@lines) { + my @split_line = split(/;/, $line, 2); + $found_emails{$split_line[0]} = $split_line[1]; +} + +my $output = ""; +foreach my $email_from (@emails_to_expect) { + my @email_split = split(/:/, $email_from); + my $email = $email_split[0]; + my $from = $email_split[1]; + + if ( exists $found_emails{$email} ) { + my $email_date = str2time($found_emails{$email}); + my $current_date = strftime "%s", localtime; + + if ($current_date - $email_date > 60*30) { + $output = "$output$email ($found_emails{$email} from $from) "; + } + } else { + $output = "$output$email (missing) " + } +} + +if ($output ne '') { + print "Emails $host CRITICAL - expecting emails: $output\n"; + exit($STATE_CRITICAL); +} else { + print "Emails $host OK\n"; + exit($STATE_OK); +} + +sub print_help() { + print << "EOF"; +Check sent emails + +Options: +-h + Print detailed help screen + +-H + Host to check + +-l + Login + +-i + Identity file + +-n + Don’t use ssh, pass that directory to script + +-r + Return path for local e-mails + +-s + Comma separated list of emails to send from the host. + +-f + Comma separated list of emails to expect on the host. +EOF +} + diff --git a/modules/private/monitoring/send_mails b/modules/private/monitoring/send_mails new file mode 100755 index 0000000..105c505 --- /dev/null +++ b/modules/private/monitoring/send_mails @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +CHECK_DIR=$1 +shift +RETURN_PATH=$1 +shift + +for mail in "$@"; do + echo "Test Mail" | MAILRC=/dev/null mail -n -r "$RETURN_PATH" -s "TestMailImmae " "$mail" +done + +if [ -d "$CHECK_DIR" ]; then + cd $CHECK_DIR + stat -c '%n;%y' * +fi -- 2.41.0