X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=modules%2Fprivate%2Fmonitoring%2Fdefault.nix;h=d593e5503842b2067f394bdad4cdd8bcfab3b9b0;hb=c41d0de83b1841097c6be8d8baeed557b065657b;hp=b3f8cbee734cf53f3c672e88171a2f036282fc4d;hpb=6015a3b52c3b155ac444aeb39950c38a5e653101;p=perso%2FImmae%2FConfig%2FNix.git

diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index b3f8cbe..d593e55 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -1,12 +1,53 @@
-{ config, myconfig, pkgs, lib, name, hostFQDN, ... }:
+{ config, pkgs, lib, name, nodes, ... }:
 let
+  cfg = config.myServices.monitoring;
+  send_mails = pkgs.runCommand "send_mails" {
+    buildInputs = [ pkgs.makeWrapper ];
+  } ''
+    mkdir -p $out/bin
+    cp ${./send_mails} $out/bin/send_mails
+    patchShebangs $out
+    wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
+      pkgs.mailutils
+    ]}
+    '';
+  postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable
+    then config.myServices.databasesReplication.postgresql.mainPackage
+    else if config.myServices.databases.enable
+    then config.myServices.databases.postgresql.package
+    else pkgs.postgresql;
+  zfsPlugin = pkgs.fetchurl {
+    url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh";
+    sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf";
+  };
+  megacli = pkgs.megacli.overrideAttrs(old: { meta = old.meta // { license = null; }; });
+  megaCliPlugin = pkgs.runCommand "megaCliPlugin" {
+    plugin = pkgs.fetchurl {
+      name = "check_megaraid_sas";
+      url = "https://exchange.nagios.org/components/com_mtree/attachment.php?link_id=6381&cf_id=24";
+      sha256 = "0yf60p4c0hb4q3fng9fc14qc89bqm0f1sijayzygadaqcl44jx4p";
+    };
+  } ''
+    mkdir $out
+    cp $plugin $out/check_megaraid_sas
+    chmod +x $out/check_megaraid_sas
+    patchShebangs $out
+    substituteInPlace $out/check_megaraid_sas --replace /usr/sbin/MegaCli ${megacli}/bin/MegaCli64
+    substituteInPlace $out/check_megaraid_sas --replace 'sudo $megacli' '/run/wrappers/bin/sudo $megacli'
+    sed -i -e "s/use utils qw(%ERRORS);/my %ERRORS = ('OK' => 0, 'WARNING' => 1, 'CRITICAL' => 2, 'UNKNOWN' => 3);/" $out/check_megaraid_sas
+  '';
   myplugins = pkgs.runCommand "buildplugins" {
     buildInputs = [ pkgs.makeWrapper pkgs.perl ];
   } ''
     mkdir $out
+    cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh
+    cp ${megaCliPlugin}/check_megaraid_sas $out/
     cp ${./plugins}/* $out/
     patchShebangs $out
     wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
+    wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [
+      pkgs.which pkgs.zfs pkgs.gawk
+    ]}
     wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
       pkgs.curl pkgs.jq
     ]}
@@ -14,7 +55,7 @@ let
       pkgs.gnugrep pkgs.gawk pkgs.procps-ng
     ]}
     wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
-      pkgs.postgresql
+      postgresqlBinary
     ]}
     wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [
       pkgs.gnugrep pkgs.coreutils pkgs.redis
@@ -22,27 +63,142 @@ let
     wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [
       pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb
     ]}
+    wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
+      pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
+    ]}
+    wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
+      pkgs.openssh send_mails
+    ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
+      pkgs.perlPackages.TimeDate
+    ]}
+    wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
+      pkgs.lftp
+    ]}
+    wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [
+      pkgs.git pkgs.openssh
+    ]}
+    wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [
+      pkgs.openssl
+    ]}
+    wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
+      pkgs.s3cmd pkgs.python3
+    ]}
+    makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
+      pkgs.duplicity
+    ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
+    makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
+      pkgs.duplicity
+    ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
+    wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
+      pkgs.mailutils
+    ]}
+    wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
+      pkgs.curl pkgs.jq
+    ]}
+    wrapProgram $out/notify_eban_url --prefix PATH : ${lib.makeBinPath [
+      pkgs.curl
+    ]}
+    wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
+      (pkgs.python3.withPackages (ps: [ps.ovh]))
+    ]}
+    wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
+      pkgs.iproute pkgs.bc
+    ]}
     '';
   toObjects = pkgs.callPackage ./to_objects.nix {};
   commonConfig = {
+    dilion = {
+      processWarn = "250"; processAlert = "400";
+      loadWarn = "1.0"; loadAlert = "1.2";
+      interface = "eth0";
+    };
     eldiron = {
       processWarn = "250"; processAlert = "400";
-      loadWarn = "8.0"; loadAlert = "10.0";
+      loadWarn = "1.0"; loadAlert = "1.2";
+      interface = "eth0";
     };
     backup-2 = {
-      processWarn = "50"; processAlert = "60";
+      processWarn = "60"; processAlert = "70";
       loadWarn = "1.0"; loadAlert = "2.0";
+      interface = "ens3";
+    };
+    monitoring-1 = {
+      processWarn = "50"; processAlert = "60";
+      loadWarn = "4.0"; loadAlert = "6.0";
+      load15Warn = "1.0"; load15Alert = "2.0";
+      interface = "ens3";
     };
+    quatresaisons = {
+      processWarn = "250"; processAlert = "400";
+      loadWarn = "1.0"; loadAlert = "1.2";
+      interface = "eth0";
+    };
+  };
+  externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
+    (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
+  masterPassiveObjects = let
+    passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
+    toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
+    passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
+      (_: n: toPassiveServices n.config.myServices.monitoring.services)
+      passiveNodes
+      ) ++ lib.flatten (lib.attrsets.mapAttrsToList
+      (_: n: toPassiveServices n.service)
+      externalObjects);
+  in {
+    service = passiveServices;
+    host = lib.lists.foldr
+      (a: b: a//b)
+      {}
+      (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
+      ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
   };
+  emailCheck = host: hostFQDN: let
+    allCfg = config.myEnv.monitoring.email_check;
+    cfg = allCfg."${host}";
+    reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
+    to_email = cfg': host':
+      let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
+      in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
+    mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
+    mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
+    command = if cfg.local
+    then
+      [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
+    else
+      [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
+  in
+    {
+      service_description = "${hostFQDN} email service is active";
+      use = "mail-service";
+      host_name = hostFQDN;
+      servicegroups = "webstatus-email";
+      check_command = command;
+    };
+  otherObjects = map
+    (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
+    [ "ulminfo-fr" "phare" "eban" ];
+  masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
   commonObjects = pkgs.callPackage ./objects_common.nix ({
-    inherit hostFQDN;
+    master = cfg.master;
+    hostFQDN = config.hostEnv.fqdn;
+    hostName = name;
     sudo = "/run/wrappers/bin/sudo";
   } // builtins.getAttr name commonConfig);
   hostObjects =
     let
       specific_file = ./. + "/objects_" + name + ".nix";
     in
-      lib.attrsets.optionalAttrs (builtins.pathExists specific_file) (pkgs.callPackage specific_file {});
+      lib.attrsets.optionalAttrs
+        (builtins.pathExists specific_file)
+        (pkgs.callPackage specific_file {
+          inherit config nodes emailCheck;
+          hostFQDN = config.hostEnv.fqdn;
+          hostName = name;
+        });
+  objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
+    "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
+  ) externalObjects;
 in
 {
   options = {
@@ -54,10 +210,31 @@ in
           Whether to enable monitoring.
         '';
       };
+      master = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          This instance is the master instance
+        '';
+      };
+      hosts = lib.mkOption {
+        readOnly = true;
+        description = "Hosts list for this host";
+        default = (commonObjects.host or {}) // (hostObjects.host or {});
+      };
+      services = lib.mkOption {
+        readOnly = true;
+        description = "Services list for this host";
+        default = commonObjects.service ++ hostObjects.service;
+      };
     };
   };
 
-  config = lib.mkIf config.myServices.monitoring.enable {
+  config = lib.mkIf cfg.enable {
+    services.nginx = lib.mkIf config.myServices.status.enable {
+      virtualHosts."status.immae.eu".locations = objectsFiles;
+    };
+
     services.duplyBackup.profiles.monitoring = {
       rootDir = config.services.naemon.varDir;
     };
@@ -66,69 +243,118 @@ in
         commands = [
           { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
           { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
+          { command = "${megacli}/bin/MegaCli64"; options = [ "NOPASSWD" ]; }
         ];
         users = [ "naemon" ];
         runAs = "root";
       }
       {
         commands = [
-          { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
           { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
         ];
         users = [ "naemon" ];
+        runAs = "ALL";
+      }
+      {
+        commands = [
+          { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
+        ];
+        users = [ "naemon" ];
         runAs = "postgres";
       }
       {
         commands = [
           { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; }
-          { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
         ];
         users = [ "naemon" ];
         runAs = "mysql";
       }
       {
         commands = [
-          { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; }
-          { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
+          { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; }
         ];
         users = [ "naemon" ];
-        runAs = "redis";
+        runAs = "openldap";
       }
       {
         commands = [
-          { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
+          { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; }
         ];
         users = [ "naemon" ];
-        runAs = "backup";
+        runAs = "redis";
       }
     ];
     environment.etc."mdadm.conf" = {
       enable = true;
       mode = "0644";
       user = "root";
-      text = "MAILADDR ${myconfig.env.monitoring.email}";
+      text = "MAILADDR ${config.myEnv.monitoring.email}";
     };
 
+    secrets.keys = [
+      {
+        dest = "naemon/id_rsa";
+        user = "naemon";
+        group = "naemon";
+        permissions = "0400";
+        text = config.myEnv.monitoring.ssh_secret_key;
+      }
+    ] ++ lib.optionals cfg.master (
+      lib.mapAttrsToList (k: v:
+      {
+        dest = "${k}_access_key";
+        user = "naemon";
+        group = "naemon";
+        permissions = "0400";
+        text = ''
+          export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
+          export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
+          export BASE_URL="${v.remote "immae-eldiron"}"
+        '';
+      }) config.myEnv.backup.remotes
+    );
     # needed since extraResource is not in the closure
     systemd.services.naemon.path = [ myplugins ];
     services.naemon = {
       enable = true;
       extraConfig = ''
-        broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
         use_syslog=1
         log_initial_states=1
         date_format=iso8601
-        admin_email=${myconfig.env.monitoring.email}
-
+        admin_email=${config.myEnv.monitoring.email}
+      '' + lib.optionalString (!cfg.master) ''
         obsess_over_services=1
         ocsp_command=notify-master
+      '' + lib.optionalString (cfg.master) ''
+        broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
+        broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
       '';
       extraResource = ''
         $USER2$=${myplugins}
-        $USER200$=${myconfig.env.monitoring.status_url}
-        $USER201$=${myconfig.env.monitoring.status_token}
+        $USER200$=${config.myEnv.monitoring.status_url}
+        $USER201$=${config.myEnv.monitoring.status_token}
+        $USER202$=${config.myEnv.monitoring.http_user_password}
+        $USER203$=${config.secrets.fullPaths."naemon/id_rsa"}
+        $USER204$=${config.myEnv.monitoring.imap_login}
+        $USER205$=${config.myEnv.monitoring.imap_password}
+        $USER206$=${config.myEnv.monitoring.slack_channel}
+        $USER207$=${config.myEnv.monitoring.slack_url}
+        $USER208$=${builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys)}
+        $USER209$=${builtins.concatStringsSep "," [
+          config.myEnv.monitoring.ovh_sms.endpoint
+          config.myEnv.monitoring.ovh_sms.application_key
+          config.myEnv.monitoring.ovh_sms.application_secret
+          config.myEnv.monitoring.ovh_sms.consumer_key
+          config.myEnv.monitoring.ovh_sms.account
+        ]}
+        $USER210$=${config.myEnv.monitoring.eban.user}
+        $USER211$=${config.myEnv.monitoring.eban.password}
       '';
-      objectDefs = toObjects commonObjects + toObjects hostObjects;
+      objectDefs = toObjects commonObjects
+        + toObjects hostObjects
+        + lib.optionalString cfg.master (toObjects masterObjects)
+        + lib.optionalString cfg.master (toObjects masterPassiveObjects)
+        + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
     };
   };
 }