modules/private/monitoring/default.nix

   1 { config, pkgs, lib, name, nodes, ... }:
   2 let
   3   cfg = config.myServices.monitoring;
   4   send_mails = pkgs.runCommand "send_mails" {
   5     buildInputs = [ pkgs.makeWrapper ];
   6   } ''
   7     mkdir -p $out/bin
   8     cp ${./send_mails} $out/bin/send_mails
   9     patchShebangs $out
  10     wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
  11       pkgs.mailutils
  12     ]}
  13     '';
  14   postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable
  15     then config.myServices.databasesReplication.postgresql.mainPackage
  16     else if config.myServices.databases.enable
  17     then config.myServices.databases.postgresql.package
  18     else pkgs.postgresql;
  19   myplugins = pkgs.runCommand "buildplugins" {
  20     buildInputs = [ pkgs.makeWrapper pkgs.perl ];
  21   } ''
  22     mkdir $out
  23     cp ${./plugins}/* $out/
  24     patchShebangs $out
  25     wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
  26     wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
  27       pkgs.curl pkgs.jq
  28     ]}
  29     wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
  30       pkgs.gnugrep pkgs.gawk pkgs.procps-ng
  31     ]}
  32     wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
  33       postgresqlBinary
  34     ]}
  35     wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [
  36       pkgs.gnugrep pkgs.coreutils pkgs.redis
  37     ]}
  38     wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [
  39       pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb
  40     ]}
  41     wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
  42       pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
  43     ]}
  44     wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
  45       pkgs.openssh send_mails
  46     ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
  47       pkgs.perlPackages.TimeDate
  48     ]}
  49     wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
  50       pkgs.lftp
  51     ]}
  52     wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [
  53       pkgs.git pkgs.openssh
  54     ]}
  55     wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [
  56       pkgs.openssl
  57     ]}
  58     wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
  59       pkgs.s3cmd pkgs.python3
  60     ]}
  61     makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
  62       pkgs.duplicity
  63     ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
  64     makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
  65       pkgs.duplicity
  66     ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
  67     wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
  68       pkgs.mailutils
  69     ]}
  70     wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
  71       pkgs.curl pkgs.jq
  72     ]}
  73     wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
  74       (pkgs.python3.withPackages (ps: [ps.ovh]))
  75     ]}
  76     wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
  77       pkgs.iproute pkgs.bc
  78     ]}
  79     '';
  80   toObjects = pkgs.callPackage ./to_objects.nix {};
  81   commonConfig = {
  82     eldiron = {
  83       processWarn = "250"; processAlert = "400";
  84       loadWarn = "8.0"; loadAlert = "10.0";
  85       interface = "eth0";
  86     };
  87     backup-2 = {
  88       processWarn = "60"; processAlert = "70";
  89       loadWarn = "1.0"; loadAlert = "2.0";
  90       interface = "ens3";
  91     };
  92     monitoring-1 = {
  93       processWarn = "50"; processAlert = "60";
  94       loadWarn = "1.0"; loadAlert = "2.0";
  95       interface = "ens3";
  96     };
  97   };
  98   externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
  99     (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
 100   masterPassiveObjects = let
 101     passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron"]) nodes;
 102     toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
 103     passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
 104       (_: n: toPassiveServices n.config.myServices.monitoring.services)
 105       passiveNodes
 106       ) ++ lib.flatten (lib.attrsets.mapAttrsToList
 107       (_: n: toPassiveServices n.service)
 108       externalObjects);
 109   in {
 110     service = passiveServices;
 111     host = lib.lists.foldr
 112       (a: b: a//b)
 113       {}
 114       (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
 115       ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
 116   };
 117   emailCheck = host: hostFQDN: let
 118     allCfg = config.myEnv.monitoring.email_check;
 119     cfg = allCfg."${host}";
 120     reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
 121     to_email = cfg': host':
 122       let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
 123       in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
 124     mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
 125     mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
 126     command = if cfg.local
 127     then
 128       [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
 129     else
 130       [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
 131   in
 132     {
 133       service_description = "${hostFQDN} email service is active";
 134       use = "mail-service";
 135       host_name = hostFQDN;
 136       servicegroups = "webstatus-email";
 137       check_command = command;
 138     };
 139   otherObjects = map
 140     (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
 141     [ "ulminfo-fr" "phare" ];
 142   masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
 143   commonObjects = pkgs.callPackage ./objects_common.nix ({
 144     master = cfg.master;
 145     hostFQDN = config.hostEnv.fqdn;
 146     hostName = name;
 147     sudo = "/run/wrappers/bin/sudo";
 148   } // builtins.getAttr name commonConfig);
 149   hostObjects =
 150     let
 151       specific_file = ./. + "/objects_" + name + ".nix";
 152     in
 153       lib.attrsets.optionalAttrs
 154         (builtins.pathExists specific_file)
 155         (pkgs.callPackage specific_file {
 156           inherit config nodes emailCheck;
 157           hostFQDN = config.hostEnv.fqdn;
 158           hostName = name;
 159         });
 160   objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
 161     "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
 162   ) externalObjects;
 163 in
 164 {
 165   options = {
 166     myServices.monitoring = {
 167       enable = lib.mkOption {
 168         type = lib.types.bool;
 169         default = false;
 170         description = ''
 171           Whether to enable monitoring.
 172         '';
 173       };
 174       master = lib.mkOption {
 175         type = lib.types.bool;
 176         default = false;
 177         description = ''
 178           This instance is the master instance
 179         '';
 180       };
 181       hosts = lib.mkOption {
 182         readOnly = true;
 183         description = "Hosts list for this host";
 184         default = (commonObjects.host or {}) // (hostObjects.host or {});
 185       };
 186       services = lib.mkOption {
 187         readOnly = true;
 188         description = "Services list for this host";
 189         default = commonObjects.service ++ hostObjects.service;
 190       };
 191     };
 192   };
 193
 194   config = lib.mkIf cfg.enable {
 195     services.nginx = lib.mkIf config.myServices.status.enable {
 196       virtualHosts."status.immae.eu".locations = objectsFiles;
 197     };
 198
 199     services.duplyBackup.profiles.monitoring = {
 200       rootDir = config.services.naemon.varDir;
 201     };
 202     security.sudo.extraRules = [
 203       {
 204         commands = [
 205           { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
 206           { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
 207         ];
 208         users = [ "naemon" ];
 209         runAs = "root";
 210       }
 211       {
 212         commands = [
 213           { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
 214         ];
 215         users = [ "naemon" ];
 216         runAs = "ALL";
 217       }
 218       {
 219         commands = [
 220           { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
 221         ];
 222         users = [ "naemon" ];
 223         runAs = "postgres";
 224       }
 225       {
 226         commands = [
 227           { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; }
 228         ];
 229         users = [ "naemon" ];
 230         runAs = "mysql";
 231       }
 232       {
 233         commands = [
 234           { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; }
 235         ];
 236         users = [ "naemon" ];
 237         runAs = "openldap";
 238       }
 239       {
 240         commands = [
 241           { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; }
 242         ];
 243         users = [ "naemon" ];
 244         runAs = "redis";
 245       }
 246     ];
 247     environment.etc."mdadm.conf" = {
 248       enable = true;
 249       mode = "0644";
 250       user = "root";
 251       text = "MAILADDR ${config.myEnv.monitoring.email}";
 252     };
 253
 254     secrets.keys = [
 255       {
 256         dest = "naemon/id_rsa";
 257         user = "naemon";
 258         group = "naemon";
 259         permissions = "0400";
 260         text = config.myEnv.monitoring.ssh_secret_key;
 261       }
 262     ] ++ lib.optionals cfg.master (
 263       lib.mapAttrsToList (k: v:
 264       {
 265         dest = "${k}_access_key";
 266         user = "naemon";
 267         group = "naemon";
 268         permissions = "0400";
 269         text = ''
 270           export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
 271           export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
 272           export BASE_URL="${v.remote "immae-eldiron"}"
 273         '';
 274       }) config.myEnv.backup.remotes
 275     );
 276     # needed since extraResource is not in the closure
 277     systemd.services.naemon.path = [ myplugins ];
 278     services.naemon = {
 279       enable = true;
 280       extraConfig = ''
 281         use_syslog=1
 282         log_initial_states=1
 283         date_format=iso8601
 284         admin_email=${config.myEnv.monitoring.email}
 285       '' + lib.optionalString (!cfg.master) ''
 286         obsess_over_services=1
 287         ocsp_command=notify-master
 288       '' + lib.optionalString (cfg.master) ''
 289         broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
 290         broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
 291       '';
 292       extraResource = ''
 293         $USER2$=${myplugins}
 294         $USER200$=${config.myEnv.monitoring.status_url}
 295         $USER201$=${config.myEnv.monitoring.status_token}
 296         $USER202$=${config.myEnv.monitoring.http_user_password}
 297         $USER203$=${config.secrets.fullPaths."naemon/id_rsa"}
 298         $USER204$=${config.myEnv.monitoring.imap_login}
 299         $USER205$=${config.myEnv.monitoring.imap_password}
 300         $USER206$=${config.myEnv.monitoring.slack_channel}
 301         $USER207$=${config.myEnv.monitoring.slack_url}
 302         $USER208$=${builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys)}
 303         $USER209$=${builtins.concatStringsSep "," [
 304           config.myEnv.monitoring.ovh_sms.endpoint
 305           config.myEnv.monitoring.ovh_sms.application_key
 306           config.myEnv.monitoring.ovh_sms.application_secret
 307           config.myEnv.monitoring.ovh_sms.consumer_key
 308           config.myEnv.monitoring.ovh_sms.account
 309         ]}
 310       '';
 311       objectDefs = toObjects commonObjects
 312         + toObjects hostObjects
 313         + lib.optionalString cfg.master (toObjects masterObjects)
 314         + lib.optionalString cfg.master (toObjects masterPassiveObjects)
 315         + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
 316     };
 317   };
 318 }