modules/private/monitoring/default.nix

   1 { config, pkgs, lib, name, nodes, ... }:
   2 let
   3   cfg = config.myServices.monitoring;
   4   send_mails = pkgs.runCommand "send_mails" {
   5     buildInputs = [ pkgs.makeWrapper ];
   6   } ''
   7     mkdir -p $out/bin
   8     cp ${./send_mails} $out/bin/send_mails
   9     patchShebangs $out
  10     wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
  11       pkgs.mailutils
  12     ]}
  13     '';
  14   postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable
  15     then config.myServices.databasesReplication.postgresql.mainPackage
  16     else if config.myServices.databases.enable
  17     then config.myServices.databases.postgresql.package
  18     else pkgs.postgresql;
  19   zfsPlugin = pkgs.fetchurl {
  20     url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh";
  21     sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf";
  22   };
  23   megacli = pkgs.megacli.overrideAttrs(old: { meta = old.meta // { license = null; }; });
  24   megaCliPlugin = pkgs.runCommand "megaCliPlugin" {
  25     plugin = pkgs.fetchurl {
  26       name = "check_megaraid_sas";
  27       url = "https://exchange.nagios.org/components/com_mtree/attachment.php?link_id=6381&cf_id=24";
  28       sha256 = "0yf60p4c0hb4q3fng9fc14qc89bqm0f1sijayzygadaqcl44jx4p";
  29     };
  30   } ''
  31     mkdir $out
  32     cp $plugin $out/check_megaraid_sas
  33     chmod +x $out/check_megaraid_sas
  34     patchShebangs $out
  35     substituteInPlace $out/check_megaraid_sas --replace /usr/sbin/MegaCli ${megacli}/bin/MegaCli64
  36     substituteInPlace $out/check_megaraid_sas --replace 'sudo $megacli' '/run/wrappers/bin/sudo $megacli'
  37     sed -i -e "s/use utils qw(%ERRORS);/my %ERRORS = ('OK' => 0, 'WARNING' => 1, 'CRITICAL' => 2, 'UNKNOWN' => 3);/" $out/check_megaraid_sas
  38   '';
  39   myplugins = pkgs.runCommand "buildplugins" {
  40     buildInputs = [ pkgs.makeWrapper pkgs.perl ];
  41   } ''
  42     mkdir $out
  43     cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh
  44     cp ${megaCliPlugin}/check_megaraid_sas $out/
  45     cp ${./plugins}/* $out/
  46     patchShebangs $out
  47     wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
  48     wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [
  49       pkgs.which pkgs.zfs pkgs.gawk
  50     ]}
  51     wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
  52       pkgs.curl pkgs.jq
  53     ]}
  54     wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
  55       pkgs.gnugrep pkgs.gawk pkgs.procps-ng
  56     ]}
  57     wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
  58       postgresqlBinary
  59     ]}
  60     wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [
  61       pkgs.gnugrep pkgs.coreutils pkgs.redis
  62     ]}
  63     wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [
  64       pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb
  65     ]}
  66     wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
  67       pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
  68     ]}
  69     wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
  70       pkgs.openssh send_mails
  71     ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
  72       pkgs.perlPackages.TimeDate
  73     ]}
  74     wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
  75       pkgs.lftp
  76     ]}
  77     wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [
  78       pkgs.git pkgs.openssh
  79     ]}
  80     wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [
  81       pkgs.openssl
  82     ]}
  83     wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
  84       pkgs.s3cmd pkgs.python3
  85     ]}
  86     makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
  87       pkgs.duplicity
  88     ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
  89     makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
  90       pkgs.duplicity
  91     ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
  92     wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
  93       pkgs.mailutils
  94     ]}
  95     wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
  96       pkgs.curl pkgs.jq
  97     ]}
  98     wrapProgram $out/notify_eban_url --prefix PATH : ${lib.makeBinPath [
  99       pkgs.curl
 100     ]}
 101     wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
 102       (pkgs.python3.withPackages (ps: [ps.ovh]))
 103     ]}
 104     wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
 105       pkgs.iproute pkgs.bc
 106     ]}
 107     '';
 108   toObjects = pkgs.callPackage ./to_objects.nix {};
 109   commonConfig = {
 110     dilion = {
 111       processWarn = "250"; processAlert = "400";
 112       loadWarn = "8.0"; loadAlert = "10.0";
 113       interface = "eth0";
 114     };
 115     eldiron = {
 116       processWarn = "250"; processAlert = "400";
 117       loadWarn = "8.0"; loadAlert = "10.0";
 118       interface = "eth0";
 119     };
 120     backup-2 = {
 121       processWarn = "60"; processAlert = "70";
 122       loadWarn = "1.0"; loadAlert = "2.0";
 123       interface = "ens3";
 124     };
 125     monitoring-1 = {
 126       processWarn = "50"; processAlert = "60";
 127       loadWarn = "4.0"; loadAlert = "6.0";
 128       load15Warn = "1.0"; load15Alert = "2.0";
 129       interface = "ens3";
 130     };
 131     quatresaisons = {
 132       processWarn = "250"; processAlert = "400";
 133       loadWarn = "8.0"; loadAlert = "10.0";
 134       interface = "eth0";
 135     };
 136   };
 137   externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
 138     (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
 139   masterPassiveObjects = let
 140     passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
 141     toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
 142     passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
 143       (_: n: toPassiveServices n.config.myServices.monitoring.services)
 144       passiveNodes
 145       ) ++ lib.flatten (lib.attrsets.mapAttrsToList
 146       (_: n: toPassiveServices n.service)
 147       externalObjects);
 148   in {
 149     service = passiveServices;
 150     host = lib.lists.foldr
 151       (a: b: a//b)
 152       {}
 153       (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
 154       ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
 155   };
 156   emailCheck = host: hostFQDN: let
 157     allCfg = config.myEnv.monitoring.email_check;
 158     cfg = allCfg."${host}";
 159     reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
 160     to_email = cfg': host':
 161       let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
 162       in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
 163     mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
 164     mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
 165     command = if cfg.local
 166     then
 167       [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
 168     else
 169       [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
 170   in
 171     {
 172       service_description = "${hostFQDN} email service is active";
 173       use = "mail-service";
 174       host_name = hostFQDN;
 175       servicegroups = "webstatus-email";
 176       check_command = command;
 177     };
 178   otherObjects = map
 179     (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
 180     [ "ulminfo-fr" "phare" "eban" ];
 181   masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
 182   commonObjects = pkgs.callPackage ./objects_common.nix ({
 183     master = cfg.master;
 184     hostFQDN = config.hostEnv.fqdn;
 185     hostName = name;
 186     sudo = "/run/wrappers/bin/sudo";
 187   } // builtins.getAttr name commonConfig);
 188   hostObjects =
 189     let
 190       specific_file = ./. + "/objects_" + name + ".nix";
 191     in
 192       lib.attrsets.optionalAttrs
 193         (builtins.pathExists specific_file)
 194         (pkgs.callPackage specific_file {
 195           inherit config nodes emailCheck;
 196           hostFQDN = config.hostEnv.fqdn;
 197           hostName = name;
 198         });
 199   objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
 200     "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
 201   ) externalObjects;
 202 in
 203 {
 204   options = {
 205     myServices.monitoring = {
 206       enable = lib.mkOption {
 207         type = lib.types.bool;
 208         default = false;
 209         description = ''
 210           Whether to enable monitoring.
 211         '';
 212       };
 213       master = lib.mkOption {
 214         type = lib.types.bool;
 215         default = false;
 216         description = ''
 217           This instance is the master instance
 218         '';
 219       };
 220       hosts = lib.mkOption {
 221         readOnly = true;
 222         description = "Hosts list for this host";
 223         default = (commonObjects.host or {}) // (hostObjects.host or {});
 224       };
 225       services = lib.mkOption {
 226         readOnly = true;
 227         description = "Services list for this host";
 228         default = commonObjects.service ++ hostObjects.service;
 229       };
 230     };
 231   };
 232
 233   config = lib.mkIf cfg.enable {
 234     services.nginx = lib.mkIf config.myServices.status.enable {
 235       virtualHosts."status.immae.eu".locations = objectsFiles;
 236     };
 237
 238     services.duplyBackup.profiles.monitoring = {
 239       rootDir = config.services.naemon.varDir;
 240     };
 241     security.sudo.extraRules = [
 242       {
 243         commands = [
 244           { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
 245           { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
 246           { command = "${megacli}/bin/MegaCli64"; options = [ "NOPASSWD" ]; }
 247         ];
 248         users = [ "naemon" ];
 249         runAs = "root";
 250       }
 251       {
 252         commands = [
 253           { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
 254         ];
 255         users = [ "naemon" ];
 256         runAs = "ALL";
 257       }
 258       {
 259         commands = [
 260           { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
 261         ];
 262         users = [ "naemon" ];
 263         runAs = "postgres";
 264       }
 265       {
 266         commands = [
 267           { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; }
 268         ];
 269         users = [ "naemon" ];
 270         runAs = "mysql";
 271       }
 272       {
 273         commands = [
 274           { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; }
 275         ];
 276         users = [ "naemon" ];
 277         runAs = "openldap";
 278       }
 279       {
 280         commands = [
 281           { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; }
 282         ];
 283         users = [ "naemon" ];
 284         runAs = "redis";
 285       }
 286     ];
 287     environment.etc."mdadm.conf" = {
 288       enable = true;
 289       mode = "0644";
 290       user = "root";
 291       text = "MAILADDR ${config.myEnv.monitoring.email}";
 292     };
 293
 294     secrets.keys = [
 295       {
 296         dest = "naemon/id_rsa";
 297         user = "naemon";
 298         group = "naemon";
 299         permissions = "0400";
 300         text = config.myEnv.monitoring.ssh_secret_key;
 301       }
 302     ] ++ lib.optionals cfg.master (
 303       lib.mapAttrsToList (k: v:
 304       {
 305         dest = "${k}_access_key";
 306         user = "naemon";
 307         group = "naemon";
 308         permissions = "0400";
 309         text = ''
 310           export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
 311           export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
 312           export BASE_URL="${v.remote "immae-eldiron"}"
 313         '';
 314       }) config.myEnv.backup.remotes
 315     );
 316     # needed since extraResource is not in the closure
 317     systemd.services.naemon.path = [ myplugins ];
 318     services.naemon = {
 319       enable = true;
 320       extraConfig = ''
 321         use_syslog=1
 322         log_initial_states=1
 323         date_format=iso8601
 324         admin_email=${config.myEnv.monitoring.email}
 325       '' + lib.optionalString (!cfg.master) ''
 326         obsess_over_services=1
 327         ocsp_command=notify-master
 328       '' + lib.optionalString (cfg.master) ''
 329         broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
 330         broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
 331       '';
 332       extraResource = ''
 333         $USER2$=${myplugins}
 334         $USER200$=${config.myEnv.monitoring.status_url}
 335         $USER201$=${config.myEnv.monitoring.status_token}
 336         $USER202$=${config.myEnv.monitoring.http_user_password}
 337         $USER203$=${config.secrets.fullPaths."naemon/id_rsa"}
 338         $USER204$=${config.myEnv.monitoring.imap_login}
 339         $USER205$=${config.myEnv.monitoring.imap_password}
 340         $USER206$=${config.myEnv.monitoring.slack_channel}
 341         $USER207$=${config.myEnv.monitoring.slack_url}
 342         $USER208$=${builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys)}
 343         $USER209$=${builtins.concatStringsSep "," [
 344           config.myEnv.monitoring.ovh_sms.endpoint
 345           config.myEnv.monitoring.ovh_sms.application_key
 346           config.myEnv.monitoring.ovh_sms.application_secret
 347           config.myEnv.monitoring.ovh_sms.consumer_key
 348           config.myEnv.monitoring.ovh_sms.account
 349         ]}
 350         $USER210$=${config.myEnv.monitoring.eban.password}
 351       '';
 352       objectDefs = toObjects commonObjects
 353         + toObjects hostObjects
 354         + lib.optionalString cfg.master (toObjects masterObjects)
 355         + lib.optionalString cfg.master (toObjects masterPassiveObjects)
 356         + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
 357     };
 358   };
 359 }