modules/private/monitoring/default.nix

   1 { config, pkgs, lib, name, nodes, ... }:
   2 let
   3   cfg = config.myServices.monitoring;
   4   activatedPlugins = [ "memory" "command" "bandwidth" ]
   5     ++ (if cfg.master then (masterObjects.activatedPlugins or []) else [])
   6     ++ (if cfg.master then (lib.flatten (map (v: v.activatedPlugins or []) otherObjects)) else [])
   7     ++ (hostObjects.activatedPlugins or [])
   8     ++ (if cfg.master then ["notify-primary"] else ["notify-secondary"]);
   9   allPluginsConfig = import ./myplugins.nix {
  10     inherit pkgs lib config;
  11     sudo = "/run/wrappers/bin/sudo";
  12   };
  13   mypluginsConfig = lib.getAttrs activatedPlugins allPluginsConfig;
  14   myplugins = let
  15     mypluginsChunk = builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList (k: v: v.chunk or "") mypluginsConfig);
  16   in pkgs.runCommand "buildplugins" {
  17     buildInputs = [ pkgs.makeWrapper pkgs.perl ];
  18   } ''
  19     mkdir $out
  20     ${mypluginsChunk}
  21     '';
  22   toObjects = pkgs.callPackage ./to_objects.nix {};
  23   commonConfig = {
  24     dilion = {
  25       processWarn = "250"; processAlert = "400";
  26       loadWarn = "1.0"; loadAlert = "1.2";
  27       interface = "eth0";
  28     };
  29     eldiron = {
  30       processWarn = "550"; processAlert = "650";
  31       loadWarn = "1.0"; loadAlert = "1.2";
  32       interface = "eth0";
  33     };
  34     backup-2 = {
  35       processWarn = "60"; processAlert = "70";
  36       loadWarn = "1.0"; loadAlert = "2.0";
  37       interface = "ens3";
  38     };
  39     monitoring-1 = {
  40       processWarn = "50"; processAlert = "60";
  41       loadWarn = "4.0"; loadAlert = "6.0";
  42       load15Warn = "1.0"; load15Alert = "2.0";
  43       interface = "ens3";
  44     };
  45     quatresaisons = {
  46       processWarn = "250"; processAlert = "400";
  47       loadWarn = "1.0"; loadAlert = "1.2";
  48       interface = "eth0";
  49     };
  50   };
  51   externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
  52     (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
  53   masterPassiveObjects = let
  54     passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
  55     toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
  56     passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
  57       (_: n: toPassiveServices n.config.myServices.monitoring.services)
  58       passiveNodes
  59       ) ++ lib.flatten (lib.attrsets.mapAttrsToList
  60       (_: n: toPassiveServices n.service)
  61       externalObjects);
  62   in {
  63     service = passiveServices;
  64     host = lib.lists.foldr
  65       (a: b: a//b)
  66       {}
  67       (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
  68       ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
  69   };
  70   emailCheck = host: hostFQDN: let
  71     allCfg = config.myEnv.monitoring.email_check;
  72     cfg = allCfg."${host}";
  73     reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
  74     to_email = cfg': host':
  75       let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
  76       in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
  77     mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
  78     mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
  79     command = if cfg.local
  80     then
  81       [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
  82     else
  83       [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
  84   in
  85     {
  86       service_description = "${hostFQDN} email service is active";
  87       use = "mail-service";
  88       host_name = hostFQDN;
  89       servicegroups = "webstatus-email";
  90       check_command = command;
  91     };
  92   otherObjects = map
  93     (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
  94     [ "ulminfo-fr" "phare" ];
  95   masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
  96   commonObjects = pkgs.callPackage ./objects_common.nix ({
  97     master = cfg.master;
  98     hostFQDN = config.hostEnv.fqdn;
  99     hostName = name;
 100     inherit mypluginsConfig;
 101   } // builtins.getAttr name commonConfig);
 102   hostObjects =
 103     let
 104       specific_file = ./. + "/objects_" + name + ".nix";
 105     in
 106       lib.attrsets.optionalAttrs
 107         (builtins.pathExists specific_file)
 108         (pkgs.callPackage specific_file {
 109           inherit config nodes emailCheck;
 110           hostFQDN = config.hostEnv.fqdn;
 111           hostName = name;
 112         });
 113   objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
 114     "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
 115   ) externalObjects;
 116 in
 117 {
 118   options = {
 119     myServices.monitoring = {
 120       enable = lib.mkOption {
 121         type = lib.types.bool;
 122         default = false;
 123         description = ''
 124           Whether to enable monitoring.
 125         '';
 126       };
 127       master = lib.mkOption {
 128         type = lib.types.bool;
 129         default = false;
 130         description = ''
 131           This instance is the master instance
 132         '';
 133       };
 134       hosts = lib.mkOption {
 135         readOnly = true;
 136         description = "Hosts list for this host";
 137         default = (commonObjects.host or {}) // (hostObjects.host or {});
 138       };
 139       services = lib.mkOption {
 140         readOnly = true;
 141         description = "Services list for this host";
 142         default = commonObjects.service ++ hostObjects.service;
 143       };
 144     };
 145   };
 146
 147   config = lib.mkIf cfg.enable {
 148     myServices.chatonsProperties.hostings.monitoring = lib.mkIf cfg.master {
 149       file.datetime = "2022-08-27T16:00:00";
 150       hosting = {
 151         name = "Monitoring";
 152         description = "Website and server health monitoring";
 153         website = "https://status.immae.eu";
 154         logo = "https://www.naemon.io/favicon.ico";
 155         status.level = "OK";
 156         status.description = "OK";
 157         registration.load = "OPEN";
 158         install.type = "PACKAGE";
 159       };
 160       software = {
 161         name = "naemon";
 162         website = "https://www.naemon.io/";
 163         license.url = "https://github.com/naemon/naemon-core/blob/master/COPYING";
 164         license.name = "GNU General Public License v2.0";
 165         version = config.services.naemon.package.version;
 166         source.url = "https://github.com/naemon/naemon-core";
 167         modules = "livestatus,status-engine";
 168       };
 169     };
 170     services.nginx = lib.mkIf config.myServices.status.enable {
 171       virtualHosts."status.immae.eu".locations = objectsFiles // {
 172         "=/common/immae.cfg" = {
 173           alias = pkgs.writeText "immae.cfg" ''
 174             # put me for instance in /etc/naemon/module-conf.d/immae.cfg
 175             # Make sure that you have include_dir=module-conf.d in
 176             # naemon.cfg
 177             log_initial_states=1
 178             date_format=iso8601
 179             admin_email=${config.myEnv.monitoring.email}
 180             obsess_over_services=1
 181             ocsp_command=notify-master
 182           '';
 183         };
 184         "=/common/resource.cfg" = {
 185           alias = pkgs.writeText "resource.cfg" ''
 186             # Resource.cfg file
 187             # Replace this with path to monitoring plugins
 188             $USER1$=@@COMMON_PLUGINS@@
 189             # Replace this with a path to scripts from
 190             # https://git.immae.eu/cgit/perso/Immae/Config/Nix.git/tree/modules/private/monitoring/plugins
 191             $USER2$=@@IMMAE_PLUGINS@@
 192             $USER200$=https://status.immae.eu/
 193             $USER201$=@@TOKEN@@
 194           '';
 195         };
 196       };
 197     };
 198
 199     security.sudo.extraRules = let
 200       pluginsSudo = lib.lists.remove null (lib.attrsets.mapAttrsToList (k: v:
 201         if (v ? sudo)
 202         then ({ users = [ "naemon" ]; } // (v.sudo myplugins))
 203         else null) mypluginsConfig);
 204     in [
 205       {
 206         commands = [
 207           { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
 208           { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
 209         ];
 210         users = [ "naemon" ];
 211         runAs = "root";
 212       }
 213     ] ++ pluginsSudo;
 214     environment.etc."mdadm.conf" = {
 215       enable = true;
 216       mode = "0644";
 217       user = "root";
 218       text = "MAILADDR ${config.myEnv.monitoring.email}";
 219     };
 220
 221     secrets.keys = {
 222       "naemon/id_rsa" = {
 223         user = "naemon";
 224         group = "naemon";
 225         permissions = "0400";
 226         text = config.myEnv.monitoring.ssh_secret_key;
 227       };
 228     } // lib.optionalAttrs cfg.master (
 229       lib.mapAttrs' (k: v: lib.nameValuePair "${k}_access_key" {
 230         user = "naemon";
 231         group = "naemon";
 232         permissions = "0400";
 233         text = ''
 234           export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
 235           export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
 236           export BASE_URL="${v.remote "immae-eldiron"}"
 237         '';
 238       }) config.myEnv.backup.remotes
 239     );
 240     # needed since extraResource is not in the closure
 241     systemd.services.naemon.path = [ myplugins ];
 242     services.naemon = {
 243       enable = true;
 244       extraConfig = ''
 245         use_syslog=1
 246         log_initial_states=1
 247         date_format=iso8601
 248         admin_email=${config.myEnv.monitoring.email}
 249       '' + lib.optionalString (!cfg.master) ''
 250         obsess_over_services=1
 251         ocsp_command=notify-master
 252       '' + lib.optionalString (cfg.master) ''
 253         broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
 254         broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
 255       '';
 256       extraResource = let
 257         resources = [hostObjects.resources or {}] ++ (lib.mapAttrsToList (k: v: v.resources or {}) mypluginsConfig);
 258         joined = lib.zipAttrsWith (n: v: if builtins.length (lib.unique v) == 1 then builtins.head v else abort "Non-unique resources names") resources;
 259         joinedStr = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "$" + "${k}$=${v}") joined);
 260       in ''
 261         $USER2$=${myplugins}
 262         ${joinedStr}
 263       '';
 264       objectDefs = toObjects commonObjects
 265         + toObjects hostObjects
 266         + lib.optionalString cfg.master (toObjects masterObjects)
 267         + lib.optionalString cfg.master (toObjects masterPassiveObjects)
 268         + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
 269     };
 270   };
 271 }