path: root/flakes/private/monitoring/flake.nix

{
  inputs = {
    environment.url = "path:../environment";
    secrets.url = "path:../../secrets";
    naemon.url = "path:../../naemon";
    nixpkgs-lib.url = "github:NixOS/nixpkgs?dir=lib";
  };
  outputs = { self, environment, nixpkgs-lib, secrets, naemon }: {
    nagios-cli-config = ./nagios-cli.cfg;
    lib = rec {
      expandedObject = kind: object: objects:
        if object ? "use"
        then expandedObject kind objects.templates.${kind}.${object.use} objects // object
        else object;

      objectsCommon = import ./objects_common.nix;
      toObjects = import ./to_objects.nix { inherit (nixpkgs-lib) lib; };

      toMasterPassiveObject = svcTemplate: freshnessThresholdMultiplier: objects:
        {
          service = with nixpkgs-lib.lib; map (s:
            {
              host_name = (expandedObject "service" s objects).host_name;
              use = svcTemplate;
              retry_interval = "1";
              freshness_threshold = let
                fs = expandedObject "service" s objects;
              in if builtins.isInt fs.check_interval
                then builtins.ceil (freshnessThresholdMultiplier * 60 * fs.check_interval)
                else fs.check_interval;
            }
            // filterAttrs (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_") s
            // mapAttrs'
                (n: nameValuePair (removePrefix "__passive_" n))
                (filterAttrs (k: _: hasPrefix "__passive_" k) s)
          ) objects.service;
          host = objects.host;
        };

      emailCheck = allCfg: host: hostFQDN: let
        cfg = allCfg."${host}";
        reverseTargets = builtins.attrNames (nixpkgs-lib.lib.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
        to_email = cfg': host':
          let sep = if nixpkgs-lib.lib.hasInfix "+" cfg'.mail_address then "_" else "+";
          in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
        mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
        mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
        command = if cfg.local
        then
          [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
        else
          [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
      in
        {
          service_description = "${hostFQDN} email service is active";
          use = "mail-service";
          host_name = hostFQDN;
          servicegroups = "webstatus-email";
          check_command = command;
        };
    };
    nixosModule = self.nixosModules.monitoring;
    nixosModules.monitoring = { config, pkgs, lib, ... }:
      let
        cfg = config.myServices.monitoring;
        allPluginsConfig = import ./myplugins.nix {
          inherit pkgs lib config;
          sudo = "/run/wrappers/bin/sudo";
        };
        mypluginsConfig = lib.mapAttrs (n: v:
          if builtins.isFunction v
          then v (cfg.pluginsArgs."${n}" or {})
          else v
        ) (lib.getAttrs cfg.activatedPlugins allPluginsConfig);
        myplugins = let
          mypluginsChunk = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: v.chunk or "") mypluginsConfig);
        in pkgs.runCommand "buildplugins" {
          buildInputs = [ pkgs.makeWrapper pkgs.perl ];
        } ''
          mkdir $out
          ${mypluginsChunk}
          '';
        objectsModule = with lib.types; submodule {
          options = {
            command = lib.mkOption {
              type = attrsOf str;
              default = {};
              description = "Command definitions";
            };

            host = lib.mkOption {
              type = attrsOf (attrsOf str);
              default = {};
              description = "Host definitions";
            };
            hostgroup = lib.mkOption {
              type = attrsOf (attrsOf str);
              default = {};
              description = "Host group definitions";
            };
            hostdependency = lib.mkOption {
              type = listOf (attrsOf str);
              default = [];
              description = "Host dependency definitions";
            };

            service = lib.mkOption {
              type = listOf (attrsOf (oneOf [ str (listOf str) int ]));
              # str -> string
              # listOf str -> list to be concatenated with "!"
              # int -> toString
              default = [];
              description = "Service definitions";
            };
            servicegroup = lib.mkOption {
              type = attrsOf (attrsOf str);
              default = {};
              description = "Service group definitions";
            };
            servicedependency = lib.mkOption {
              type = listOf (attrsOf str);
              default = [];
              description = "Service dependency definitions";
            };

            contact = lib.mkOption {
              type = attrsOf (attrsOf str);
              default = {};
              description = "Contact definitions";
            };
            contactgroup = lib.mkOption {
              type = attrsOf (attrsOf str);
              default = {};
              description = "Contact group definitions";
            };

            timeperiod = lib.mkOption {
              type = attrsOf (attrsOf str);
              default = {};
              description = "Time period definitions";
            };

            templates = lib.mkOption {
              description = "Template definitions";
              default = {};
              type = submodule {
                options = {
                  service = lib.mkOption { type = attrsOf (attrsOf (either str int)); default = {}; };
                  contact = lib.mkOption { type = attrsOf (attrsOf str); default = {}; };
                  host = lib.mkOption { type = attrsOf (attrsOf str); default = {}; };
                };
              };
            };
          };
        };
      in
      {
        options = {
          myServices.monitoring = {
            enable = lib.mkOption {
              type = lib.types.bool;
              default = false;
              description = ''
                Whether to enable monitoring.
              '';
            };
            smartdDisks = lib.mkOption {
              type = lib.types.listOf lib.types.str;
              default = [];
              description = ''
                List of smartd disks ids (symlinks in /dev/disk/by-id/) to monitor
              '';
            };
            master = lib.mkOption {
              type = lib.types.bool;
              default = false;
              description = ''
                This instance is the master instance
              '';
            };
            pluginsArgs = lib.mkOption {
              default = {};
              description = "Arguments to pass to the naemon plugin configuration";
              type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified);
            };
            activatedPlugins = lib.mkOption {
              default = [];
              description = "List of naemon plugins to activate";
              type = lib.types.listOf (lib.types.enum (builtins.attrNames allPluginsConfig));
            };
            fromMasterActivatedPlugins = lib.mkOption {
              default = [];
              description = "List of naemon plugins to activate from master";
              type = lib.types.listOf (lib.types.str);
            };
            resources = lib.mkOption {
              default = {};
              description = "List of additionnal resources elements";
              type = lib.types.attrsOf (lib.types.str);
            };
            objects = lib.mkOption {
              default = {};
              description = "Object definitions";
              type = objectsModule;
            };
            fromMasterObjects = lib.mkOption {
              default = {};
              description = "Object definitions of checks that should be executed from master";
              type = objectsModule;
            };
          };
        };

        imports = [
          environment.nixosModule
          secrets.nixosModule
          naemon.nixosModule
        ];
        config = lib.mkIf cfg.enable {
          myServices.monitoring.objects.command =
            lib.foldr (v: o: o // (v.commands or {})) {} (builtins.attrValues mypluginsConfig);

          security.sudo.extraRules = let
            pluginsSudo = lib.lists.remove null (lib.mapAttrsToList (k: v:
              if (v ? sudo)
              then ({ users = [ "naemon" ]; } // (v.sudo myplugins))
              else null) mypluginsConfig);
          in pluginsSudo;

          environment.etc.cnagios.source = "${pkgs.cnagios}/share/doc/cnagios";
          environment.systemPackages = let
            nagios-cli = pkgs.writeScriptBin "nagios-cli" ''
              #!${pkgs.stdenv.shell}
              sudo -u naemon ${pkgs.nagios-cli}/bin/nagios-cli -c ${self.nagios-cli-config}
              '';
          in [
            pkgs.cnagios
            nagios-cli
          ];
          secrets.keys = {
            "naemon/id_rsa" = {
              user = "naemon";
              group = "naemon";
              permissions = "0400";
              text = config.myEnv.monitoring.ssh_secret_key;
            };
            "naemon/resources.cfg".keyDependencies = [ myplugins ];
          };
          services.naemon = {
            enable = true;
            extraConfig = ''
              use_syslog=1
              log_initial_states=1
              date_format=iso8601
              admin_email=${config.myEnv.monitoring.email}
            '' + lib.optionalString (!cfg.master) ''
              obsess_over_services=1
              ocsp_command=notify-master
            '';
            extraResource = let
              resources = [cfg.resources or {}] ++ (lib.mapAttrsToList (k: v: v.resources or {}) mypluginsConfig);
              joined = lib.zipAttrsWith (n: v: if builtins.length (lib.unique v) == 1 then builtins.head v else abort "Non-unique resources names") resources;
              joinedStr = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "$" + "${k}$=${v}") joined);
            in ''
              $USER2$=${myplugins}
              ${joinedStr}
            '';
            objectDefs =
              self.lib.toObjects cfg.objects;
          };

          myServices.monitoring.objects.service = builtins.map (d: {
            service_description = "Disk /dev/disk/by-id/${d} is sane";
            use = "local-service";
            check_command = [ "check_smartctl" "/dev/disk/by-id/${d}" ];
            __passive_servicegroups = "webstatus-resources";

            check_interval = 60;
          }) cfg.smartdDisks;

          systemd = let
            checkShortTimer = {
              timerConfig = {
                OnCalendar = "monthly";
                RandomizedDelaySec = "3 weeks";
                FixedRandomDelay = true;
              };
              wantedBy = [ "timers.target" ];
            };
            checkLongTimer = {
              timerConfig = {
                OnCalendar = "monthly";
                RandomizedDelaySec = "3 weeks";
                FixedRandomDelay = true;
              };
              wantedBy = [ "timers.target" ];
            };
            toSDTimers = id: {
              "check-smartd-long-${id}" = checkLongTimer;
              "check-smartd-short-${id}" = checkShortTimer;
            };
            toCheckService = id: type: {
              description = "Run ${type} smartctl test for /dev/disk/by-id/${id}";
              after = [ "multi-user.target" ];
              serviceConfig = {
                Type = "oneshot";
                ExecStart = "${pkgs.smartmontools}/bin/smartctl -t ${type} /dev/disk/by-id/${id}";
              };
            };
            toSDServices = id: {
              "check-smartd-long-${id}" = toCheckService id "long";
              "check-smartd-short-${id}" = toCheckService id "short";
            };

          in {
            services = lib.attrsets.mergeAttrsList (builtins.map toSDServices cfg.smartdDisks);
            timers = lib.attrsets.mergeAttrsList (builtins.map toSDTimers cfg.smartdDisks);
          };
        };
      };
  };
}