X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=modules%2Fprivate%2Fmonitoring%2Fdefault.nix;h=f00fb7c33c38a09f57e0f1a415d10c86006dcbb0;hb=acab8301f6a0a76d97b36d009a60d4172bf62981;hp=e44b127488e3691090721c9d856dbde2534af985;hpb=16b80abd57bb215d0e72f3983f997a007743b8fb;p=perso%2FImmae%2FConfig%2FNix.git diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix index e44b127..f00fb7c 100644 --- a/modules/private/monitoring/default.nix +++ b/modules/private/monitoring/default.nix @@ -1,51 +1,118 @@ -{ config, myconfig, pkgs, lib, name, hostFQDN, ... }: +{ config, pkgs, lib, name, nodes, ... }: let - myplugins = pkgs.runCommand "buildplugins" { + cfg = config.myServices.monitoring; + activatedPlugins = [ "memory" "command" "bandwidth" ] + ++ (if cfg.master then (masterObjects.activatedPlugins or []) else []) + ++ (if cfg.master then (lib.flatten (map (v: v.activatedPlugins or []) otherObjects)) else []) + ++ (hostObjects.activatedPlugins or []) + ++ (if cfg.master then ["notify-primary"] else ["notify-secondary"]); + allPluginsConfig = import ./myplugins.nix { + inherit pkgs lib config; + sudo = "/run/wrappers/bin/sudo"; + }; + mypluginsConfig = lib.getAttrs activatedPlugins allPluginsConfig; + myplugins = let + mypluginsChunk = builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList (k: v: v.chunk or "") mypluginsConfig); + in pkgs.runCommand "buildplugins" { buildInputs = [ pkgs.makeWrapper pkgs.perl ]; } '' mkdir $out - cp ${./plugins}/* $out/ - patchShebangs $out - wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir} - wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [ - pkgs.curl pkgs.jq - ]} - wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [ - pkgs.gnugrep pkgs.gawk pkgs.procps-ng - ]} - wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [ - pkgs.postgresql - ]} - wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [ - pkgs.gnugrep pkgs.coreutils pkgs.redis - ]} - wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [ - pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb - ]} - wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [ - pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap - ]} + ${mypluginsChunk} ''; toObjects = pkgs.callPackage ./to_objects.nix {}; commonConfig = { + dilion = { + processWarn = "250"; processAlert = "400"; + loadWarn = "1.0"; loadAlert = "1.2"; + interface = "eth0"; + }; eldiron = { processWarn = "250"; processAlert = "400"; - loadWarn = "8.0"; loadAlert = "10.0"; + loadWarn = "1.0"; loadAlert = "1.2"; + interface = "eth0"; }; backup-2 = { - processWarn = "50"; processAlert = "60"; + processWarn = "60"; processAlert = "70"; loadWarn = "1.0"; loadAlert = "2.0"; + interface = "ens3"; + }; + monitoring-1 = { + processWarn = "50"; processAlert = "60"; + loadWarn = "4.0"; loadAlert = "6.0"; + load15Warn = "1.0"; load15Alert = "2.0"; + interface = "ens3"; + }; + quatresaisons = { + processWarn = "250"; processAlert = "400"; + loadWarn = "1.0"; loadAlert = "1.2"; + interface = "eth0"; }; }; + externalObjects = lib.genAttrs [ "tiboqorl-fr" ] + (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }); + masterPassiveObjects = let + passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes; + toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo); + passiveServices = lib.flatten (lib.attrsets.mapAttrsToList + (_: n: toPassiveServices n.config.myServices.monitoring.services) + passiveNodes + ) ++ lib.flatten (lib.attrsets.mapAttrsToList + (_: n: toPassiveServices n.service) + externalObjects); + in { + service = passiveServices; + host = lib.lists.foldr + (a: b: a//b) + {} + (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes + ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects); + }; + emailCheck = host: hostFQDN: let + allCfg = config.myEnv.monitoring.email_check; + cfg = allCfg."${host}"; + reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg); + to_email = cfg': host': + let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+"; + in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}"; + mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets); + mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets); + command = if cfg.local + then + [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ] + else + [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ]; + in + { + service_description = "${hostFQDN} email service is active"; + use = "mail-service"; + host_name = hostFQDN; + servicegroups = "webstatus-email"; + check_command = command; + }; + otherObjects = map + (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; })) + [ "ulminfo-fr" "phare" "eban" ]; + masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; }; commonObjects = pkgs.callPackage ./objects_common.nix ({ - inherit hostFQDN; - sudo = "/run/wrappers/bin/sudo"; + master = cfg.master; + hostFQDN = config.hostEnv.fqdn; + hostName = name; + inherit mypluginsConfig; } // builtins.getAttr name commonConfig); hostObjects = let specific_file = ./. + "/objects_" + name + ".nix"; in - lib.attrsets.optionalAttrs (builtins.pathExists specific_file) (pkgs.callPackage specific_file { inherit config; }); + lib.attrsets.optionalAttrs + (builtins.pathExists specific_file) + (pkgs.callPackage specific_file { + inherit config nodes emailCheck; + hostFQDN = config.hostEnv.fqdn; + hostName = name; + }); + objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair + "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); } + ) externalObjects; in { options = { @@ -57,14 +124,65 @@ in Whether to enable monitoring. ''; }; + master = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + This instance is the master instance + ''; + }; + hosts = lib.mkOption { + readOnly = true; + description = "Hosts list for this host"; + default = (commonObjects.host or {}) // (hostObjects.host or {}); + }; + services = lib.mkOption { + readOnly = true; + description = "Services list for this host"; + default = commonObjects.service ++ hostObjects.service; + }; }; }; - config = lib.mkIf config.myServices.monitoring.enable { + config = lib.mkIf cfg.enable { + services.nginx = lib.mkIf config.myServices.status.enable { + virtualHosts."status.immae.eu".locations = objectsFiles // { + "=/common/immae.cfg" = { + alias = pkgs.writeText "immae.cfg" '' + # put me for instance in /etc/naemon/module-conf.d/immae.cfg + # Make sure that you have include_dir=module-conf.d in + # naemon.cfg + log_initial_states=1 + date_format=iso8601 + admin_email=${config.myEnv.monitoring.email} + obsess_over_services=1 + ocsp_command=notify-master + ''; + }; + "=/common/resource.cfg" = { + alias = pkgs.writeText "resource.cfg" '' + # Resource.cfg file + # Replace this with path to monitoring plugins + $USER1$=@@COMMON_PLUGINS@@ + # Replace this with a path to scripts from + # https://git.immae.eu/cgit/perso/Immae/Config/Nix.git/tree/modules/private/monitoring/plugins + $USER2$=@@IMMAE_PLUGINS@@ + $USER200$=https://status.immae.eu/ + $USER201$=@@TOKEN@@ + ''; + }; + }; + }; + services.duplyBackup.profiles.monitoring = { rootDir = config.services.naemon.varDir; }; - security.sudo.extraRules = [ + security.sudo.extraRules = let + pluginsSudo = lib.lists.remove null (lib.attrsets.mapAttrsToList (k: v: + if (v ? sudo) + then ({ users = [ "naemon" ]; } // (v.sudo myplugins)) + else null) mypluginsConfig); + in [ { commands = [ { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; } @@ -73,69 +191,65 @@ in users = [ "naemon" ]; runAs = "root"; } - { - commands = [ - { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "ALL"; - } - { - commands = [ - { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "postgres"; - } - { - commands = [ - { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "mysql"; - } - { - commands = [ - { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "openldap"; - } - { - commands = [ - { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "redis"; - } - ]; + ] ++ pluginsSudo; environment.etc."mdadm.conf" = { enable = true; mode = "0644"; user = "root"; - text = "MAILADDR ${myconfig.env.monitoring.email}"; + text = "MAILADDR ${config.myEnv.monitoring.email}"; }; + secrets.keys = [ + { + dest = "naemon/id_rsa"; + user = "naemon"; + group = "naemon"; + permissions = "0400"; + text = config.myEnv.monitoring.ssh_secret_key; + } + ] ++ lib.optionals cfg.master ( + lib.mapAttrsToList (k: v: + { + dest = "${k}_access_key"; + user = "naemon"; + group = "naemon"; + permissions = "0400"; + text = '' + export AWS_ACCESS_KEY_ID="${v.accessKeyId}" + export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}" + export BASE_URL="${v.remote "immae-eldiron"}" + ''; + }) config.myEnv.backup.remotes + ); # needed since extraResource is not in the closure systemd.services.naemon.path = [ myplugins ]; services.naemon = { enable = true; extraConfig = '' - broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live use_syslog=1 log_initial_states=1 date_format=iso8601 - admin_email=${myconfig.env.monitoring.email} - + admin_email=${config.myEnv.monitoring.email} + '' + lib.optionalString (!cfg.master) '' obsess_over_services=1 ocsp_command=notify-master + '' + lib.optionalString (cfg.master) '' + broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live + broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0 ''; - extraResource = '' + extraResource = let + resources = lib.mapAttrsToList (k: v: v.resources or {}) mypluginsConfig; + joined = lib.zipAttrsWith (n: v: if builtins.length (lib.unique v) == 1 then builtins.head v else abort "Non-unique resources names") resources; + joinedStr = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "$" + "${k}$=${v}") joined); + in '' $USER2$=${myplugins} - $USER200$=${myconfig.env.monitoring.status_url} - $USER201$=${myconfig.env.monitoring.status_token} + ${joinedStr} ''; - objectDefs = toObjects commonObjects + toObjects hostObjects; + objectDefs = toObjects commonObjects + + toObjects hostObjects + + lib.optionalString cfg.master (toObjects masterObjects) + + lib.optionalString cfg.master (toObjects masterPassiveObjects) + + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects)); }; }; }