From 6ee7783682e977f3ceebfb5a8ee8ddd1db3f7cc6 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Sun, 27 Dec 2020 15:36:09 +0100 Subject: [PATCH] Add monitoring for dilion and quatresaisons --- modules/private/monitoring/default.nix | 20 ++++++++++++++++- modules/private/monitoring/objects_common.nix | 1 + modules/private/monitoring/objects_dilion.nix | 22 +++++++++++++++++++ .../private/monitoring/objects_eldiron.nix | 6 +++++ .../monitoring/objects_monitoring-1.nix | 18 +++++++-------- .../monitoring/objects_quatresaisons.nix | 16 ++++++++++++++ modules/private/system/dilion.nix | 3 +++ modules/private/system/quatresaisons.nix | 1 + 8 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 modules/private/monitoring/objects_dilion.nix create mode 100644 modules/private/monitoring/objects_quatresaisons.nix diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix index 344e369..a298f92 100644 --- a/modules/private/monitoring/default.nix +++ b/modules/private/monitoring/default.nix @@ -16,13 +16,21 @@ let else if config.myServices.databases.enable then config.myServices.databases.postgresql.package else pkgs.postgresql; + zfsPlugin = pkgs.fetchurl { + url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh"; + sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf"; + }; myplugins = pkgs.runCommand "buildplugins" { buildInputs = [ pkgs.makeWrapper pkgs.perl ]; } '' mkdir $out + cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh cp ${./plugins}/* $out/ patchShebangs $out wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir} + wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [ + pkgs.which pkgs.zfs pkgs.gawk + ]} wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [ pkgs.curl pkgs.jq ]} @@ -79,6 +87,11 @@ let ''; toObjects = pkgs.callPackage ./to_objects.nix {}; commonConfig = { + dilion = { + processWarn = "250"; processAlert = "400"; + loadWarn = "8.0"; loadAlert = "10.0"; + interface = "eth0"; + }; eldiron = { processWarn = "250"; processAlert = "400"; loadWarn = "8.0"; loadAlert = "10.0"; @@ -95,11 +108,16 @@ let load15Warn = "1.0"; load15Alert = "2.0"; interface = "ens3"; }; + quatresaisons = { + processWarn = "250"; processAlert = "400"; + loadWarn = "8.0"; loadAlert = "10.0"; + interface = "eth0"; + }; }; externalObjects = lib.genAttrs [ "tiboqorl-fr" ] (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }); masterPassiveObjects = let - passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron"]) nodes; + passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes; toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo); passiveServices = lib.flatten (lib.attrsets.mapAttrsToList (_: n: toPassiveServices n.config.myServices.monitoring.services) diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix index 82043eb..7f553a0 100644 --- a/modules/private/monitoring/objects_common.nix +++ b/modules/private/monitoring/objects_common.nix @@ -143,6 +143,7 @@ in check_smtp = "$USER1$/check_smtp -H $HOSTADDRESS$ -p 25 -S -D 21,15"; check_tcp = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -e \"$ARG2$\" -Mcrit"; check_tcp_ssl = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -S -D 21,15"; + check_zfs = "$USER2$/check_zpool.sh -p ALL"; check_host_alive = "$USER1$/check_ping -H $HOSTADDRESS$ -w 3000.0,80% -c 5000.0,100% -p 5"; check_last_file_date = "${sudo} -u \"$ARG3$\" $USER2$/check_last_file_date \"$ARG1$\" \"$ARG2$\""; diff --git a/modules/private/monitoring/objects_dilion.nix b/modules/private/monitoring/objects_dilion.nix new file mode 100644 index 0000000..ea4ec37 --- /dev/null +++ b/modules/private/monitoring/objects_dilion.nix @@ -0,0 +1,22 @@ +{ lib, hostFQDN, emailCheck, ... }: +let + defaultPassiveInfo = { + filter = lib.attrsets.filterAttrs + (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_"); + use = "external-passive-service"; + freshness_threshold = "450"; + retry_interval = "1"; + servicegroups = "webstatus-resources"; + host_name = hostFQDN; + }; +in +{ + service = [ + { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-resources"; }; + service_description = "No ZFS pool is degraded"; + use = "local-service"; + check_command = ["check_zfs"]; + } + ]; +} diff --git a/modules/private/monitoring/objects_eldiron.nix b/modules/private/monitoring/objects_eldiron.nix index 92f997f..2c15dd6 100644 --- a/modules/private/monitoring/objects_eldiron.nix +++ b/modules/private/monitoring/objects_eldiron.nix @@ -18,6 +18,12 @@ in use = "local-service"; check_command = ["check_postgresql_replication" "backup-2" "/run/postgresql" "5432"]; } + { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-resources"; }; + service_description = "No ZFS pool is degraded"; + use = "local-service"; + check_command = ["check_zfs"]; + } { passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-email"; }; service_description = "mailq is empty"; diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix index 94b676b..4b784ed 100644 --- a/modules/private/monitoring/objects_monitoring-1.nix +++ b/modules/private/monitoring/objects_monitoring-1.nix @@ -113,17 +113,17 @@ in } # Backup services - { - service_description = "eriomem backup is up and not full"; - host_name = "eldiron.immae.eu"; - use = "external-service"; - check_command = "check_backup_eriomem"; + # { + # service_description = "eriomem backup is up and not full"; + # host_name = "eldiron.immae.eu"; + # use = "external-service"; + # check_command = "check_backup_eriomem"; - check_interval = "120"; - notification_interval = "1440"; + # check_interval = "120"; + # notification_interval = "1440"; - servicegroups = "webstatus-backup"; - } + # servicegroups = "webstatus-backup"; + # } { service_description = "ovh backup is up and not full"; host_name = "eldiron.immae.eu"; diff --git a/modules/private/monitoring/objects_quatresaisons.nix b/modules/private/monitoring/objects_quatresaisons.nix new file mode 100644 index 0000000..3927ef5 --- /dev/null +++ b/modules/private/monitoring/objects_quatresaisons.nix @@ -0,0 +1,16 @@ +{ lib, hostFQDN, emailCheck, ... }: +let + defaultPassiveInfo = { + filter = lib.attrsets.filterAttrs + (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_"); + use = "external-passive-service"; + freshness_threshold = "450"; + retry_interval = "1"; + servicegroups = "webstatus-resources"; + host_name = hostFQDN; + }; +in +{ + service = [ + ]; +} diff --git a/modules/private/system/dilion.nix b/modules/private/system/dilion.nix index 4e2a11f..b89b364 100644 --- a/modules/private/system/dilion.nix +++ b/modules/private/system/dilion.nix @@ -24,6 +24,8 @@ "/boot/pass.key" = "/boot/pass.key"; }; kernel.sysctl."vm.nr_hugepages" = 256; # for xmr-stak + # available in nixos-20.09 + #zfs.requestEncryptionCredentials = [ "zpool/root" ]; }; nix.maxJobs = 8; powerManagement.cpuFreqGovernor = "powersave"; @@ -147,6 +149,7 @@ nix.binaryCaches = [ "https://hydra.iohk.io" "https://cache.nixos.org" ]; nix.binaryCachePublicKeys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" ]; + myServices.monitoring.enable = true; myServices.certificates.enable = true; security.acme.certs."${name}" = { user = config.services.nginx.user; diff --git a/modules/private/system/quatresaisons.nix b/modules/private/system/quatresaisons.nix index 353323f..e905460 100644 --- a/modules/private/system/quatresaisons.nix +++ b/modules/private/system/quatresaisons.nix @@ -274,6 +274,7 @@ in } ]; + myServices.monitoring.enable = true; myServices.certificates.enable = true; users.mutableUsers = true; system.stateVersion = "21.03"; -- 2.41.0