From: Ismaël Bouya Date: Sat, 13 Apr 2024 10:39:31 +0000 (+0200) Subject: Add monitoring script with smartctl X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=408ce9363bf574796487fedfe59a4a2ff0cbb4cb;p=perso%2FImmae%2FConfig%2FNix.git Add monitoring script with smartctl --- diff --git a/deploy/flake.lock b/deploy/flake.lock index c766b08..d1eba4c 100644 --- a/deploy/flake.lock +++ b/deploy/flake.lock @@ -2783,7 +2783,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-N8FlN9eUGGmz7MevjaGC7Y07AEebJGRzcESmOW87/0Q=", + "narHash": "sha256-W4TIMgSVOiYTGt6hXrMNVxrJ9QZaQrL+awvGLk2UXjU=", "path": "../flakes", "type": "path" }, @@ -2894,7 +2894,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2912,7 +2912,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2930,7 +2930,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2948,7 +2948,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -3832,7 +3832,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-B2JdIp5q/8/LurN4lXB17C1+mpEKeqPzObmNOObO5rQ=", + "narHash": "sha256-KL5aC6l/mM+ucHadjCtlSY6iBZ9/VKRf2Rgw2tjwfqo=", "path": "../systems/backup-2", "type": "path" }, @@ -3855,7 +3855,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-dWAbGxB91utxwTlxsJZdp2OSaIAo8y5LXjdTCFMRn+4=", + "narHash": "sha256-VliQ62boc8jNgPCBX6JjVS/pA6YoikZVkvE1xxHAITk=", "path": "../systems/dilion", "type": "path" }, @@ -3903,7 +3903,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-OvT4dJGxdFBUgwjTu6FzejI/euWYXWsX1vD952B8934=", + "narHash": "sha256-YVJi4KPT8rsGNzZshYfUyz9l27QGAEsb8Iyu3O0/bMA=", "path": "../systems/eldiron", "type": "path" }, @@ -3929,7 +3929,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-6hqQjoy1lyIJziqe0AJ62aq62soi5D14NKOzNEY3SsQ=", + "narHash": "sha256-HiDwzScPHp084MwtWET+17WbrWf3zs9qJzv/EqNaSqM=", "path": "../systems/monitoring-1", "type": "path" }, @@ -3954,7 +3954,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-yBuAfW59X0e840QBFtQrefG3T/IQIQSXiZ4WU2/+gBI=", + "narHash": "sha256-YWEuXALLICHE+WsMtMuiCm6b2IRQyJrFpfMgH6ZLD0A=", "path": "../systems/quatresaisons", "type": "path" }, @@ -7541,7 +7541,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -8412,7 +8412,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "./private/monitoring", "type": "path" }, diff --git a/flake.lock b/flake.lock index f22d393..5485356 100644 --- a/flake.lock +++ b/flake.lock @@ -2664,7 +2664,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-N8FlN9eUGGmz7MevjaGC7Y07AEebJGRzcESmOW87/0Q=", + "narHash": "sha256-W4TIMgSVOiYTGt6hXrMNVxrJ9QZaQrL+awvGLk2UXjU=", "path": "./flakes", "type": "path" }, @@ -2910,7 +2910,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2928,7 +2928,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2946,7 +2946,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2964,7 +2964,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -3848,7 +3848,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-B2JdIp5q/8/LurN4lXB17C1+mpEKeqPzObmNOObO5rQ=", + "narHash": "sha256-KL5aC6l/mM+ucHadjCtlSY6iBZ9/VKRf2Rgw2tjwfqo=", "path": "../systems/backup-2", "type": "path" }, @@ -3871,7 +3871,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-dWAbGxB91utxwTlxsJZdp2OSaIAo8y5LXjdTCFMRn+4=", + "narHash": "sha256-VliQ62boc8jNgPCBX6JjVS/pA6YoikZVkvE1xxHAITk=", "path": "../systems/dilion", "type": "path" }, @@ -3919,7 +3919,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-OvT4dJGxdFBUgwjTu6FzejI/euWYXWsX1vD952B8934=", + "narHash": "sha256-YVJi4KPT8rsGNzZshYfUyz9l27QGAEsb8Iyu3O0/bMA=", "path": "../systems/eldiron", "type": "path" }, @@ -3945,7 +3945,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-6hqQjoy1lyIJziqe0AJ62aq62soi5D14NKOzNEY3SsQ=", + "narHash": "sha256-HiDwzScPHp084MwtWET+17WbrWf3zs9qJzv/EqNaSqM=", "path": "../systems/monitoring-1", "type": "path" }, @@ -3970,7 +3970,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-yBuAfW59X0e840QBFtQrefG3T/IQIQSXiZ4WU2/+gBI=", + "narHash": "sha256-YWEuXALLICHE+WsMtMuiCm6b2IRQyJrFpfMgH6ZLD0A=", "path": "../systems/quatresaisons", "type": "path" }, @@ -7557,7 +7557,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -8428,7 +8428,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "./private/monitoring", "type": "path" }, diff --git a/flakes/flake.lock b/flakes/flake.lock index 17264a4..85690fa 100644 --- a/flakes/flake.lock +++ b/flakes/flake.lock @@ -2815,7 +2815,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2833,7 +2833,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2851,7 +2851,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2869,7 +2869,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -3753,7 +3753,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-B2JdIp5q/8/LurN4lXB17C1+mpEKeqPzObmNOObO5rQ=", + "narHash": "sha256-KL5aC6l/mM+ucHadjCtlSY6iBZ9/VKRf2Rgw2tjwfqo=", "path": "../systems/backup-2", "type": "path" }, @@ -3776,7 +3776,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-dWAbGxB91utxwTlxsJZdp2OSaIAo8y5LXjdTCFMRn+4=", + "narHash": "sha256-VliQ62boc8jNgPCBX6JjVS/pA6YoikZVkvE1xxHAITk=", "path": "../systems/dilion", "type": "path" }, @@ -3824,7 +3824,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-OvT4dJGxdFBUgwjTu6FzejI/euWYXWsX1vD952B8934=", + "narHash": "sha256-YVJi4KPT8rsGNzZshYfUyz9l27QGAEsb8Iyu3O0/bMA=", "path": "../systems/eldiron", "type": "path" }, @@ -3850,7 +3850,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-6hqQjoy1lyIJziqe0AJ62aq62soi5D14NKOzNEY3SsQ=", + "narHash": "sha256-HiDwzScPHp084MwtWET+17WbrWf3zs9qJzv/EqNaSqM=", "path": "../systems/monitoring-1", "type": "path" }, @@ -3875,7 +3875,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-yBuAfW59X0e840QBFtQrefG3T/IQIQSXiZ4WU2/+gBI=", + "narHash": "sha256-YWEuXALLICHE+WsMtMuiCm6b2IRQyJrFpfMgH6ZLD0A=", "path": "../systems/quatresaisons", "type": "path" }, @@ -7384,7 +7384,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -8294,7 +8294,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "./private/monitoring", "type": "path" }, diff --git a/flakes/private/monitoring/myplugins.nix b/flakes/private/monitoring/myplugins.nix index f76f2c1..bb3a383 100644 --- a/flakes/private/monitoring/myplugins.nix +++ b/flakes/private/monitoring/myplugins.nix @@ -284,6 +284,30 @@ in ]} ''; }; + smartctl = { + commands = { + check_smartctl = "$USER2$/check_smartctl -i auto -d $ARG1$"; + }; + chunk = let + smartPlugin = pkgs.fetchurl { + url = "https://www.claudiokuenzler.com/monitoring-plugins/check_smart.pl"; + sha256 = "sha256-gxGkzyycUl+I3WziKhOnZSoQjpqbPqjbunfUQxmeb7w="; + }; + in '' + cp ${smartPlugin} $out/check_smartctl + chmod +x $out/check_smartctl + patchShebangs $out + substituteInPlace $out/check_smartctl --replace "/usr/bin /bin /usr/sbin /sbin /usr/local/bin /usr/local/sbin" "${pkgs.smartmontools}/bin" + substituteInPlace $out/check_smartctl --replace "sudo" "${sudo}" + ''; + + sudo = _: { + commands = [ + { command = "${pkgs.smartmontools}/bin/smartctl *"; options = [ "NOPASSWD" ]; } + ]; + runAs = "root"; + }; + }; mysql = { commands = { check_mysql_replication = "${sudo} -u mysql $USER2$/check_mysql_replication \"$ARG1$\" \"$ARG2$\""; diff --git a/systems/backup-2/flake.lock b/systems/backup-2/flake.lock index a8d55bf..d6e56a5 100644 --- a/systems/backup-2/flake.lock +++ b/systems/backup-2/flake.lock @@ -389,7 +389,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, diff --git a/systems/dilion/flake.lock b/systems/dilion/flake.lock index 3096384..a690504 100644 --- a/systems/dilion/flake.lock +++ b/systems/dilion/flake.lock @@ -207,7 +207,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, diff --git a/systems/dilion/monitoring.nix b/systems/dilion/monitoring.nix index af7f224..5c4e5b0 100644 --- a/systems/dilion/monitoring.nix +++ b/systems/dilion/monitoring.nix @@ -9,7 +9,7 @@ let }; in { - config.myServices.monitoring.activatedPlugins = [ "memory" "command" "bandwidth" "mdadm" "zfs" "notify-secondary"]; + config.myServices.monitoring.activatedPlugins = [ "memory" "command" "bandwidth" "mdadm" "zfs" "notify-secondary" "smartctl" ]; config.myServices.monitoring.objects = lib.mkMerge [ (monitoring.lib.objectsCommon { inherit hostFQDN; @@ -22,6 +22,38 @@ in { service = [ + { + service_description = "Disk /dev/disk/by-id/ata-Micron_1100_MTFDDAK512TBN_171216BD2A4B is sane"; + use = "local-service"; + check_command = [ "check_smartctl" "/dev/disk/by-id/ata-Micron_1100_MTFDDAK512TBN_171216BD2A4B" ]; + __passive_servicegroups = "webstatus-resources"; + + check_interval = 60; + } + { + service_description = "Disk /dev/disk/by-id/ata-Micron_1100_MTFDDAK512TBN_171216BD2A70 is sane"; + use = "local-service"; + check_command = [ "check_smartctl" "/dev/disk/by-id/ata-Micron_1100_MTFDDAK512TBN_171216BD2A70" ]; + __passive_servicegroups = "webstatus-resources"; + + check_interval = 60; + } + { + service_description = "Disk /dev/disk/by-id/ata-TOSHIBA_MG04ACA400N_96K1K87YFVLC is sane"; + use = "local-service"; + check_command = [ "check_smartctl" "/dev/disk/by-id/ata-TOSHIBA_MG04ACA400N_96K1K87YFVLC" ]; + __passive_servicegroups = "webstatus-resources"; + + check_interval = 60; + } + { + service_description = "Disk /dev/disk/by-id/ata-WDC_WD3000FYYZ-01UL1B2_WD-WMC1F0E4X6WP is sane"; + use = "local-service"; + check_command = [ "check_smartctl" "/dev/disk/by-id/ata-WDC_WD3000FYYZ-01UL1B2_WD-WMC1F0E4X6WP" ]; + __passive_servicegroups = "webstatus-resources"; + + check_interval = 60; + } { service_description = "No mdadm array is degraded"; use = "local-service"; diff --git a/systems/eldiron/flake.lock b/systems/eldiron/flake.lock index f5b9184..461bebb 100644 --- a/systems/eldiron/flake.lock +++ b/systems/eldiron/flake.lock @@ -2038,7 +2038,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, diff --git a/systems/eldiron/monitoring.nix b/systems/eldiron/monitoring.nix index 2aa7f8f..92da7cf 100644 --- a/systems/eldiron/monitoring.nix +++ b/systems/eldiron/monitoring.nix @@ -4,7 +4,7 @@ let emailCheck = monitoring.lib.emailCheck config.myEnv.monitoring.email_check; in { - config.myServices.monitoring.activatedPlugins = [ "memory" "command" "bandwidth" "emails" "mdadm" "postfix" "postgresql" "zfs" "notify-secondary"]; + config.myServices.monitoring.activatedPlugins = [ "memory" "command" "bandwidth" "emails" "mdadm" "postfix" "postgresql" "zfs" "notify-secondary" "smartctl" ]; config.myServices.monitoring.pluginsArgs.postgresql.package = config.myServices.databases.postgresql.package; config.myServices.monitoring.objects = lib.mkMerge [ (monitoring.lib.objectsCommon { @@ -18,6 +18,38 @@ in { service = [ + { + service_description = "Disk /dev/disk/by-id/ata-ST33000650NS_Z29540DM is sane"; + use = "local-service"; + check_command = [ "check_smartctl" "/dev/disk/by-id/ata-ST33000650NS_Z29540DM" ]; + __passive_servicegroups = "webstatus-resources"; + + check_interval = 60; + } + { + service_description = "Disk /dev/disk/by-id/ata-ST33000650NS_Z296JGJ4 is sane"; + use = "local-service"; + check_command = [ "check_smartctl" "/dev/disk/by-id/ata-ST33000650NS_Z296JGJ4" ]; + __passive_servicegroups = "webstatus-resources"; + + check_interval = 60; + } + { + service_description = "Disk /dev/disk/by-id/ata-SAMSUNG_MZ7LM480HMHQ-00005_S2UJNX0HA05895 is sane"; + use = "local-service"; + check_command = [ "check_smartctl" "/dev/disk/by-id/ata-SAMSUNG_MZ7LM480HMHQ-00005_S2UJNX0HA05895" ]; + __passive_servicegroups = "webstatus-resources"; + + check_interval = 60; + } + { + service_description = "Disk /dev/disk/by-id/ata-Micron_5200_MTFDDAK480TDC_18011BEE03B3 is sane"; + use = "local-service"; + check_command = [ "check_smartctl" "/dev/disk/by-id/ata-Micron_5200_MTFDDAK480TDC_18011BEE03B3" ]; + __passive_servicegroups = "webstatus-resources"; + + check_interval = 60; + } { service_description = "No mdadm array is degraded"; use = "local-service"; diff --git a/systems/monitoring-1/flake.lock b/systems/monitoring-1/flake.lock index ed711f4..643683a 100644 --- a/systems/monitoring-1/flake.lock +++ b/systems/monitoring-1/flake.lock @@ -277,7 +277,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" }, diff --git a/systems/quatresaisons/flake.lock b/systems/quatresaisons/flake.lock index a76103c..afb007b 100644 --- a/systems/quatresaisons/flake.lock +++ b/systems/quatresaisons/flake.lock @@ -239,7 +239,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=", + "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=", "path": "../../flakes/private/monitoring", "type": "path" },