]> git.immae.eu Git - perso/Immae/Config/Nix.git/commitdiff
Add monitoring script with smartctl
authorIsmaël Bouya <ismael.bouya@normalesup.org>
Sat, 13 Apr 2024 10:39:31 +0000 (12:39 +0200)
committerIsmaël Bouya <ismael.bouya@normalesup.org>
Sat, 13 Apr 2024 10:44:20 +0000 (12:44 +0200)
deploy/flake.lock
flake.lock
flakes/flake.lock
flakes/private/monitoring/myplugins.nix
systems/backup-2/flake.lock
systems/dilion/flake.lock
systems/dilion/monitoring.nix
systems/eldiron/flake.lock
systems/eldiron/monitoring.nix
systems/monitoring-1/flake.lock
systems/quatresaisons/flake.lock

index c766b089eb7511cf6851f8beb5693ad5dd830aed..d1eba4ce3be542328aae1aa24607f382929e7d1b 100644 (file)
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-N8FlN9eUGGmz7MevjaGC7Y07AEebJGRzcESmOW87/0Q=",
+        "narHash": "sha256-W4TIMgSVOiYTGt6hXrMNVxrJ9QZaQrL+awvGLk2UXjU=",
         "path": "../flakes",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-B2JdIp5q/8/LurN4lXB17C1+mpEKeqPzObmNOObO5rQ=",
+        "narHash": "sha256-KL5aC6l/mM+ucHadjCtlSY6iBZ9/VKRf2Rgw2tjwfqo=",
         "path": "../systems/backup-2",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-dWAbGxB91utxwTlxsJZdp2OSaIAo8y5LXjdTCFMRn+4=",
+        "narHash": "sha256-VliQ62boc8jNgPCBX6JjVS/pA6YoikZVkvE1xxHAITk=",
         "path": "../systems/dilion",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-OvT4dJGxdFBUgwjTu6FzejI/euWYXWsX1vD952B8934=",
+        "narHash": "sha256-YVJi4KPT8rsGNzZshYfUyz9l27QGAEsb8Iyu3O0/bMA=",
         "path": "../systems/eldiron",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-6hqQjoy1lyIJziqe0AJ62aq62soi5D14NKOzNEY3SsQ=",
+        "narHash": "sha256-HiDwzScPHp084MwtWET+17WbrWf3zs9qJzv/EqNaSqM=",
         "path": "../systems/monitoring-1",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-yBuAfW59X0e840QBFtQrefG3T/IQIQSXiZ4WU2/+gBI=",
+        "narHash": "sha256-YWEuXALLICHE+WsMtMuiCm6b2IRQyJrFpfMgH6ZLD0A=",
         "path": "../systems/quatresaisons",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "./private/monitoring",
         "type": "path"
       },
index f22d39372d88caecb2919f42d92d8ee915abd7cb..5485356215989198b7989d96b9d7f6dbd7af56df 100644 (file)
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-N8FlN9eUGGmz7MevjaGC7Y07AEebJGRzcESmOW87/0Q=",
+        "narHash": "sha256-W4TIMgSVOiYTGt6hXrMNVxrJ9QZaQrL+awvGLk2UXjU=",
         "path": "./flakes",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-B2JdIp5q/8/LurN4lXB17C1+mpEKeqPzObmNOObO5rQ=",
+        "narHash": "sha256-KL5aC6l/mM+ucHadjCtlSY6iBZ9/VKRf2Rgw2tjwfqo=",
         "path": "../systems/backup-2",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-dWAbGxB91utxwTlxsJZdp2OSaIAo8y5LXjdTCFMRn+4=",
+        "narHash": "sha256-VliQ62boc8jNgPCBX6JjVS/pA6YoikZVkvE1xxHAITk=",
         "path": "../systems/dilion",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-OvT4dJGxdFBUgwjTu6FzejI/euWYXWsX1vD952B8934=",
+        "narHash": "sha256-YVJi4KPT8rsGNzZshYfUyz9l27QGAEsb8Iyu3O0/bMA=",
         "path": "../systems/eldiron",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-6hqQjoy1lyIJziqe0AJ62aq62soi5D14NKOzNEY3SsQ=",
+        "narHash": "sha256-HiDwzScPHp084MwtWET+17WbrWf3zs9qJzv/EqNaSqM=",
         "path": "../systems/monitoring-1",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-yBuAfW59X0e840QBFtQrefG3T/IQIQSXiZ4WU2/+gBI=",
+        "narHash": "sha256-YWEuXALLICHE+WsMtMuiCm6b2IRQyJrFpfMgH6ZLD0A=",
         "path": "../systems/quatresaisons",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "./private/monitoring",
         "type": "path"
       },
index 17264a4857347e45a551c4423a3977344cf3a70e..85690faa9ca81d06d94a4784ee147cdd0da5c8b7 100644 (file)
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-B2JdIp5q/8/LurN4lXB17C1+mpEKeqPzObmNOObO5rQ=",
+        "narHash": "sha256-KL5aC6l/mM+ucHadjCtlSY6iBZ9/VKRf2Rgw2tjwfqo=",
         "path": "../systems/backup-2",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-dWAbGxB91utxwTlxsJZdp2OSaIAo8y5LXjdTCFMRn+4=",
+        "narHash": "sha256-VliQ62boc8jNgPCBX6JjVS/pA6YoikZVkvE1xxHAITk=",
         "path": "../systems/dilion",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-OvT4dJGxdFBUgwjTu6FzejI/euWYXWsX1vD952B8934=",
+        "narHash": "sha256-YVJi4KPT8rsGNzZshYfUyz9l27QGAEsb8Iyu3O0/bMA=",
         "path": "../systems/eldiron",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-6hqQjoy1lyIJziqe0AJ62aq62soi5D14NKOzNEY3SsQ=",
+        "narHash": "sha256-HiDwzScPHp084MwtWET+17WbrWf3zs9qJzv/EqNaSqM=",
         "path": "../systems/monitoring-1",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-yBuAfW59X0e840QBFtQrefG3T/IQIQSXiZ4WU2/+gBI=",
+        "narHash": "sha256-YWEuXALLICHE+WsMtMuiCm6b2IRQyJrFpfMgH6ZLD0A=",
         "path": "../systems/quatresaisons",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "./private/monitoring",
         "type": "path"
       },
index f76f2c1a4dfe93dbe79a5978d09d2331758a3280..bb3a383c05ff8fd812c8f9e277a68ce0366994ec 100644 (file)
@@ -284,6 +284,30 @@ in
       ]}
     '';
   };
+  smartctl = {
+    commands = {
+      check_smartctl = "$USER2$/check_smartctl -i auto -d $ARG1$";
+    };
+    chunk = let
+      smartPlugin = pkgs.fetchurl {
+        url = "https://www.claudiokuenzler.com/monitoring-plugins/check_smart.pl";
+        sha256 = "sha256-gxGkzyycUl+I3WziKhOnZSoQjpqbPqjbunfUQxmeb7w=";
+      };
+    in ''
+      cp ${smartPlugin} $out/check_smartctl
+      chmod +x $out/check_smartctl
+      patchShebangs $out
+      substituteInPlace $out/check_smartctl --replace "/usr/bin /bin /usr/sbin /sbin /usr/local/bin /usr/local/sbin" "${pkgs.smartmontools}/bin"
+      substituteInPlace $out/check_smartctl --replace "sudo" "${sudo}"
+    '';
+
+    sudo = _: {
+      commands = [
+        { command = "${pkgs.smartmontools}/bin/smartctl *"; options = [ "NOPASSWD" ]; }
+      ];
+      runAs = "root";
+    };
+  };
   mysql = {
     commands = {
       check_mysql_replication = "${sudo} -u mysql $USER2$/check_mysql_replication \"$ARG1$\" \"$ARG2$\"";
index a8d55bf50b0271d9b2a81d93871edd210b7502e8..d6e56a5d4bf6f5bd9ff0215748ca57644b5cd595 100644 (file)
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
index 30963845f2ba36eb099b4339cb530042dd496a8b..a690504e103a44fbc018eac9580c4eae51f9733c 100644 (file)
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
index af7f22434854dba5f93c045810cace92d48b4c9c..5c4e5b0853d67c5570120f23e70e37403ae6bee8 100644 (file)
@@ -9,7 +9,7 @@ let
   };
 in
 {
-  config.myServices.monitoring.activatedPlugins = [ "memory" "command" "bandwidth" "mdadm" "zfs" "notify-secondary"];
+  config.myServices.monitoring.activatedPlugins = [ "memory" "command" "bandwidth" "mdadm" "zfs" "notify-secondary" "smartctl" ];
   config.myServices.monitoring.objects = lib.mkMerge [
     (monitoring.lib.objectsCommon {
       inherit hostFQDN;
@@ -22,6 +22,38 @@ in
 
     {
       service = [
+        {
+          service_description = "Disk /dev/disk/by-id/ata-Micron_1100_MTFDDAK512TBN_171216BD2A4B is sane";
+          use = "local-service";
+          check_command = [ "check_smartctl" "/dev/disk/by-id/ata-Micron_1100_MTFDDAK512TBN_171216BD2A4B" ];
+          __passive_servicegroups = "webstatus-resources";
+
+          check_interval = 60;
+        }
+        {
+          service_description = "Disk /dev/disk/by-id/ata-Micron_1100_MTFDDAK512TBN_171216BD2A70 is sane";
+          use = "local-service";
+          check_command = [ "check_smartctl" "/dev/disk/by-id/ata-Micron_1100_MTFDDAK512TBN_171216BD2A70" ];
+          __passive_servicegroups = "webstatus-resources";
+
+          check_interval = 60;
+        }
+        {
+          service_description = "Disk /dev/disk/by-id/ata-TOSHIBA_MG04ACA400N_96K1K87YFVLC is sane";
+          use = "local-service";
+          check_command = [ "check_smartctl" "/dev/disk/by-id/ata-TOSHIBA_MG04ACA400N_96K1K87YFVLC" ];
+          __passive_servicegroups = "webstatus-resources";
+
+          check_interval = 60;
+        }
+        {
+          service_description = "Disk /dev/disk/by-id/ata-WDC_WD3000FYYZ-01UL1B2_WD-WMC1F0E4X6WP is sane";
+          use = "local-service";
+          check_command = [ "check_smartctl" "/dev/disk/by-id/ata-WDC_WD3000FYYZ-01UL1B2_WD-WMC1F0E4X6WP" ];
+          __passive_servicegroups = "webstatus-resources";
+
+          check_interval = 60;
+        }
         {
           service_description = "No mdadm array is degraded";
           use = "local-service";
index f5b9184fc563f76d7343825100c9369ccc80e1e1..461bebb2c990bb360a3c4e81cd8f87021443d2d5 100644 (file)
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
index 2aa7f8f1370a4e7bc9c54e6fef221c5580e5dc55..92da7cf1340a816766277a22df44c05e1c1c04a1 100644 (file)
@@ -4,7 +4,7 @@ let
   emailCheck = monitoring.lib.emailCheck config.myEnv.monitoring.email_check;
 in
 {
-  config.myServices.monitoring.activatedPlugins = [ "memory" "command" "bandwidth" "emails" "mdadm" "postfix" "postgresql" "zfs" "notify-secondary"];
+  config.myServices.monitoring.activatedPlugins = [ "memory" "command" "bandwidth" "emails" "mdadm" "postfix" "postgresql" "zfs" "notify-secondary" "smartctl" ];
   config.myServices.monitoring.pluginsArgs.postgresql.package = config.myServices.databases.postgresql.package;
   config.myServices.monitoring.objects = lib.mkMerge [
     (monitoring.lib.objectsCommon {
@@ -18,6 +18,38 @@ in
 
     {
       service = [
+        {
+          service_description = "Disk /dev/disk/by-id/ata-ST33000650NS_Z29540DM is sane";
+          use = "local-service";
+          check_command = [ "check_smartctl" "/dev/disk/by-id/ata-ST33000650NS_Z29540DM" ];
+          __passive_servicegroups = "webstatus-resources";
+
+          check_interval = 60;
+        }
+        {
+          service_description = "Disk /dev/disk/by-id/ata-ST33000650NS_Z296JGJ4 is sane";
+          use = "local-service";
+          check_command = [ "check_smartctl" "/dev/disk/by-id/ata-ST33000650NS_Z296JGJ4" ];
+          __passive_servicegroups = "webstatus-resources";
+
+          check_interval = 60;
+        }
+        {
+          service_description = "Disk /dev/disk/by-id/ata-SAMSUNG_MZ7LM480HMHQ-00005_S2UJNX0HA05895 is sane";
+          use = "local-service";
+          check_command = [ "check_smartctl" "/dev/disk/by-id/ata-SAMSUNG_MZ7LM480HMHQ-00005_S2UJNX0HA05895" ];
+          __passive_servicegroups = "webstatus-resources";
+
+          check_interval = 60;
+        }
+        {
+          service_description = "Disk /dev/disk/by-id/ata-Micron_5200_MTFDDAK480TDC_18011BEE03B3 is sane";
+          use = "local-service";
+          check_command = [ "check_smartctl" "/dev/disk/by-id/ata-Micron_5200_MTFDDAK480TDC_18011BEE03B3" ];
+          __passive_servicegroups = "webstatus-resources";
+
+          check_interval = 60;
+        }
         {
           service_description = "No mdadm array is degraded";
           use = "local-service";
index ed711f4d03bfc0e7f7626716c2fb7086a41c6d9b..643683a2678c16ce6a70f41820423bca4f35ce76 100644 (file)
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
index a76103c5af037fc9299580bb1b2eed22275e5ad2..afb007be48cdb270c9eac9fca1f57b67620ae997 100644 (file)
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-PicwNqwQaCKtN5zPpTfpSDRXbyn9xh8aHqoPnm9cFz8=",
+        "narHash": "sha256-A50vXg0qfzUhyQZKToEseYsLgtaYke41KE/4uuyj+PY=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },