Add alternate cloud storage for daily backups

4 files changed, 49 insertions, 31 deletions

diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index d5bf7fb..c573af2 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -58,9 +58,12 @@ let
     wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
       pkgs.s3cmd pkgs.python3
     ]}
-    wrapProgram $out/check_eriomem_age --prefix PATH : ${lib.makeBinPath [
+    makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
       pkgs.duplicity
     ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
+    makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
+      pkgs.duplicity
+    ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
     wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
       pkgs.mailutils
     ]}
@@ -256,18 +259,19 @@ in
         permissions = "0400";
         text = config.myEnv.monitoring.ssh_secret_key;
       }
-    ] ++ lib.optional cfg.master (
+    ] ++ lib.optionals cfg.master (
+      lib.mapAttrsToList (k: v:
       {
-        dest = "eriomem_access_key";
+        dest = "${k}_access_key";
         user = "naemon";
         group = "naemon";
         permissions = "0400";
         text = ''
-          export AWS_ACCESS_KEY_ID="${config.myEnv.backup.accessKeyId}"
-          export AWS_SECRET_ACCESS_KEY="${config.myEnv.backup.secretAccessKey}"
-          export BASE_URL="${config.myEnv.backup.remote}"
+          export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
+          export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
+          export BASE_URL="${v.remote "immae-eldiron"}"
         '';
-      }
+      }) config.myEnv.backup.remotes
     );
     # needed since extraResource is not in the closure
     systemd.services.naemon.path = [ myplugins ];
diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix
index d3a46ce..25f8124 100644
--- a/modules/private/monitoring/objects_common.nix
+++ b/modules/private/monitoring/objects_common.nix
@@ -108,8 +108,9 @@ in
     check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$";
     check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$";
     check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$";
-    check_eriomem = "$USER2$/check_eriomem $USER208$";
-    check_eriomem_age = "$USER2$/check_eriomem_age $ARG1$";
+    check_backup_eriomem = "$USER2$/check_eriomem $USER208$";
+    check_backup_eriomem_age = "$USER2$/check_backup_eriomem_age $ARG1$";
+    check_backup_ovh_age = "$USER2$/check_backup_ovh_age $ARG1$";
     check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$";
     check_ftp_database = "$USER2$/check_ftp_database";
     check_git = "$USER2$/check_git $USER203$";
diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix
index 2cba58e..1aebd26 100644
--- a/modules/private/monitoring/objects_monitoring-1.nix
+++ b/modules/private/monitoring/objects_monitoring-1.nix
@@ -1,22 +1,24 @@
-{ config, pkgs, nodes, hostFQDN, emailCheck, ... }:
+{ config, pkgs, nodes, hostFQDN, emailCheck, lib, ... }:
 let
-  to_eriomem_age_dependency = name: {
-    dependent_host_name = "eldiron.immae.eu";
-    host_name           = "eldiron.immae.eu";
-    dependent_service_description = "Eriomem backup for ${name} is not too old";
-    service_description = "Eriomem backup is up and not full";
-    execution_failure_criteria = "u";
-    notification_failure_criteria = "u";
-  };
-  to_eriomem_age = name: {
-    service_description = "Eriomem backup for ${name} is not too old";
-    host_name = "eldiron.immae.eu";
-    use = "external-service";
-    check_command = ["check_eriomem_age" name];
-
-    check_interval = "120";
-    notification_interval = "1440";
-  };
+  to_backup_age_dependency = name: profile: map (remote:
+    {
+      dependent_host_name = "eldiron.immae.eu";
+      host_name           = "eldiron.immae.eu";
+      dependent_service_description = "${remote} backup for ${name} is not too old";
+      service_description = "${remote} backup is up and not full";
+      execution_failure_criteria = "u";
+      notification_failure_criteria = "u";
+    }) profile.remotes;
+  to_backup_age = name: profile: map (remote:
+    {
+      service_description = "${remote} backup for ${name} is not too old";
+      host_name = "eldiron.immae.eu";
+      use = "external-service";
+      check_command = ["check_backup_${remote}_age" name];
+
+      check_interval = "120";
+      notification_interval = "1440";
+    }) profile.remotes;
 in
 {
   host = {
@@ -111,10 +113,21 @@ in
 
     # Backup services
     {
-      service_description = "Eriomem backup is up and not full";
+      service_description = "eriomem backup is up and not full";
+      host_name = "eldiron.immae.eu";
+      use = "external-service";
+      check_command = "check_backup_eriomem";
+
+      check_interval = "120";
+      notification_interval = "1440";
+
+      servicegroups = "webstatus-backup";
+    }
+    {
+      service_description = "ovh backup is up and not full";
       host_name = "eldiron.immae.eu";
       use = "external-service";
-      check_command = "check_eriomem";
+      check_command = "check_ok";
 
       check_interval = "120";
       notification_interval = "1440";
@@ -673,7 +686,7 @@ in
       _webstatus_name = "LDAP";
       _webstatus_url = "ldap.immae.eu";
     }
-  ] ++ map to_eriomem_age (builtins.attrNames nodes.eldiron.config.services.duplyBackup.profiles);
+  ] ++ lib.flatten (lib.mapAttrsToList to_backup_age nodes.eldiron.config.services.duplyBackup.profiles);
   contact = {
     telio-tortay = config.myEnv.monitoring.contacts.telio-tortay // {
       use = "generic-contact";
@@ -688,5 +701,5 @@ in
     telio-tortay = { alias = "Telio Tortay"; members = "immae"; };
     tiboqorl = { alias = "Tiboqorl"; members = "immae"; };
   };
-  servicedependency = map to_eriomem_age_dependency (builtins.attrNames nodes.eldiron.config.services.duplyBackup.profiles);
+  servicedependency = lib.flatten (lib.mapAttrsToList to_backup_age_dependency nodes.eldiron.config.services.duplyBackup.profiles);
 }
diff --git a/modules/private/monitoring/plugins/check_eriomem_age b/modules/private/monitoring/plugins/check_backup_age
index 4d03b82..4d03b82 100755
--- a/modules/private/monitoring/plugins/check_eriomem_age
+++ b/modules/private/monitoring/plugins/check_backup_age