Add alternate cloud storage for daily backups

14 files changed, 116 insertions, 56 deletions

diff --git a/modules/duply_backup/default.nix b/modules/duply_backup/default.nix
index bce4d65..73ac8f0 100644
--- a/modules/duply_backup/default.nix
+++ b/modules/duply_backup/default.nix
@@ -1,13 +1,13 @@
-{ lib, pkgs, config, ... }:
+{ lib, pkgs, config, name, ... }:
 
 let
   cfg = config.myEnv.backup;
   varDir = "/var/lib/duply";
-  duplyProfile = profile: prefix: ''
+  duplyProfile = profile: remote: prefix: ''
     GPG_PW="${cfg.password}"
-    TARGET="${cfg.remote}${prefix}"
-    export AWS_ACCESS_KEY_ID="${cfg.accessKeyId}"
-    export AWS_SECRET_ACCESS_KEY="${cfg.secretAccessKey}"
+    TARGET="${cfg.remotes.${remote}.remote profile.bucket}${prefix}"
+    export AWS_ACCESS_KEY_ID="${cfg.remotes.${remote}.accessKeyId}"
+    export AWS_SECRET_ACCESS_KEY="${cfg.remotes.${remote}.secretAccessKey}"
     SOURCE="${profile.rootDir}"
     FILENAME=".duplicity-ignore"
     DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'"
@@ -24,6 +24,8 @@ let
     MAX_FULLS_WITH_INCRS=2
   '';
   action = "bkp_purge_purgeFull_purgeIncr";
+  varName = k: remoteName:
+    if remoteName == "eriomem" then k else remoteName + "_" + k;
 in
 {
   options = {
@@ -43,6 +45,20 @@ in
               Path to backup
               '';
           };
+          bucket = lib.mkOption {
+            type = lib.types.str;
+            default = "immae-${name}";
+            description = ''
+              Bucket to use
+              '';
+          };
+          remotes = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = ["eriomem"];
+            description = ''
+              Remotes to use for backup
+              '';
+          };
           excludeFile = lib.mkOption {
             type = lib.types.lines;
             default = "";
@@ -59,18 +75,19 @@ in
     system.activationScripts.backup = ''
       install -m 0700 -o root -g root -d ${varDir} ${varDir}/caches
       '';
-    secrets.keys = lib.flatten (lib.mapAttrsToList (k: v: [
-      {
-        permissions = "0400";
-        dest = "backup/${k}/conf";
-        text = duplyProfile v "${k}/";
-      }
-      {
-        permissions = "0400";
-        dest = "backup/${k}/exclude";
-        text = v.excludeFile;
-      }
-    ]) config.services.duplyBackup.profiles);
+    secrets.keys = lib.flatten (lib.mapAttrsToList (k: v:
+      map (remote: [
+        {
+          permissions = "0400";
+          dest = "backup/${varName k remote}/conf";
+          text = duplyProfile v remote "${k}/";
+        }
+        {
+          permissions = "0400";
+          dest = "backup/${varName k remote}/exclude";
+          text = v.excludeFile;
+        }
+    ]) v.remotes) config.services.duplyBackup.profiles);
 
     services.cron = {
       enable = true;
@@ -78,13 +95,15 @@ in
         backups = pkgs.writeScript "backups" ''
           #!${pkgs.stdenv.shell}
 
-          ${builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v:
+          ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v:
+            map (remote: [
             ''
-              touch ${varDir}/${k}.log
-              ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/${k}/ ${action} --force >> ${varDir}/${k}.log
-              [[ $? = 0 ]] || echo -e "Error when doing backup for ${k}, see above\n---------------------------------------" >&2
+              touch ${varDir}/${varName k remote}.log
+              ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/${varName k remote}/ ${action} --force >> ${varDir}/${varName k remote}.log
+              [[ $? = 0 ]] || echo -e "Error when doing backup for ${varName k remote}, see above\n---------------------------------------" >&2
             ''
-          ) config.services.duplyBackup.profiles)}
+            ]) v.remotes
+          ) config.services.duplyBackup.profiles))}
         '';
       in
         [
diff --git a/modules/private/environment.nix b/modules/private/environment.nix
index b8c4dd2..069a344 100644
--- a/modules/private/environment.nix
+++ b/modules/private/environment.nix
@@ -441,9 +441,24 @@ in
       type = submodule {
         options = {
           password = mkOption { type = str; description = "Password for encrypting files"; };
-          remote = mkOption { type = str; description = "Remote url access"; };
-          accessKeyId = mkOption { type = str; description = "Remote access-key"; };
-          secretAccessKey = mkOption { type = str; description = "Remote access secret"; };
+          remotes = mkOption {
+            type = attrsOf (submodule {
+              options = {
+                remote = mkOption {
+                  type = unspecified;
+                  example = literalExample ''
+                    bucket: "s3://some_host/${bucket}";
+                    '';
+                  description = ''
+                    Function.
+                    Takes a bucket name as argument and returns a url
+                    '';
+                };
+                accessKeyId = mkOption { type = str; description = "Remote access-key"; };
+                secretAccessKey = mkOption { type = str; description = "Remote access secret"; };
+              };
+            });
+          };
         };
       };
     };
diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index d5bf7fb..c573af2 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -58,9 +58,12 @@ let
     wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
       pkgs.s3cmd pkgs.python3
     ]}
-    wrapProgram $out/check_eriomem_age --prefix PATH : ${lib.makeBinPath [
+    makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
       pkgs.duplicity
     ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
+    makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
+      pkgs.duplicity
+    ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
     wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
       pkgs.mailutils
     ]}
@@ -256,18 +259,19 @@ in
         permissions = "0400";
         text = config.myEnv.monitoring.ssh_secret_key;
       }
-    ] ++ lib.optional cfg.master (
+    ] ++ lib.optionals cfg.master (
+      lib.mapAttrsToList (k: v:
       {
-        dest = "eriomem_access_key";
+        dest = "${k}_access_key";
         user = "naemon";
         group = "naemon";
         permissions = "0400";
         text = ''
-          export AWS_ACCESS_KEY_ID="${config.myEnv.backup.accessKeyId}"
-          export AWS_SECRET_ACCESS_KEY="${config.myEnv.backup.secretAccessKey}"
-          export BASE_URL="${config.myEnv.backup.remote}"
+          export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
+          export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
+          export BASE_URL="${v.remote "immae-eldiron"}"
         '';
-      }
+      }) config.myEnv.backup.remotes
     );
     # needed since extraResource is not in the closure
     systemd.services.naemon.path = [ myplugins ];
diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix
index d3a46ce..25f8124 100644
--- a/modules/private/monitoring/objects_common.nix
+++ b/modules/private/monitoring/objects_common.nix
@@ -108,8 +108,9 @@ in
     check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$";
     check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$";
     check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$";
-    check_eriomem = "$USER2$/check_eriomem $USER208$";
-    check_eriomem_age = "$USER2$/check_eriomem_age $ARG1$";
+    check_backup_eriomem = "$USER2$/check_eriomem $USER208$";
+    check_backup_eriomem_age = "$USER2$/check_backup_eriomem_age $ARG1$";
+    check_backup_ovh_age = "$USER2$/check_backup_ovh_age $ARG1$";
     check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$";
     check_ftp_database = "$USER2$/check_ftp_database";
     check_git = "$USER2$/check_git $USER203$";
diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix
index 2cba58e..1aebd26 100644
--- a/modules/private/monitoring/objects_monitoring-1.nix
+++ b/modules/private/monitoring/objects_monitoring-1.nix
@@ -1,22 +1,24 @@
-{ config, pkgs, nodes, hostFQDN, emailCheck, ... }:
+{ config, pkgs, nodes, hostFQDN, emailCheck, lib, ... }:
 let
-  to_eriomem_age_dependency = name: {
-    dependent_host_name = "eldiron.immae.eu";
-    host_name           = "eldiron.immae.eu";
-    dependent_service_description = "Eriomem backup for ${name} is not too old";
-    service_description = "Eriomem backup is up and not full";
-    execution_failure_criteria = "u";
-    notification_failure_criteria = "u";
-  };
-  to_eriomem_age = name: {
-    service_description = "Eriomem backup for ${name} is not too old";
-    host_name = "eldiron.immae.eu";
-    use = "external-service";
-    check_command = ["check_eriomem_age" name];
-
-    check_interval = "120";
-    notification_interval = "1440";
-  };
+  to_backup_age_dependency = name: profile: map (remote:
+    {
+      dependent_host_name = "eldiron.immae.eu";
+      host_name           = "eldiron.immae.eu";
+      dependent_service_description = "${remote} backup for ${name} is not too old";
+      service_description = "${remote} backup is up and not full";
+      execution_failure_criteria = "u";
+      notification_failure_criteria = "u";
+    }) profile.remotes;
+  to_backup_age = name: profile: map (remote:
+    {
+      service_description = "${remote} backup for ${name} is not too old";
+      host_name = "eldiron.immae.eu";
+      use = "external-service";
+      check_command = ["check_backup_${remote}_age" name];
+
+      check_interval = "120";
+      notification_interval = "1440";
+    }) profile.remotes;
 in
 {
   host = {
@@ -111,10 +113,21 @@ in
 
     # Backup services
     {
-      service_description = "Eriomem backup is up and not full";
+      service_description = "eriomem backup is up and not full";
+      host_name = "eldiron.immae.eu";
+      use = "external-service";
+      check_command = "check_backup_eriomem";
+
+      check_interval = "120";
+      notification_interval = "1440";
+
+      servicegroups = "webstatus-backup";
+    }
+    {
+      service_description = "ovh backup is up and not full";
       host_name = "eldiron.immae.eu";
       use = "external-service";
-      check_command = "check_eriomem";
+      check_command = "check_ok";
 
       check_interval = "120";
       notification_interval = "1440";
@@ -673,7 +686,7 @@ in
       _webstatus_name = "LDAP";
       _webstatus_url = "ldap.immae.eu";
     }
-  ] ++ map to_eriomem_age (builtins.attrNames nodes.eldiron.config.services.duplyBackup.profiles);
+  ] ++ lib.flatten (lib.mapAttrsToList to_backup_age nodes.eldiron.config.services.duplyBackup.profiles);
   contact = {
     telio-tortay = config.myEnv.monitoring.contacts.telio-tortay // {
       use = "generic-contact";
@@ -688,5 +701,5 @@ in
     telio-tortay = { alias = "Telio Tortay"; members = "immae"; };
     tiboqorl = { alias = "Tiboqorl"; members = "immae"; };
   };
-  servicedependency = map to_eriomem_age_dependency (builtins.attrNames nodes.eldiron.config.services.duplyBackup.profiles);
+  servicedependency = lib.flatten (lib.mapAttrsToList to_backup_age_dependency nodes.eldiron.config.services.duplyBackup.profiles);
 }
diff --git a/modules/private/monitoring/plugins/check_eriomem_age b/modules/private/monitoring/plugins/check_backup_age
index 4d03b82..4d03b82 100755
--- a/modules/private/monitoring/plugins/check_eriomem_age
+++ b/modules/private/monitoring/plugins/check_backup_age
diff --git a/modules/private/websites/chloe/production.nix b/modules/private/websites/chloe/production.nix
index b5233c6..7f8f1de 100644
--- a/modules/private/websites/chloe/production.nix
+++ b/modules/private/websites/chloe/production.nix
@@ -15,6 +15,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     services.duplyBackup.profiles.chloe_production.rootDir = app.varDir;
+    services.duplyBackup.profiles.chloe_production.remotes = ["eriomem" "ovh"];
     secrets.keys = [
       {
         dest = "websites/chloe/production";
diff --git a/modules/private/websites/connexionswing/production.nix b/modules/private/websites/connexionswing/production.nix
index e172f84..f6a059d 100644
--- a/modules/private/websites/connexionswing/production.nix
+++ b/modules/private/websites/connexionswing/production.nix
@@ -14,6 +14,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     services.duplyBackup.profiles.connexionswing_production.rootDir = app.varDir;
+    services.duplyBackup.profiles.connexionswing_production.remotes = ["eriomem" "ovh"];
     services.webstats.sites = [ { name = "connexionswing.com"; } ];
     services.phpApplication.apps.connexionswing_production = {
       websiteEnv = "production";
diff --git a/modules/private/websites/isabelle/aten_production.nix b/modules/private/websites/isabelle/aten_production.nix
index 4fa3622..3671712 100644
--- a/modules/private/websites/isabelle/aten_production.nix
+++ b/modules/private/websites/isabelle/aten_production.nix
@@ -13,6 +13,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     services.duplyBackup.profiles.isabelle_aten_production.rootDir = app.varDir;
+    services.duplyBackup.profiles.isabelle_aten_production.remotes = ["eriomem" "ovh"];
     services.webstats.sites = [ { name = "aten.pro"; } ];
     services.phpApplication.apps.isabelle_aten_production = {
       websiteEnv = "production";
diff --git a/modules/private/websites/isabelle/iridologie.nix b/modules/private/websites/isabelle/iridologie.nix
index 5fa87ce..14296bf 100644
--- a/modules/private/websites/isabelle/iridologie.nix
+++ b/modules/private/websites/isabelle/iridologie.nix
@@ -17,6 +17,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     services.duplyBackup.profiles.isabelle_iridologie.rootDir = app.varDir;
+    services.duplyBackup.profiles.isabelle_iridologie.remotes = ["eriomem" "ovh"];
     secrets.keys = [
       {
         dest = "websites/isabelle/iridologie";
diff --git a/modules/private/websites/ludivine/production.nix b/modules/private/websites/ludivine/production.nix
index b30f488..3a9895d 100644
--- a/modules/private/websites/ludivine/production.nix
+++ b/modules/private/websites/ludivine/production.nix
@@ -14,6 +14,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     services.duplyBackup.profiles.ludivine_production.rootDir = app.varDir;
+    services.duplyBackup.profiles.ludivine_production.remotes = ["eriomem" "ovh"];
     services.webstats.sites = [ { name = "ludivinecassal.com"; } ];
     services.phpApplication.apps.ludivine_production = {
       websiteEnv = "production";
diff --git a/modules/private/websites/piedsjaloux/production.nix b/modules/private/websites/piedsjaloux/production.nix
index 03b9ec5..e12b046 100644
--- a/modules/private/websites/piedsjaloux/production.nix
+++ b/modules/private/websites/piedsjaloux/production.nix
@@ -15,6 +15,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     services.duplyBackup.profiles.piedsjaloux_production.rootDir = app.varDir;
+    services.duplyBackup.profiles.piedsjaloux_production.remotes = ["eriomem" "ovh"];
     services.webstats.sites = [ { name = "piedsjaloux.fr"; } ];
     services.phpApplication.apps.piedsjaloux_production = {
       websiteEnv = "production";
diff --git a/modules/private/websites/richie/production.nix b/modules/private/websites/richie/production.nix
index a6957af..2d85175 100644
--- a/modules/private/websites/richie/production.nix
+++ b/modules/private/websites/richie/production.nix
@@ -26,6 +26,7 @@ in
   options.myServices.websites.richie.production.enable = lib.mkEnableOption "enable Richie's website";
   config = lib.mkIf cfg.enable {
     services.duplyBackup.profiles.richie_production.rootDir = vardir;
+    services.duplyBackup.profiles.richie_production.remotes = ["eriomem" "ovh"];
     services.webstats.sites = [ { name = "europe-richie.org"; } ];
 
     secrets.keys = [{
diff --git a/modules/private/websites/syden/peertube.nix b/modules/private/websites/syden/peertube.nix
index b17e775..5970cca 100644
--- a/modules/private/websites/syden/peertube.nix
+++ b/modules/private/websites/syden/peertube.nix
@@ -12,6 +12,7 @@ in
   config = lib.mkIf scfg.enable {
     services.duplyBackup.profiles.syden_peertube = {
       rootDir = dataDir;
+      remotes = ["eriomem" "ovh"];
     };
     users.users.peertube = {
       uid = config.ids.uids.peertube;