From 5a61f6ad5164a735be26e016c59e72252ffb49b7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Thu, 16 Jul 2020 01:10:17 +0200 Subject: [PATCH] Add alternate cloud storage for daily backups --- modules/duply_backup/default.nix | 63 ++++++++++++------- modules/private/environment.nix | 21 ++++++- modules/private/monitoring/default.nix | 18 +++--- modules/private/monitoring/objects_common.nix | 5 +- .../monitoring/objects_monitoring-1.nix | 57 ++++++++++------- .../{check_eriomem_age => check_backup_age} | 0 modules/private/websites/chloe/production.nix | 1 + .../websites/connexionswing/production.nix | 1 + .../websites/isabelle/aten_production.nix | 1 + .../private/websites/isabelle/iridologie.nix | 1 + .../private/websites/ludivine/production.nix | 1 + .../websites/piedsjaloux/production.nix | 1 + .../private/websites/richie/production.nix | 1 + modules/private/websites/syden/peertube.nix | 1 + 14 files changed, 116 insertions(+), 56 deletions(-) rename modules/private/monitoring/plugins/{check_eriomem_age => check_backup_age} (100%) diff --git a/modules/duply_backup/default.nix b/modules/duply_backup/default.nix index bce4d65..73ac8f0 100644 --- a/modules/duply_backup/default.nix +++ b/modules/duply_backup/default.nix @@ -1,13 +1,13 @@ -{ lib, pkgs, config, ... }: +{ lib, pkgs, config, name, ... }: let cfg = config.myEnv.backup; varDir = "/var/lib/duply"; - duplyProfile = profile: prefix: '' + duplyProfile = profile: remote: prefix: '' GPG_PW="${cfg.password}" - TARGET="${cfg.remote}${prefix}" - export AWS_ACCESS_KEY_ID="${cfg.accessKeyId}" - export AWS_SECRET_ACCESS_KEY="${cfg.secretAccessKey}" + TARGET="${cfg.remotes.${remote}.remote profile.bucket}${prefix}" + export AWS_ACCESS_KEY_ID="${cfg.remotes.${remote}.accessKeyId}" + export AWS_SECRET_ACCESS_KEY="${cfg.remotes.${remote}.secretAccessKey}" SOURCE="${profile.rootDir}" FILENAME=".duplicity-ignore" DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'" @@ -24,6 +24,8 @@ let MAX_FULLS_WITH_INCRS=2 ''; action = "bkp_purge_purgeFull_purgeIncr"; + varName = k: remoteName: + if remoteName == "eriomem" then k else remoteName + "_" + k; in { options = { @@ -43,6 +45,20 @@ in Path to backup ''; }; + bucket = lib.mkOption { + type = lib.types.str; + default = "immae-${name}"; + description = '' + Bucket to use + ''; + }; + remotes = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = ["eriomem"]; + description = '' + Remotes to use for backup + ''; + }; excludeFile = lib.mkOption { type = lib.types.lines; default = ""; @@ -59,18 +75,19 @@ in system.activationScripts.backup = '' install -m 0700 -o root -g root -d ${varDir} ${varDir}/caches ''; - secrets.keys = lib.flatten (lib.mapAttrsToList (k: v: [ - { - permissions = "0400"; - dest = "backup/${k}/conf"; - text = duplyProfile v "${k}/"; - } - { - permissions = "0400"; - dest = "backup/${k}/exclude"; - text = v.excludeFile; - } - ]) config.services.duplyBackup.profiles); + secrets.keys = lib.flatten (lib.mapAttrsToList (k: v: + map (remote: [ + { + permissions = "0400"; + dest = "backup/${varName k remote}/conf"; + text = duplyProfile v remote "${k}/"; + } + { + permissions = "0400"; + dest = "backup/${varName k remote}/exclude"; + text = v.excludeFile; + } + ]) v.remotes) config.services.duplyBackup.profiles); services.cron = { enable = true; @@ -78,13 +95,15 @@ in backups = pkgs.writeScript "backups" '' #!${pkgs.stdenv.shell} - ${builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: + ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: + map (remote: [ '' - touch ${varDir}/${k}.log - ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/${k}/ ${action} --force >> ${varDir}/${k}.log - [[ $? = 0 ]] || echo -e "Error when doing backup for ${k}, see above\n---------------------------------------" >&2 + touch ${varDir}/${varName k remote}.log + ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/${varName k remote}/ ${action} --force >> ${varDir}/${varName k remote}.log + [[ $? = 0 ]] || echo -e "Error when doing backup for ${varName k remote}, see above\n---------------------------------------" >&2 '' - ) config.services.duplyBackup.profiles)} + ]) v.remotes + ) config.services.duplyBackup.profiles))} ''; in [ diff --git a/modules/private/environment.nix b/modules/private/environment.nix index b8c4dd2..069a344 100644 --- a/modules/private/environment.nix +++ b/modules/private/environment.nix @@ -441,9 +441,24 @@ in type = submodule { options = { password = mkOption { type = str; description = "Password for encrypting files"; }; - remote = mkOption { type = str; description = "Remote url access"; }; - accessKeyId = mkOption { type = str; description = "Remote access-key"; }; - secretAccessKey = mkOption { type = str; description = "Remote access secret"; }; + remotes = mkOption { + type = attrsOf (submodule { + options = { + remote = mkOption { + type = unspecified; + example = literalExample '' + bucket: "s3://some_host/${bucket}"; + ''; + description = '' + Function. + Takes a bucket name as argument and returns a url + ''; + }; + accessKeyId = mkOption { type = str; description = "Remote access-key"; }; + secretAccessKey = mkOption { type = str; description = "Remote access secret"; }; + }; + }); + }; }; }; }; diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix index d5bf7fb..c573af2 100644 --- a/modules/private/monitoring/default.nix +++ b/modules/private/monitoring/default.nix @@ -58,9 +58,12 @@ let wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [ pkgs.s3cmd pkgs.python3 ]} - wrapProgram $out/check_eriomem_age --prefix PATH : ${lib.makeBinPath [ + makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [ pkgs.duplicity ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"} + makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [ + pkgs.duplicity + ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"} wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [ pkgs.mailutils ]} @@ -256,18 +259,19 @@ in permissions = "0400"; text = config.myEnv.monitoring.ssh_secret_key; } - ] ++ lib.optional cfg.master ( + ] ++ lib.optionals cfg.master ( + lib.mapAttrsToList (k: v: { - dest = "eriomem_access_key"; + dest = "${k}_access_key"; user = "naemon"; group = "naemon"; permissions = "0400"; text = '' - export AWS_ACCESS_KEY_ID="${config.myEnv.backup.accessKeyId}" - export AWS_SECRET_ACCESS_KEY="${config.myEnv.backup.secretAccessKey}" - export BASE_URL="${config.myEnv.backup.remote}" + export AWS_ACCESS_KEY_ID="${v.accessKeyId}" + export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}" + export BASE_URL="${v.remote "immae-eldiron"}" ''; - } + }) config.myEnv.backup.remotes ); # needed since extraResource is not in the closure systemd.services.naemon.path = [ myplugins ]; diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix index d3a46ce..25f8124 100644 --- a/modules/private/monitoring/objects_common.nix +++ b/modules/private/monitoring/objects_common.nix @@ -108,8 +108,9 @@ in check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$"; check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$"; check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$"; - check_eriomem = "$USER2$/check_eriomem $USER208$"; - check_eriomem_age = "$USER2$/check_eriomem_age $ARG1$"; + check_backup_eriomem = "$USER2$/check_eriomem $USER208$"; + check_backup_eriomem_age = "$USER2$/check_backup_eriomem_age $ARG1$"; + check_backup_ovh_age = "$USER2$/check_backup_ovh_age $ARG1$"; check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$"; check_ftp_database = "$USER2$/check_ftp_database"; check_git = "$USER2$/check_git $USER203$"; diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix index 2cba58e..1aebd26 100644 --- a/modules/private/monitoring/objects_monitoring-1.nix +++ b/modules/private/monitoring/objects_monitoring-1.nix @@ -1,22 +1,24 @@ -{ config, pkgs, nodes, hostFQDN, emailCheck, ... }: +{ config, pkgs, nodes, hostFQDN, emailCheck, lib, ... }: let - to_eriomem_age_dependency = name: { - dependent_host_name = "eldiron.immae.eu"; - host_name = "eldiron.immae.eu"; - dependent_service_description = "Eriomem backup for ${name} is not too old"; - service_description = "Eriomem backup is up and not full"; - execution_failure_criteria = "u"; - notification_failure_criteria = "u"; - }; - to_eriomem_age = name: { - service_description = "Eriomem backup for ${name} is not too old"; - host_name = "eldiron.immae.eu"; - use = "external-service"; - check_command = ["check_eriomem_age" name]; - - check_interval = "120"; - notification_interval = "1440"; - }; + to_backup_age_dependency = name: profile: map (remote: + { + dependent_host_name = "eldiron.immae.eu"; + host_name = "eldiron.immae.eu"; + dependent_service_description = "${remote} backup for ${name} is not too old"; + service_description = "${remote} backup is up and not full"; + execution_failure_criteria = "u"; + notification_failure_criteria = "u"; + }) profile.remotes; + to_backup_age = name: profile: map (remote: + { + service_description = "${remote} backup for ${name} is not too old"; + host_name = "eldiron.immae.eu"; + use = "external-service"; + check_command = ["check_backup_${remote}_age" name]; + + check_interval = "120"; + notification_interval = "1440"; + }) profile.remotes; in { host = { @@ -111,10 +113,21 @@ in # Backup services { - service_description = "Eriomem backup is up and not full"; + service_description = "eriomem backup is up and not full"; + host_name = "eldiron.immae.eu"; + use = "external-service"; + check_command = "check_backup_eriomem"; + + check_interval = "120"; + notification_interval = "1440"; + + servicegroups = "webstatus-backup"; + } + { + service_description = "ovh backup is up and not full"; host_name = "eldiron.immae.eu"; use = "external-service"; - check_command = "check_eriomem"; + check_command = "check_ok"; check_interval = "120"; notification_interval = "1440"; @@ -673,7 +686,7 @@ in _webstatus_name = "LDAP"; _webstatus_url = "ldap.immae.eu"; } - ] ++ map to_eriomem_age (builtins.attrNames nodes.eldiron.config.services.duplyBackup.profiles); + ] ++ lib.flatten (lib.mapAttrsToList to_backup_age nodes.eldiron.config.services.duplyBackup.profiles); contact = { telio-tortay = config.myEnv.monitoring.contacts.telio-tortay // { use = "generic-contact"; @@ -688,5 +701,5 @@ in telio-tortay = { alias = "Telio Tortay"; members = "immae"; }; tiboqorl = { alias = "Tiboqorl"; members = "immae"; }; }; - servicedependency = map to_eriomem_age_dependency (builtins.attrNames nodes.eldiron.config.services.duplyBackup.profiles); + servicedependency = lib.flatten (lib.mapAttrsToList to_backup_age_dependency nodes.eldiron.config.services.duplyBackup.profiles); } diff --git a/modules/private/monitoring/plugins/check_eriomem_age b/modules/private/monitoring/plugins/check_backup_age similarity index 100% rename from modules/private/monitoring/plugins/check_eriomem_age rename to modules/private/monitoring/plugins/check_backup_age diff --git a/modules/private/websites/chloe/production.nix b/modules/private/websites/chloe/production.nix index b5233c6..7f8f1de 100644 --- a/modules/private/websites/chloe/production.nix +++ b/modules/private/websites/chloe/production.nix @@ -15,6 +15,7 @@ in { config = lib.mkIf cfg.enable { services.duplyBackup.profiles.chloe_production.rootDir = app.varDir; + services.duplyBackup.profiles.chloe_production.remotes = ["eriomem" "ovh"]; secrets.keys = [ { dest = "websites/chloe/production"; diff --git a/modules/private/websites/connexionswing/production.nix b/modules/private/websites/connexionswing/production.nix index e172f84..f6a059d 100644 --- a/modules/private/websites/connexionswing/production.nix +++ b/modules/private/websites/connexionswing/production.nix @@ -14,6 +14,7 @@ in { config = lib.mkIf cfg.enable { services.duplyBackup.profiles.connexionswing_production.rootDir = app.varDir; + services.duplyBackup.profiles.connexionswing_production.remotes = ["eriomem" "ovh"]; services.webstats.sites = [ { name = "connexionswing.com"; } ]; services.phpApplication.apps.connexionswing_production = { websiteEnv = "production"; diff --git a/modules/private/websites/isabelle/aten_production.nix b/modules/private/websites/isabelle/aten_production.nix index 4fa3622..3671712 100644 --- a/modules/private/websites/isabelle/aten_production.nix +++ b/modules/private/websites/isabelle/aten_production.nix @@ -13,6 +13,7 @@ in { config = lib.mkIf cfg.enable { services.duplyBackup.profiles.isabelle_aten_production.rootDir = app.varDir; + services.duplyBackup.profiles.isabelle_aten_production.remotes = ["eriomem" "ovh"]; services.webstats.sites = [ { name = "aten.pro"; } ]; services.phpApplication.apps.isabelle_aten_production = { websiteEnv = "production"; diff --git a/modules/private/websites/isabelle/iridologie.nix b/modules/private/websites/isabelle/iridologie.nix index 5fa87ce..14296bf 100644 --- a/modules/private/websites/isabelle/iridologie.nix +++ b/modules/private/websites/isabelle/iridologie.nix @@ -17,6 +17,7 @@ in { config = lib.mkIf cfg.enable { services.duplyBackup.profiles.isabelle_iridologie.rootDir = app.varDir; + services.duplyBackup.profiles.isabelle_iridologie.remotes = ["eriomem" "ovh"]; secrets.keys = [ { dest = "websites/isabelle/iridologie"; diff --git a/modules/private/websites/ludivine/production.nix b/modules/private/websites/ludivine/production.nix index b30f488..3a9895d 100644 --- a/modules/private/websites/ludivine/production.nix +++ b/modules/private/websites/ludivine/production.nix @@ -14,6 +14,7 @@ in { config = lib.mkIf cfg.enable { services.duplyBackup.profiles.ludivine_production.rootDir = app.varDir; + services.duplyBackup.profiles.ludivine_production.remotes = ["eriomem" "ovh"]; services.webstats.sites = [ { name = "ludivinecassal.com"; } ]; services.phpApplication.apps.ludivine_production = { websiteEnv = "production"; diff --git a/modules/private/websites/piedsjaloux/production.nix b/modules/private/websites/piedsjaloux/production.nix index 03b9ec5..e12b046 100644 --- a/modules/private/websites/piedsjaloux/production.nix +++ b/modules/private/websites/piedsjaloux/production.nix @@ -15,6 +15,7 @@ in { config = lib.mkIf cfg.enable { services.duplyBackup.profiles.piedsjaloux_production.rootDir = app.varDir; + services.duplyBackup.profiles.piedsjaloux_production.remotes = ["eriomem" "ovh"]; services.webstats.sites = [ { name = "piedsjaloux.fr"; } ]; services.phpApplication.apps.piedsjaloux_production = { websiteEnv = "production"; diff --git a/modules/private/websites/richie/production.nix b/modules/private/websites/richie/production.nix index a6957af..2d85175 100644 --- a/modules/private/websites/richie/production.nix +++ b/modules/private/websites/richie/production.nix @@ -26,6 +26,7 @@ in options.myServices.websites.richie.production.enable = lib.mkEnableOption "enable Richie's website"; config = lib.mkIf cfg.enable { services.duplyBackup.profiles.richie_production.rootDir = vardir; + services.duplyBackup.profiles.richie_production.remotes = ["eriomem" "ovh"]; services.webstats.sites = [ { name = "europe-richie.org"; } ]; secrets.keys = [{ diff --git a/modules/private/websites/syden/peertube.nix b/modules/private/websites/syden/peertube.nix index b17e775..5970cca 100644 --- a/modules/private/websites/syden/peertube.nix +++ b/modules/private/websites/syden/peertube.nix @@ -12,6 +12,7 @@ in config = lib.mkIf scfg.enable { services.duplyBackup.profiles.syden_peertube = { rootDir = dataDir; + remotes = ["eriomem" "ovh"]; }; users.users.peertube = { uid = config.ids.uids.peertube; -- 2.41.0