From 1c90c0dd73b5b74612be736ac8deeaa4547e2c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 10 Jun 2024 13:13:11 +0200 Subject: Migrate to borg backup --- systems/eldiron/duply_backup.nix | 278 ++++++++++++++++++++++++++++----------- 1 file changed, 200 insertions(+), 78 deletions(-) (limited to 'systems/eldiron/duply_backup.nix') diff --git a/systems/eldiron/duply_backup.nix b/systems/eldiron/duply_backup.nix index 590d125..5143302 100644 --- a/systems/eldiron/duply_backup.nix +++ b/systems/eldiron/duply_backup.nix @@ -3,29 +3,108 @@ let cfg = config.myEnv.backup; varDir = "/var/lib/duply"; - duplyProfile = profile: remote: prefix: '' - GPG_PW="${cfg.password}" - TARGET="${cfg.remotes.${remote}.remote profile.bucket}${prefix}" - export AWS_ACCESS_KEY_ID="${cfg.remotes.${remote}.accessKeyId}" - export AWS_SECRET_ACCESS_KEY="${cfg.remotes.${remote}.secretAccessKey}" + default_action = "pre_bkp_purge_purgeFull_purgeIncr"; + duply_backup_full_with_ignored = pkgs.writeScriptBin "duply_full_with_ignored" '' + #!${pkgs.stdenv.shell} + + export DUPLY_FULL_BACKUP_WITH_IGNORED=yes + if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then + echo "duply_full_with_ignored /path/to/profile" + echo "Does a full backup including directories with .duplicity-ignore" + exit 1 + fi + ${pkgs.duply}/bin/duply "$1" pre_full --force + ''; + duply_backup = pkgs.writeScriptBin "duply_backup" '' + #!${pkgs.stdenv.shell} + + declare -a profiles + profiles=() + ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [ + ''profiles+=("${remote}_${k}")'' + ]) v.remotes) config.services.duplyBackup.profiles))} + + if [ -f "${varDir}/last_backup_profile" ]; then + last_backup=$(cat ${varDir}/last_backup_profile) + for i in "''${!profiles[@]}"; do + if [[ "''${profiles[$i]}" = "$last_backup" ]]; then + break + fi + done + ((i+=1)) + profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}") + fi + + # timeout in minutes + timeout="''${1:-180}" + timeout_timestamp=$(date +%s -d "$timeout minutes") + for profile in "''${profiles[@]}"; do + if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then + break + fi + + touch "${varDir}/$profile.log" + ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/$profile/ ${default_action} --force >> ${varDir}/$profile.log + [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2 + echo "$profile" > ${varDir}/last_backup_profile + done + ''; + + check_backups = pkgs.writeScriptBin "duply_list_not_backuped" '' + #!${pkgs.stdenv.shell} + + do_check() { + local dir="$1" path ignored_path + find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do + if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."backup/backuped_list"}; then + continue + elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."backup/backuped_list"}; then + do_check "$path" + else + while IFS= read -r ignored_path; do + if [[ "$path" =~ ^$ignored_path$ ]]; then + continue 2 + fi + done < ${config.secrets.fullPaths."backup/ignored_list"} + printf '%s\n' "$path" + fi + done + } + + do_check /var/lib + ''; + duplyProfile = profile: remote: bucket: let + remote' = cfg.remotes.${remote}; + in '' + if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then + GPG_PW="${cfg.password}" + fi + TARGET="${remote'.remote bucket}" + ${lib.optionalString (remote'.remote_type == "s3") '' + export AWS_ACCESS_KEY_ID="${remote'.s3AccessKeyId}" + export AWS_SECRET_ACCESS_KEY="${remote'.s3SecretAccessKey}" + ''} + ${lib.optionalString (remote'.remote_type == "rsync") '' + DUPL_PARAMS="$DUPL_PARAMS --ssh-options=-oIdentityFile='${config.secrets.fullPaths."backup/identity"}' " + ''} SOURCE="${profile.rootDir}" - FILENAME=".duplicity-ignore" - DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'" + if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then + FILENAME=".duplicity-ignore" + DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'" + fi VERBOSITY=4 ARCH_DIR="${varDir}/caches" + DUPL_PYTHON_BIN="" - # Do a full backup after 1 month - MAX_FULLBKP_AGE=1M - DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --exclude-other-filesystems --full-if-older-than $MAX_FULLBKP_AGE " - # Backups older than 2months are deleted - MAX_AGE=2M - # Keep 2 full backups - MAX_FULL_BACKUPS=2 - MAX_FULLS_WITH_INCRS=2 + # Do a full backup after 6 month + MAX_FULLBKP_AGE=6M + DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --full-if-older-than $MAX_FULLBKP_AGE " + # Backups older than 1months are deleted + MAX_AGE=1M + # Keep 1 full backup + MAX_FULL_BACKUPS=1 + MAX_FULLS_WITH_INCRS=1 ''; - action = "bkp_purge_purgeFull_purgeIncr"; - varName = k: remoteName: - if remoteName == "eriomem" then k else remoteName + "_" + k; in { options = { @@ -39,26 +118,46 @@ in services.duplyBackup.profiles = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule { options = { + hash = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Hash bucket and directory names + ''; + }; + excludeRootDir = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Exclude root dir in exclusion file + ''; + }; rootDir = lib.mkOption { type = lib.types.path; + default = "/var/lib"; description = '' Path to backup ''; }; bucket = lib.mkOption { type = lib.types.str; - default = "immae-${name}"; description = '' Bucket to use ''; }; remotes = lib.mkOption { type = lib.types.listOf lib.types.str; - default = ["eriomem"]; description = '' Remotes to use for backup ''; }; + includedPaths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = '' + Included paths (subdirs of rootDir) + ''; + }; excludeFile = lib.mkOption { type = lib.types.lines; default = ""; @@ -66,6 +165,14 @@ in Content to put in exclude file ''; }; + ignoredPaths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = '' + List of paths to ignore when checking non-backed-up directories + Can use (POSIX extended) regex + ''; + }; }; }); }; @@ -76,76 +183,91 @@ in install -m 0700 -o root -g root -d ${varDir} ${varDir}/caches ''; secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v: - map (remote: [ - (lib.nameValuePair "backup/${varName k remote}/conf" { + let + bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket; + in map (remote: [ + (lib.nameValuePair "backup/${remote}_${k}/conf" { permissions = "0400"; - text = duplyProfile v remote "${k}/"; + text = duplyProfile v remote bucket; }) - (lib.nameValuePair "backup/${varName k remote}/exclude" { + (lib.nameValuePair "backup/${remote}_${k}/exclude" { permissions = "0400"; - text = v.excludeFile; + text = v.excludeFile + (builtins.concatStringsSep "\n" (map (p: "+ ${v.rootDir}/${p}") v.includedPaths)) + (lib.optionalString v.excludeRootDir '' + + - ** + ''); }) - (lib.nameValuePair "backup/${varName k remote}" { + (lib.nameValuePair "backup/${remote}_${k}/pre" { + keyDependencies = [ + pkgs.bash + pkgs.rsync + ]; permissions = "0500"; + text = let + remote' = cfg.remotes.${remote}; + in '' + #!${pkgs.stdenv.shell} + + ${lib.optionalString (remote'.remote_type == "rsync") '' + # Recreate directory structure before synchronizing + mkdir -p ${varDir}/rsync_remotes/${remote}/${bucket} + ${pkgs.rsync}/bin/rsync -av -e \ + "ssh -p ${remote'.sshRsyncPort} -oIdentityFile=${config.secrets.fullPaths."backup/identity"}" \ + "${varDir}/rsync_remotes/${remote}/" \ + ${remote'.sshRsyncHost}: + ''} + ''; + }) + (lib.nameValuePair "backup/${remote}_${k}" { + permissions = "0700"; isDir = true; }) - ]) v.remotes) config.services.duplyBackup.profiles)); + ]) v.remotes) config.services.duplyBackup.profiles)) // { + "backup/identity" = { + permissions = "0400"; + text = "{{ .ssl_keys.duply_backup }}"; + }; + "backup/ignored_list" = { + permissions = "0400"; + text = let + ignored = map + (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths) + (builtins.attrValues config.services.duplyBackup.profiles); + in builtins.concatStringsSep "\n" (lib.flatten ignored); + }; + "backup/backuped_list" = { + permissions = "0400"; + text = let + included = map + (v: map (p: "${v.rootDir}/${p}") v.includedPaths) + (builtins.attrValues config.services.duplyBackup.profiles); + in builtins.concatStringsSep "\n" (lib.flatten included); + }; + }; + programs.ssh.knownHostsFiles = [ + (pkgs.writeText + "duply_backup_known_hosts" + (builtins.concatStringsSep + "\n" + (builtins.filter + (v: v != null) + (builtins.map + (v: v.sshKnownHosts) + (builtins.attrValues cfg.remotes) + ) + ) + ) + ) + ]; + environment.systemPackages = [ pkgs.duply check_backups duply_backup_full_with_ignored duply_backup ]; services.cron = { enable = true; - systemCronJobs = let - backups = pkgs.writeScript "backups" '' - #!${pkgs.stdenv.shell} - - ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: - map (remote: [ - '' - touch ${varDir}/${varName k remote}.log - ${pkgs.duply}/bin/duply ${config.secrets.fullPaths."backup/${varName k remote}"}/ ${action} --force >> ${varDir}/${varName k remote}.log - [[ $? = 0 ]] || echo -e "Error when doing backup for ${varName k remote}, see above\n---------------------------------------" >&2 - '' - ]) v.remotes - ) config.services.duplyBackup.profiles))} - ''; - in - [ - "0 2 * * * root ${backups}" - ]; + systemCronJobs = [ + "0 0 * * * root ${duply_backup}/bin/duply_backup 90" + ]; }; - security.pki.certificateFiles = [ - (pkgs.fetchurl { - url = "http://downloads.e.eriomem.net/eriomemca.pem"; - sha256 = "1ixx4c6j3m26j8dp9a3dkvxc80v1nr5aqgmawwgs06bskasqkvvh"; - }) - ]; - - myServices.monitoring.fromMasterActivatedPlugins = [ "eriomem" ]; - myServices.monitoring.fromMasterObjects.service = [ - { - service_description = "eriomem backup is up and not full"; - host_name = config.hostEnv.fqdn; - use = "external-service"; - check_command = "check_backup_eriomem"; - - check_interval = 120; - notification_interval = "1440"; - - servicegroups = "webstatus-backup"; - } - - { - service_description = "ovh backup is up and not full"; - host_name = config.hostEnv.fqdn; - use = "external-service"; - check_command = "check_ok"; - - check_interval = 120; - notification_interval = "1440"; - - servicegroups = "webstatus-backup"; - } - ]; }; } -- cgit v1.2.3