systems/eldiron/duply_backup.nix

   1 { lib, pkgs, config, name, ... }:
   2
   3 let
   4   cfg = config.myEnv.backup;
   5   varDir = "/var/lib/duply";
   6   default_action = "pre_bkp_purge_purgeFull_purgeIncr";
   7   duply_backup_full_with_ignored = pkgs.writeScriptBin "duply_full_with_ignored" ''
   8     #!${pkgs.stdenv.shell}
   9
  10     export DUPLY_FULL_BACKUP_WITH_IGNORED=yes
  11     if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then
  12       echo "duply_full_with_ignored /path/to/profile"
  13       echo "Does a full backup including directories with .duplicity-ignore"
  14       exit 1
  15     fi
  16     ${pkgs.duply}/bin/duply "$1" pre_full --force
  17   '';
  18   duply_backup = pkgs.writeScriptBin "duply_backup" ''
  19     #!${pkgs.stdenv.shell}
  20
  21     declare -a profiles
  22     profiles=()
  23     ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [
  24       ''profiles+=("${remote}_${k}")''
  25     ]) v.remotes) config.services.duplyBackup.profiles))}
  26
  27     if [ -f "${varDir}/last_backup_profile" ]; then
  28       last_backup=$(cat ${varDir}/last_backup_profile)
  29       for i in "''${!profiles[@]}"; do
  30         if [[ "''${profiles[$i]}" = "$last_backup" ]]; then
  31           break
  32         fi
  33       done
  34       ((i+=1))
  35       profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}")
  36     fi
  37
  38     # timeout in minutes
  39     timeout="''${1:-180}"
  40     timeout_timestamp=$(date +%s -d "$timeout minutes")
  41     for profile in "''${profiles[@]}"; do
  42       if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then
  43         break
  44       fi
  45
  46       touch "${varDir}/$profile.log"
  47       ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/$profile/ ${default_action} --force >> ${varDir}/$profile.log
  48       [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2
  49       echo "$profile" > ${varDir}/last_backup_profile
  50     done
  51   '';
  52
  53   check_backups = pkgs.writeScriptBin "duply_list_not_backuped" ''
  54     #!${pkgs.stdenv.shell}
  55
  56     do_check() {
  57       local dir="$1" path ignored_path
  58       find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do
  59         if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."backup/backuped_list"}; then
  60           continue
  61         elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."backup/backuped_list"}; then
  62           do_check "$path"
  63         else
  64           while IFS= read -r ignored_path; do
  65             if [[ "$path" =~ ^$ignored_path$ ]]; then
  66               continue 2
  67             fi
  68           done < ${config.secrets.fullPaths."backup/ignored_list"}
  69           printf '%s\n' "$path"
  70         fi
  71       done
  72     }
  73
  74     do_check /var/lib
  75   '';
  76   duplyProfile = profile: remote: bucket: let
  77     remote' = cfg.remotes.${remote};
  78   in ''
  79     if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then
  80       GPG_PW="${cfg.password}"
  81     fi
  82     TARGET="${remote'.remote bucket}"
  83     ${lib.optionalString (remote'.remote_type == "s3") ''
  84     export AWS_ACCESS_KEY_ID="${remote'.s3AccessKeyId}"
  85     export AWS_SECRET_ACCESS_KEY="${remote'.s3SecretAccessKey}"
  86     ''}
  87     ${lib.optionalString (remote'.remote_type == "rsync") ''
  88     DUPL_PARAMS="$DUPL_PARAMS --ssh-options=-oIdentityFile='${config.secrets.fullPaths."backup/identity"}' "
  89     ''}
  90     SOURCE="${profile.rootDir}"
  91     if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then
  92       FILENAME=".duplicity-ignore"
  93       DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'"
  94     fi
  95     VERBOSITY=4
  96     ARCH_DIR="${varDir}/caches"
  97     DUPL_PYTHON_BIN=""
  98
  99     # Do a full backup after 6 month
 100     MAX_FULLBKP_AGE=6M
 101     DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --full-if-older-than $MAX_FULLBKP_AGE "
 102     # Backups older than 1months are deleted
 103     MAX_AGE=1M
 104     # Keep 1 full backup
 105     MAX_FULL_BACKUPS=1
 106     MAX_FULLS_WITH_INCRS=1
 107   '';
 108 in
 109 {
 110   options = {
 111     services.duplyBackup.enable = lib.mkOption {
 112       type = lib.types.bool;
 113       default = false;
 114       description = ''
 115         Whether to enable remote backups.
 116       '';
 117     };
 118     services.duplyBackup.profiles = lib.mkOption {
 119       type = lib.types.attrsOf (lib.types.submodule {
 120         options = {
 121           hash = lib.mkOption {
 122             type = lib.types.bool;
 123             default = true;
 124             description = ''
 125               Hash bucket and directory names
 126             '';
 127           };
 128           excludeRootDir = lib.mkOption {
 129             type = lib.types.bool;
 130             default = true;
 131             description = ''
 132               Exclude root dir in exclusion file
 133             '';
 134           };
 135           rootDir = lib.mkOption {
 136             type = lib.types.path;
 137             default = "/var/lib";
 138             description = ''
 139               Path to backup
 140               '';
 141           };
 142           bucket = lib.mkOption {
 143             type = lib.types.str;
 144             description = ''
 145               Bucket to use
 146               '';
 147           };
 148           remotes = lib.mkOption {
 149             type = lib.types.listOf lib.types.str;
 150             description = ''
 151               Remotes to use for backup
 152               '';
 153           };
 154           includedPaths = lib.mkOption {
 155             type = lib.types.listOf lib.types.str;
 156             default = [];
 157             description = ''
 158               Included paths (subdirs of rootDir)
 159             '';
 160           };
 161           excludeFile = lib.mkOption {
 162             type = lib.types.lines;
 163             default = "";
 164             description = ''
 165               Content to put in exclude file
 166               '';
 167           };
 168           ignoredPaths = lib.mkOption {
 169             type = lib.types.listOf lib.types.str;
 170             default = [];
 171             description = ''
 172               List of paths to ignore when checking non-backed-up directories
 173               Can use (POSIX extended) regex
 174             '';
 175           };
 176         };
 177       });
 178     };
 179   };
 180
 181   config = lib.mkIf config.services.duplyBackup.enable {
 182     system.activationScripts.backup = ''
 183       install -m 0700 -o root -g root -d ${varDir} ${varDir}/caches
 184       '';
 185     secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v:
 186       let
 187         bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket;
 188       in map (remote: [
 189         (lib.nameValuePair "backup/${remote}_${k}/conf" {
 190           permissions = "0400";
 191           text = duplyProfile v remote bucket;
 192         })
 193         (lib.nameValuePair "backup/${remote}_${k}/exclude" {
 194           permissions = "0400";
 195           text = v.excludeFile + (builtins.concatStringsSep "\n" (map (p: "+ ${v.rootDir}/${p}") v.includedPaths)) + (lib.optionalString v.excludeRootDir ''
 196
 197             - **
 198           '');
 199         })
 200         (lib.nameValuePair "backup/${remote}_${k}/pre" {
 201           keyDependencies = [
 202             pkgs.bash
 203             pkgs.rsync
 204           ];
 205           permissions = "0500";
 206           text = let
 207             remote' = cfg.remotes.${remote};
 208           in ''
 209             #!${pkgs.stdenv.shell}
 210
 211             ${lib.optionalString (remote'.remote_type == "rsync") ''
 212               # Recreate directory structure before synchronizing
 213               mkdir -p ${varDir}/rsync_remotes/${remote}/${bucket}
 214               ${pkgs.rsync}/bin/rsync -av -e \
 215                 "ssh -p ${remote'.sshRsyncPort} -oIdentityFile=${config.secrets.fullPaths."backup/identity"}" \
 216                 "${varDir}/rsync_remotes/${remote}/" \
 217                 ${remote'.sshRsyncHost}:
 218             ''}
 219           '';
 220         })
 221         (lib.nameValuePair "backup/${remote}_${k}" {
 222           permissions = "0700";
 223           isDir = true;
 224         })
 225     ]) v.remotes) config.services.duplyBackup.profiles)) // {
 226       "backup/identity" = {
 227         permissions = "0400";
 228         text = "{{ .ssl_keys.duply_backup }}";
 229       };
 230       "backup/ignored_list" = {
 231         permissions = "0400";
 232         text = let
 233             ignored = map
 234               (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths)
 235               (builtins.attrValues config.services.duplyBackup.profiles);
 236           in builtins.concatStringsSep "\n" (lib.flatten ignored);
 237       };
 238       "backup/backuped_list" = {
 239         permissions = "0400";
 240         text = let
 241             included = map
 242               (v: map (p: "${v.rootDir}/${p}") v.includedPaths)
 243               (builtins.attrValues config.services.duplyBackup.profiles);
 244           in builtins.concatStringsSep "\n" (lib.flatten included);
 245       };
 246     };
 247
 248     programs.ssh.knownHostsFiles = [
 249       (pkgs.writeText
 250         "duply_backup_known_hosts"
 251         (builtins.concatStringsSep
 252           "\n"
 253           (builtins.filter
 254             (v: v != null)
 255             (builtins.map
 256               (v: v.sshKnownHosts)
 257               (builtins.attrValues cfg.remotes)
 258             )
 259           )
 260         )
 261       )
 262     ];
 263     environment.systemPackages = [ pkgs.duply check_backups duply_backup_full_with_ignored duply_backup ];
 264     services.cron = {
 265       enable = true;
 266       systemCronJobs = [
 267         "0 0 * * * root ${duply_backup}/bin/duply_backup 90"
 268       ];
 269
 270     };
 271
 272   };
 273 }