Migrate to borg backup

[perso/Immae/Config/Nix.git] / systems / eldiron / borg_backup.nix

{ lib, pkgs, config, name, ... }:

let
  cfg = config.myEnv.borg_backup;
  varDir = "/var/lib/borgbackup";
  borg_args = "--encryption repokey --make-parent-dirs init create prune compact check";
  borg_backup_full_with_ignored = pkgs.writeScriptBin "borg_full_with_ignored" ''
    #!${pkgs.stdenv.shell}

    if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then
      echo "borg_full_with_ignored /path/to/borgmatic.yaml"
      echo "Does a full backup including directories with .duplicity-ignore"
      exit 1
    fi
    ${pkgs.borgmatic}/bin/borgmatic -c "$1" --override 'storage.archive_name_format="{hostname}-with-ignored-{now:%Y-%m-%dT%H:%M:%S.%f}"' --override 'location.exclude_if_present=[]' ${borg_args}
  '';
  borg_backup = pkgs.writeScriptBin "borg_backup" ''
    #!${pkgs.stdenv.shell}

    declare -a profiles
    profiles=()
    ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [
      ''profiles+=("${remote}_${k}")''
    ]) v.remotes) config.services.borgBackup.profiles))}

    if [ -f "${varDir}/last_backup_profile" ]; then
      last_backup=$(cat ${varDir}/last_backup_profile)
      for i in "''${!profiles[@]}"; do
        if [[ "''${profiles[$i]}" = "$last_backup" ]]; then
          break
        fi
      done
      ((i+=1))
      profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}")
    fi

    # timeout in minutes
    timeout="''${1:-180}"
    timeout_timestamp=$(date +%s -d "$timeout minutes")
    for profile in "''${profiles[@]}"; do
      if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then
        break
      fi

      touch "${varDir}/$profile.log"
      ${pkgs.borgmatic}/bin/borgmatic -c "${config.secrets.location}/borg_backup/$profile/borgmatic.yaml" ${borg_args} >> ${varDir}/$profile.log
      [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2
      echo "$profile" > ${varDir}/last_backup_profile
    done
  '';

  check_backups = pkgs.writeScriptBin "borg_list_not_backuped" ''
    #!${pkgs.stdenv.shell}

    do_check() {
      local dir="$1" path ignored_path
      find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do
        if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then
          continue
        elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then
          do_check "$path"
        else
          while IFS= read -r ignored_path; do
            if [[ "$path" =~ ^$ignored_path$ ]]; then
              continue 2
            fi
          done < ${config.secrets.fullPaths."borg_backup/ignored_list"}
          printf '%s\n' "$path"
        fi
      done
    }

    do_check /var/lib
  '';
  borgProfile = profile: remote: bucket: builtins.toJSON {
    location = {
      source_directories = map (p: "${profile.rootDir}/${p}") profile.includedPaths;
      repositories = [
        { path = cfg.remotes.${remote}.remote bucket; label = "backupserver"; }
      ];
      one_file_system = false;
      exclude_if_present = [".duplicity-ignore"];
      source_directories_must_exist = profile.directoriesMustExist;
      borgmatic_source_directory = "${varDir}/${profile.bucket}/.borgmatic";
    };
    storage = {
      encryption_passphrase = profile.password;
      ssh_command = "ssh -i ${config.secrets.fullPaths."borg_backup/identity"}";
      compression = "zlib";
      borg_base_directory = "${varDir}/${profile.bucket}";
    };
    retention = {
      keep_within = "10d";
      keep_daily = 30;
    };
  };
in
{
  options = {
    services.borgBackup.enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Whether to enable remote backups.
      '';
    };
    services.borgBackup.profiles = lib.mkOption {
      type = lib.types.attrsOf (lib.types.submodule {
        options = {
          hash = lib.mkOption {
            type = lib.types.bool;
            default = true;
            description = ''
              Hash bucket and directory names
            '';
          };
          rootDir = lib.mkOption {
            type = lib.types.path;
            default = "/var/lib";
            description = ''
              Path to backup
              '';
          };
          password = lib.mkOption {
            type = lib.types.str;
            default = cfg.password;
            description = ''
              password to use to encrypt data
            '';
          };
          directoriesMustExist = lib.mkOption {
            type = lib.types.bool;
            default = true;
            description = ''
              Raise error if backuped directory doesn't exist
            '';
          };
          bucket = lib.mkOption {
            type = lib.types.str;
            description = ''
              Bucket to use
              '';
          };
          remotes = lib.mkOption {
            type = lib.types.listOf lib.types.str;
            description = ''
              Remotes to use for backup
              '';
          };
          includedPaths = lib.mkOption {
            type = lib.types.listOf lib.types.str;
            default = [];
            description = ''
              Included paths (subdirs of rootDir)
            '';
          };
          excludeFile = lib.mkOption {
            type = lib.types.lines;
            default = "";
            description = ''
              Content to put in exclude file
              '';
          };
          ignoredPaths = lib.mkOption {
            type = lib.types.listOf lib.types.str;
            default = [];
            description = ''
              List of paths to ignore when checking non-backed-up directories
              Can use (POSIX extended) regex
            '';
          };
        };
      });
    };
  };

  config = lib.mkIf config.services.borgBackup.enable {
    system.activationScripts.borg_backup = ''
      install -m 0700 -o root -g root -d ${varDir}
      '';
    secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v:
      let
        bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket;
      in map (remote: [
        (lib.nameValuePair "borg_backup/${remote}_${k}/borgmatic.yaml" {
          permissions = "0400";
          text = borgProfile v remote bucket;
        })
        (lib.nameValuePair "borg_backup/${remote}_${k}" {
          permissions = "0700";
          isDir = true;
        })
    ]) v.remotes) config.services.borgBackup.profiles)) // {
      "borg_backup/identity" = {
        permissions = "0400";
        text = "{{ .ssl_keys.borg_backup }}";
      };
      "borg_backup/ignored_list" = {
        permissions = "0400";
        text = let
            ignored = map
              (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths)
              (builtins.attrValues config.services.borgBackup.profiles);
          in builtins.concatStringsSep "\n" (lib.flatten ignored);
      };
      "borg_backup/backuped_list" = {
        permissions = "0400";
        text = let
            included = map
              (v: map (p: "${v.rootDir}/${p}") v.includedPaths)
              (builtins.attrValues config.services.borgBackup.profiles);
          in builtins.concatStringsSep "\n" (lib.flatten included);
      };
    };

    programs.ssh.knownHostsFiles = [
      (pkgs.writeText
        "borg_backup_known_hosts"
        (builtins.concatStringsSep
          "\n"
          (builtins.filter
            (v: v != null)
            (builtins.map
              (v: v.sshKnownHosts)
              (builtins.attrValues cfg.remotes)
            )
          )
        )
      )
    ];
    environment.systemPackages = [ pkgs.borgbackup pkgs.borgmatic borg_backup_full_with_ignored borg_backup check_backups ];
    services.cron = {
      enable = true;
      systemCronJobs = [
        "0 0 * * * root ${borg_backup}/bin/borg_backup 300"
      ];

    };

  };
}