path: root/flakes/private/borg_backup/flake.nix

{
  inputs.environment.url = "path:../environment";
  inputs.secrets.url = "path:../../secrets";

  outputs = { self, environment, secrets }: {
    nixosModule = self.nixosModules.borgBackup;
    nixosModules.borgBackup = { lib, pkgs, config, name, ... }:
      let
        cfg = config.myEnv.borg_backup;
        varDir = "/var/lib/borgbackup";
        borg_args = "--encryption repokey --make-parent-dirs init create prune compact check";
        borg_backup_full_with_ignored = pkgs.writeScriptBin "borg_full_with_ignored" ''
          #!${pkgs.stdenv.shell}

          if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then
            echo "borg_full_with_ignored /path/to/borgmatic.yaml"
            echo "Does a full backup including directories with .duplicity-ignore"
            exit 1
          fi
          ${pkgs.borgmatic}/bin/borgmatic -c "$1" --override 'storage.archive_name_format="{hostname}-with-ignored-{now:%Y-%m-%dT%H:%M:%S.%f}"' --override 'location.exclude_if_present=[]' ${borg_args}
        '';
        borg_backup = pkgs.writeScriptBin "borg_backup" ''
          #!${pkgs.stdenv.shell}

          declare -a profiles
          profiles=()
          ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [
            ''profiles+=("${remote}_${k}")''
          ]) v.remotes) config.services.borgBackup.profiles))}

          if [ -f "${varDir}/last_backup_profile" ]; then
            last_backup=$(cat ${varDir}/last_backup_profile)
            for i in "''${!profiles[@]}"; do
              if [[ "''${profiles[$i]}" = "$last_backup" ]]; then
                break
              fi
            done
            ((i+=1))
            profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}")
          fi

          # timeout in minutes
          timeout="''${1:-180}"
          timeout_timestamp=$(date +%s -d "$timeout minutes")
          for profile in "''${profiles[@]}"; do
            if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then
              break
            fi

            touch "${varDir}/$profile.log"
            ${pkgs.borgmatic}/bin/borgmatic -c "${config.secrets.location}/borg_backup/$profile/borgmatic.yaml" ${borg_args} >> ${varDir}/$profile.log
            [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2
            echo "$profile" > ${varDir}/last_backup_profile
          done
        '';

        check_mysql_backups = pkgs.writeScriptBin "borg_list_mysql_not_backuped" ''
          #!${pkgs.stdenv.shell}

          comm -13 <(cat ${config.secrets.fullPaths."borg_backup/backuped_mysql_list"} | sort) <(mysql --defaults-file=${config.secrets.fullPaths."mysql_replication/eldiron/client"} -N -s -S /run/mysqld_eldiron/mysqld.sock -e "show databases;" | sort)
        '';
        check_psql_backups = pkgs.writeScriptBin "borg_list_psql_not_backuped" ''
          #!${pkgs.stdenv.shell}

          comm -13 <(cat ${config.secrets.fullPaths."borg_backup/backuped_psql_list"} | sort) <(sudo -u postgres psql -h /backup2/eldiron/postgresql -Atq -c "SELECT datname FROM pg_catalog.pg_database" | sort)
        '';
        check_backups = pkgs.writeScriptBin "borg_list_not_backuped" ''
          #!${pkgs.stdenv.shell}

          do_check() {
            local dir="$1" path ignored_path
            find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do
              if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then
                continue
              elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then
                do_check "$path"
              else
                while IFS= read -r ignored_path; do
                  if [[ "$path" =~ ^$ignored_path$ ]]; then
                    continue 2
                  fi
                done < ${config.secrets.fullPaths."borg_backup/ignored_list"}
                printf '%s\n' "$path"
              fi
            done
          }

          do_check /var/lib
        '';
        borgProfile = profile: remote: bucket: builtins.toJSON {
          location = {
            source_directories = map (p: "${profile.rootDir}/${p}") profile.includedPaths;
            repositories = [
              { path = cfg.remotes.${remote}.remote name bucket; label = "backupserver"; }
            ];
            one_file_system = false;
            exclude_if_present = [".duplicity-ignore"];
            source_directories_must_exist = profile.directoriesMustExist;
            borgmatic_source_directory = "${varDir}/${profile.bucket}/.borgmatic";
          };
          hooks = {
            mysql_databases = profile.mariaDBDatabases;
            postgresql_databases = profile.postgresqlDatabases;
            before_backup = lib.optional (profile.postgresqlDatabasesPauseReplicationHost != null) "sudo -u postgres psql -h ${profile.postgresqlDatabasesPauseReplicationHost} -c 'SELECT pg_wal_replay_pause();' >/dev/null";
            after_backup = lib.optional (profile.postgresqlDatabasesPauseReplicationHost != null) "sudo -u postgres psql -h ${profile.postgresqlDatabasesPauseReplicationHost} -c 'SELECT pg_wal_replay_resume();' >/dev/null";
          };
          storage = {
            encryption_passphrase = profile.password;
            ssh_command = "ssh -i ${config.secrets.fullPaths."borg_backup/identity"}";
            compression = "zlib";
            borg_base_directory = "${varDir}/${profile.bucket}";
            relocated_repo_access_is_ok = true;
          };
          retention = {
            keep_within = "10d";
            keep_daily = 30;
          };
        };
      in
      {
        imports = [
          environment.nixosModule
          secrets.nixosModule
        ];
        options = {
          services.borgBackup.enable = lib.mkOption {
            type = lib.types.bool;
            default = false;
            description = ''
              Whether to enable remote backups.
            '';
          };
          services.borgBackup.cronSpec = lib.mkOption {
            type = lib.types.str;
            default = "0 0 * * *";
            description = ''
              Cron spec for running borgbackup
            '';
          };
          services.borgBackup.helpers = lib.mkOption {
            readOnly = true;
            description = ''
              Some useful functions for borgBackup configuration
            '';
            default = {
              mysqlDBFromBackup = name: primary: {
                inherit name;
                options = "--defaults-file=${config.secrets.fullPaths."mysql_replication/${primary}/mysqldump"} -S /run/mysqld_${primary}/mysqld.sock";
              };
              psqlDBWithSocket = name: socket: {
                inherit name;
                # do not use "host" otherwise it screws the path computation:
                # there’s a python code os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
                # and the first argument gets ignored since the second one
                # looks like an absolute path
                options = "--host ${socket}";
                pg_dump_command = "sudo -u postgres pg_dump";
                psql_command = "sudo -u postgres psql";
                pg_restore_command = "sudo -u postgres pg_restore";
              };
            };
          };
          services.borgBackup.profiles = lib.mkOption {
            type = lib.types.attrsOf (lib.types.submodule {
              options = {
                hash = lib.mkOption {
                  type = lib.types.bool;
                  default = true;
                  description = ''
                    Hash bucket and directory names
                  '';
                };
                rootDir = lib.mkOption {
                  type = lib.types.path;
                  default = "/var/lib";
                  description = ''
                    Path to backup
                    '';
                };
                password = lib.mkOption {
                  type = lib.types.str;
                  default = cfg.password;
                  description = ''
                    password to use to encrypt data
                  '';
                };
                directoriesMustExist = lib.mkOption {
                  type = lib.types.bool;
                  default = true;
                  description = ''
                    Raise error if backuped directory doesn't exist
                  '';
                };
                bucket = lib.mkOption {
                  type = lib.types.str;
                  description = ''
                    Bucket to use
                    '';
                };
                remotes = lib.mkOption {
                  type = lib.types.listOf lib.types.str;
                  description = ''
                    Remotes to use for backup
                    '';
                };
                includedPaths = lib.mkOption {
                  type = lib.types.listOf lib.types.str;
                  default = [];
                  description = ''
                    Included paths (subdirs of rootDir)
                  '';
                };
                mariaDBDatabases = lib.mkOption {
                  type = lib.types.listOf (lib.types.attrsOf lib.types.str);
                  default = [];
                  description = ''
                    MariaDB databases to backup
                  '';
                };
                postgresqlDatabasesPauseReplicationHost = lib.mkOption {
                  type = lib.types.nullOr lib.types.str;
                  default = null;
                  description = ''
                    Replicatino host to pause before the backup
                  '';
                };
                postgresqlDatabases = lib.mkOption {
                  type = lib.types.listOf (lib.types.attrsOf lib.types.str);
                  default = [];
                  description = ''
                    Postgresql databases to backup
                  '';
                };
                excludeFile = lib.mkOption {
                  type = lib.types.lines;
                  default = "";
                  description = ''
                    Content to put in exclude file
                    '';
                };
                ignoredPaths = lib.mkOption {
                  type = lib.types.listOf lib.types.str;
                  default = [];
                  description = ''
                    List of paths to ignore when checking non-backed-up directories
                    Can use (POSIX extended) regex
                  '';
                };
              };
            });
          };
        };

        config = lib.mkIf config.services.borgBackup.enable {
          system.activationScripts.borg_backup = ''
            install -m 0700 -o root -g root -d ${varDir}
            '';
          secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v:
            let
              bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket;
            in map (remote: [
              (lib.nameValuePair "borg_backup/${remote}_${k}/borgmatic.yaml" {
                permissions = "0400";
                text = borgProfile v remote bucket;
              })
              (lib.nameValuePair "borg_backup/${remote}_${k}" {
                permissions = "0700";
                isDir = true;
              })
          ]) v.remotes) config.services.borgBackup.profiles)) // {
            "borg_backup/identity" = {
              permissions = "0400";
              text = "{{ .ssl_keys.borg_backup }}";
            };
            "borg_backup/backuped_mysql_list" = {
              permissions = "0400";
              text = builtins.concatStringsSep "\n" (lib.concatMap (v: (builtins.map (b: b.name) v.mariaDBDatabases)) (builtins.attrValues config.services.borgBackup.profiles));
            };
            "borg_backup/backuped_psql_list" = {
              permissions = "0400";
              text = builtins.concatStringsSep "\n" (lib.concatMap (v: (builtins.map (b: b.name) v.postgresqlDatabases)) (builtins.attrValues config.services.borgBackup.profiles));
            };
            "borg_backup/ignored_list" = {
              permissions = "0400";
              text = let
                  ignored = map
                    (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths)
                    (builtins.attrValues config.services.borgBackup.profiles);
                in builtins.concatStringsSep "\n" (lib.flatten ignored);
            };
            "borg_backup/backuped_list" = {
              permissions = "0400";
              text = let
                  included = map
                    (v: map (p: "${v.rootDir}/${p}") v.includedPaths)
                    (builtins.attrValues config.services.borgBackup.profiles);
                in builtins.concatStringsSep "\n" (lib.flatten included);
            };
          };

          programs.ssh.knownHostsFiles = [
            (pkgs.writeText
              "borg_backup_known_hosts"
              (builtins.concatStringsSep
                "\n"
                (builtins.filter
                  (v: v != null)
                  (builtins.map
                    (v: v.sshKnownHosts)
                    (builtins.attrValues cfg.remotes)
                  )
                )
              )
            )
          ];
          environment.systemPackages = [ pkgs.borgbackup pkgs.borgmatic borg_backup_full_with_ignored borg_backup check_backups ] ++ (lib.optionals (name == "backup-2") [ check_mysql_backups check_psql_backups ]);
          services.cron = {
            enable = true;
            systemCronJobs = [
              "${config.services.borgBackup.cronSpec} root ${borg_backup}/bin/borg_backup 300"
            ];

          };

        };
      };
  };
}