{ inputs.environment.url = "path:../environment"; inputs.secrets.url = "path:../../secrets"; outputs = { self, environment, secrets }: { nixosModule = self.nixosModules.borgBackup; nixosModules.borgBackup = { lib, pkgs, config, name, ... }: let cfg = config.myEnv.borg_backup; varDir = "/var/lib/borgbackup"; borg_args = "--encryption repokey --make-parent-dirs init create prune compact check"; borg_backup_full_with_ignored = pkgs.writeScriptBin "borg_full_with_ignored" '' #!${pkgs.stdenv.shell} if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then echo "borg_full_with_ignored /path/to/borgmatic.yaml" echo "Does a full backup including directories with .duplicity-ignore" exit 1 fi ${pkgs.borgmatic}/bin/borgmatic -c "$1" --override 'storage.archive_name_format="{hostname}-with-ignored-{now:%Y-%m-%dT%H:%M:%S.%f}"' --override 'location.exclude_if_present=[]' ${borg_args} ''; borg_backup = pkgs.writeScriptBin "borg_backup" '' #!${pkgs.stdenv.shell} declare -a profiles profiles=() ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [ ''profiles+=("${remote}_${k}")'' ]) v.remotes) config.services.borgBackup.profiles))} if [ -f "${varDir}/last_backup_profile" ]; then last_backup=$(cat ${varDir}/last_backup_profile) for i in "''${!profiles[@]}"; do if [[ "''${profiles[$i]}" = "$last_backup" ]]; then break fi done ((i+=1)) profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}") fi # timeout in minutes timeout="''${1:-180}" timeout_timestamp=$(date +%s -d "$timeout minutes") for profile in "''${profiles[@]}"; do if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then break fi touch "${varDir}/$profile.log" ${pkgs.borgmatic}/bin/borgmatic -c "${config.secrets.location}/borg_backup/$profile/borgmatic.yaml" ${borg_args} >> ${varDir}/$profile.log [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2 echo "$profile" > ${varDir}/last_backup_profile done ''; check_mysql_backups = pkgs.writeScriptBin "borg_list_mysql_not_backuped" '' #!${pkgs.stdenv.shell} comm -13 <(cat ${config.secrets.fullPaths."borg_backup/backuped_mysql_list"} | sort) <(mysql --defaults-file=${config.secrets.fullPaths."mysql_replication/eldiron/client"} -N -s -S /run/mysqld_eldiron/mysqld.sock -e "show databases;" | sort) ''; check_psql_backups = pkgs.writeScriptBin "borg_list_psql_not_backuped" '' #!${pkgs.stdenv.shell} comm -13 <(cat ${config.secrets.fullPaths."borg_backup/backuped_psql_list"} | sort) <(sudo -u postgres psql -h /backup2/eldiron/postgresql -Atq -c "SELECT datname FROM pg_catalog.pg_database" | sort) ''; check_backups = pkgs.writeScriptBin "borg_list_not_backuped" '' #!${pkgs.stdenv.shell} do_check() { local dir="$1" path ignored_path find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then continue elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then do_check "$path" else while IFS= read -r ignored_path; do if [[ "$path" =~ ^$ignored_path$ ]]; then continue 2 fi done < ${config.secrets.fullPaths."borg_backup/ignored_list"} printf '%s\n' "$path" fi done } do_check /var/lib ''; borgProfile = profile: remote: bucket: builtins.toJSON { location = { source_directories = map (p: "${profile.rootDir}/${p}") profile.includedPaths; repositories = [ { path = cfg.remotes.${remote}.remote name bucket; label = "backupserver"; } ]; one_file_system = false; exclude_if_present = [".duplicity-ignore"]; source_directories_must_exist = profile.directoriesMustExist; borgmatic_source_directory = "${varDir}/${profile.bucket}/.borgmatic"; }; hooks = { mysql_databases = profile.mariaDBDatabases; postgresql_databases = profile.postgresqlDatabases; before_backup = lib.optional (profile.postgresqlDatabasesPauseReplicationHost != null) "sudo -u postgres psql -h ${profile.postgresqlDatabasesPauseReplicationHost} -c 'SELECT pg_wal_replay_pause();' >/dev/null"; after_backup = lib.optional (profile.postgresqlDatabasesPauseReplicationHost != null) "sudo -u postgres psql -h ${profile.postgresqlDatabasesPauseReplicationHost} -c 'SELECT pg_wal_replay_resume();' >/dev/null"; }; storage = { encryption_passphrase = profile.password; ssh_command = "ssh -i ${config.secrets.fullPaths."borg_backup/identity"}"; compression = "zlib"; borg_base_directory = "${varDir}/${profile.bucket}"; relocated_repo_access_is_ok = true; }; retention = { keep_within = "10d"; keep_daily = 30; }; }; in { imports = [ environment.nixosModule secrets.nixosModule ]; options = { services.borgBackup.enable = lib.mkOption { type = lib.types.bool; default = false; description = '' Whether to enable remote backups. ''; }; services.borgBackup.cronSpec = lib.mkOption { type = lib.types.str; default = "0 0 * * *"; description = '' Cron spec for running borgbackup ''; }; services.borgBackup.helpers = lib.mkOption { readOnly = true; description = '' Some useful functions for borgBackup configuration ''; default = { mysqlDBFromBackup = name: primary: { inherit name; options = "--defaults-file=${config.secrets.fullPaths."mysql_replication/${primary}/mysqldump"} -S /run/mysqld_${primary}/mysqld.sock"; }; psqlDBWithSocket = name: socket: { inherit name; # do not use "host" otherwise it screws the path computation: # there’s a python code os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name) # and the first argument gets ignored since the second one # looks like an absolute path options = "--host ${socket}"; pg_dump_command = "sudo -u postgres pg_dump"; psql_command = "sudo -u postgres psql"; pg_restore_command = "sudo -u postgres pg_restore"; }; }; }; services.borgBackup.profiles = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule { options = { hash = lib.mkOption { type = lib.types.bool; default = true; description = '' Hash bucket and directory names ''; }; rootDir = lib.mkOption { type = lib.types.path; default = "/var/lib"; description = '' Path to backup ''; }; password = lib.mkOption { type = lib.types.str; default = cfg.password; description = '' password to use to encrypt data ''; }; directoriesMustExist = lib.mkOption { type = lib.types.bool; default = true; description = '' Raise error if backuped directory doesn't exist ''; }; bucket = lib.mkOption { type = lib.types.str; description = '' Bucket to use ''; }; remotes = lib.mkOption { type = lib.types.listOf lib.types.str; description = '' Remotes to use for backup ''; }; includedPaths = lib.mkOption { type = lib.types.listOf lib.types.str; default = []; description = '' Included paths (subdirs of rootDir) ''; }; mariaDBDatabases = lib.mkOption { type = lib.types.listOf (lib.types.attrsOf lib.types.str); default = []; description = '' MariaDB databases to backup ''; }; postgresqlDatabasesPauseReplicationHost = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; description = '' Replicatino host to pause before the backup ''; }; postgresqlDatabases = lib.mkOption { type = lib.types.listOf (lib.types.attrsOf lib.types.str); default = []; description = '' Postgresql databases to backup ''; }; excludeFile = lib.mkOption { type = lib.types.lines; default = ""; description = '' Content to put in exclude file ''; }; ignoredPaths = lib.mkOption { type = lib.types.listOf lib.types.str; default = []; description = '' List of paths to ignore when checking non-backed-up directories Can use (POSIX extended) regex ''; }; }; }); }; }; config = lib.mkIf config.services.borgBackup.enable { system.activationScripts.borg_backup = '' install -m 0700 -o root -g root -d ${varDir} ''; secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v: let bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket; in map (remote: [ (lib.nameValuePair "borg_backup/${remote}_${k}/borgmatic.yaml" { permissions = "0400"; text = borgProfile v remote bucket; }) (lib.nameValuePair "borg_backup/${remote}_${k}" { permissions = "0700"; isDir = true; }) ]) v.remotes) config.services.borgBackup.profiles)) // { "borg_backup/identity" = { permissions = "0400"; text = "{{ .ssl_keys.borg_backup }}"; }; "borg_backup/backuped_mysql_list" = { permissions = "0400"; text = builtins.concatStringsSep "\n" (lib.concatMap (v: (builtins.map (b: b.name) v.mariaDBDatabases)) (builtins.attrValues config.services.borgBackup.profiles)); }; "borg_backup/backuped_psql_list" = { permissions = "0400"; text = builtins.concatStringsSep "\n" (lib.concatMap (v: (builtins.map (b: b.name) v.postgresqlDatabases)) (builtins.attrValues config.services.borgBackup.profiles)); }; "borg_backup/ignored_list" = { permissions = "0400"; text = let ignored = map (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths) (builtins.attrValues config.services.borgBackup.profiles); in builtins.concatStringsSep "\n" (lib.flatten ignored); }; "borg_backup/backuped_list" = { permissions = "0400"; text = let included = map (v: map (p: "${v.rootDir}/${p}") v.includedPaths) (builtins.attrValues config.services.borgBackup.profiles); in builtins.concatStringsSep "\n" (lib.flatten included); }; }; programs.ssh.knownHostsFiles = [ (pkgs.writeText "borg_backup_known_hosts" (builtins.concatStringsSep "\n" (builtins.filter (v: v != null) (builtins.map (v: v.sshKnownHosts) (builtins.attrValues cfg.remotes) ) ) ) ) ]; environment.systemPackages = [ pkgs.borgbackup pkgs.borgmatic borg_backup_full_with_ignored borg_backup check_backups ] ++ (lib.optionals (name == "backup-2") [ check_mysql_backups check_psql_backups ]); services.cron = { enable = true; systemCronJobs = [ "${config.services.borgBackup.cronSpec} root ${borg_backup}/bin/borg_backup 300" ]; }; }; }; }; }