From 1c90c0dd73b5b74612be736ac8deeaa4547e2c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 10 Jun 2024 13:13:11 +0200 Subject: Migrate to borg backup --- systems/backup-2/databases/mariadb_replication.nix | 1 - systems/backup-2/flake.lock | 26 +- systems/dilion/flake.lock | 10 +- systems/eldiron/base.nix | 14 ++ systems/eldiron/borg_backup.nix | 241 ++++++++++++++++++ systems/eldiron/databases/openldap/default.nix | 3 + systems/eldiron/duply_backup.nix | 278 +++++++++++++++------ systems/eldiron/flake.lock | 26 +- systems/eldiron/ftp.nix | 4 + systems/eldiron/gitolite/default.nix | 16 ++ systems/eldiron/pub/default.nix | 3 + systems/eldiron/websites/cloud/default.nix | 9 + systems/eldiron/websites/commento/default.nix | 3 + systems/eldiron/websites/cryptpad/default.nix | 3 + systems/eldiron/websites/tools/default.nix | 15 ++ systems/monitoring-1/flake.lock | 18 +- systems/monitoring-1/monitoring-master.nix | 6 +- systems/quatresaisons/flake.lock | 10 +- systems/zoldene/flake.lock | 6 +- 19 files changed, 562 insertions(+), 130 deletions(-) create mode 100644 systems/eldiron/borg_backup.nix (limited to 'systems') diff --git a/systems/backup-2/databases/mariadb_replication.nix b/systems/backup-2/databases/mariadb_replication.nix index 6616de0..f113219 100644 --- a/systems/backup-2/databases/mariadb_replication.nix +++ b/systems/backup-2/databases/mariadb_replication.nix @@ -161,7 +161,6 @@ in --gtid \ --master-data \ --flush-privileges \ - --ignore-database=netdata \ --all-databases > $filename ${pkgs.gzip}/bin/gzip $filename ''; diff --git a/systems/backup-2/flake.lock b/systems/backup-2/flake.lock index 7d5992e..645214d 100644 --- a/systems/backup-2/flake.lock +++ b/systems/backup-2/flake.lock @@ -22,7 +22,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-ptLDqa3BTCX2orio9YgGsOwYa5bsz2DWn6TrtR2B45w=", + "narHash": "sha256-Deh1qsi1UFskPSAwq2sUGyPeh7hVVHct8hhy4o6fEzE=", "path": "../../flakes/private/chatons", "type": "path" }, @@ -74,7 +74,7 @@ "environment": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -86,7 +86,7 @@ "environment_2": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../../flakes/private/environment", "type": "path" }, @@ -98,7 +98,7 @@ "environment_3": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -110,7 +110,7 @@ "environment_4": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -122,7 +122,7 @@ "environment_5": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -134,7 +134,7 @@ "environment_6": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -146,7 +146,7 @@ "environment_7": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -352,7 +352,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-1sEwMNWSytfyn4Rh5gUJACwCNgaTLdMDkO9hEr5OB7M=", + "narHash": "sha256-XdgjCex3Izb2hID+EBVj0YsEE5xvc+I416I2fHpi1LE=", "path": "../../flakes/private/mail-relay", "type": "path" }, @@ -371,7 +371,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-VhRXb3AAlSKwkq4BfVmoKzkLxEaAFGjYLAFETTZuhjE=", + "narHash": "sha256-bWNhkERypwoog3lphO0xURJ4xt58CZEWKn7So7A5mtM=", "path": "../../flakes/private/milters", "type": "path" }, @@ -389,7 +389,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-Lpe56lKWhlMQiQoCgvMJuqJtJ8+szDldjqMAGyt2U3U=", + "narHash": "sha256-VZjf9fXcyeS3LpVW6NvzJpiJuEtJsGlOOfH8XwL8CdI=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -925,7 +925,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-/vQ6FGFc53r79yiQrzF0NWTbRd4RKf8QiPSDhmiCciU=", + "narHash": "sha256-fntajNe0urhuR0NbTOQZLTMhtHnd7p6PVuuEf0oAoFg=", "path": "../../flakes/private/opendmarc", "type": "path" }, @@ -1104,7 +1104,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-+s9C1mPCXRA44AC0Fg+B2uD6UiK0JfUA0F8fhceH0lQ=", + "narHash": "sha256-ufaclDTTnoB7xLOCOY4EretrBp70rSmk0YzcVfglbLA=", "path": "../../flakes/private/system", "type": "path" }, diff --git a/systems/dilion/flake.lock b/systems/dilion/flake.lock index ce8bb8f..a20d25b 100644 --- a/systems/dilion/flake.lock +++ b/systems/dilion/flake.lock @@ -59,7 +59,7 @@ "environment": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../../flakes/private/environment", "type": "path" }, @@ -71,7 +71,7 @@ "environment_2": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -83,7 +83,7 @@ "environment_3": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -207,7 +207,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-Lpe56lKWhlMQiQoCgvMJuqJtJ8+szDldjqMAGyt2U3U=", + "narHash": "sha256-VZjf9fXcyeS3LpVW6NvzJpiJuEtJsGlOOfH8XwL8CdI=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -599,7 +599,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-+s9C1mPCXRA44AC0Fg+B2uD6UiK0JfUA0F8fhceH0lQ=", + "narHash": "sha256-ufaclDTTnoB7xLOCOY4EretrBp70rSmk0YzcVfglbLA=", "path": "../../flakes/private/system", "type": "path" }, diff --git a/systems/eldiron/base.nix b/systems/eldiron/base.nix index a77a438..5308ddb 100644 --- a/systems/eldiron/base.nix +++ b/systems/eldiron/base.nix @@ -118,6 +118,7 @@ ./buildbot ./coturn.nix ./dns.nix + ./borg_backup.nix ./duply_backup.nix ./gemini ./gitolite @@ -132,6 +133,19 @@ ./vpn ]; + services.borgBackup.enable = true; + services.borgBackup.profiles.global = { + bucket = "global"; + hash = false; + remotes = [ "attilax" ]; + ignoredPaths = [ + "udev" + "portables" + "machines" + "nixos" + "nixos-containers" + ]; + }; myServices.buildbot.enable = true; myServices.databases.enable = true; myServices.gitolite.enable = true; diff --git a/systems/eldiron/borg_backup.nix b/systems/eldiron/borg_backup.nix new file mode 100644 index 0000000..9956a46 --- /dev/null +++ b/systems/eldiron/borg_backup.nix @@ -0,0 +1,241 @@ +{ lib, pkgs, config, name, ... }: + +let + cfg = config.myEnv.borg_backup; + varDir = "/var/lib/borgbackup"; + borg_args = "--encryption repokey --make-parent-dirs init create prune compact check"; + borg_backup_full_with_ignored = pkgs.writeScriptBin "borg_full_with_ignored" '' + #!${pkgs.stdenv.shell} + + if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then + echo "borg_full_with_ignored /path/to/borgmatic.yaml" + echo "Does a full backup including directories with .duplicity-ignore" + exit 1 + fi + ${pkgs.borgmatic}/bin/borgmatic -c "$1" --override 'storage.archive_name_format="{hostname}-with-ignored-{now:%Y-%m-%dT%H:%M:%S.%f}"' --override 'location.exclude_if_present=[]' ${borg_args} + ''; + borg_backup = pkgs.writeScriptBin "borg_backup" '' + #!${pkgs.stdenv.shell} + + declare -a profiles + profiles=() + ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [ + ''profiles+=("${remote}_${k}")'' + ]) v.remotes) config.services.borgBackup.profiles))} + + if [ -f "${varDir}/last_backup_profile" ]; then + last_backup=$(cat ${varDir}/last_backup_profile) + for i in "''${!profiles[@]}"; do + if [[ "''${profiles[$i]}" = "$last_backup" ]]; then + break + fi + done + ((i+=1)) + profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}") + fi + + # timeout in minutes + timeout="''${1:-180}" + timeout_timestamp=$(date +%s -d "$timeout minutes") + for profile in "''${profiles[@]}"; do + if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then + break + fi + + touch "${varDir}/$profile.log" + ${pkgs.borgmatic}/bin/borgmatic -c "${config.secrets.location}/borg_backup/$profile/borgmatic.yaml" ${borg_args} >> ${varDir}/$profile.log + [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2 + echo "$profile" > ${varDir}/last_backup_profile + done + ''; + + check_backups = pkgs.writeScriptBin "borg_list_not_backuped" '' + #!${pkgs.stdenv.shell} + + do_check() { + local dir="$1" path ignored_path + find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do + if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then + continue + elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then + do_check "$path" + else + while IFS= read -r ignored_path; do + if [[ "$path" =~ ^$ignored_path$ ]]; then + continue 2 + fi + done < ${config.secrets.fullPaths."borg_backup/ignored_list"} + printf '%s\n' "$path" + fi + done + } + + do_check /var/lib + ''; + borgProfile = profile: remote: bucket: builtins.toJSON { + location = { + source_directories = map (p: "${profile.rootDir}/${p}") profile.includedPaths; + repositories = [ + { path = cfg.remotes.${remote}.remote bucket; label = "backupserver"; } + ]; + one_file_system = false; + exclude_if_present = [".duplicity-ignore"]; + source_directories_must_exist = profile.directoriesMustExist; + borgmatic_source_directory = "${varDir}/${profile.bucket}/.borgmatic"; + }; + storage = { + encryption_passphrase = profile.password; + ssh_command = "ssh -i ${config.secrets.fullPaths."borg_backup/identity"}"; + compression = "zlib"; + borg_base_directory = "${varDir}/${profile.bucket}"; + }; + retention = { + keep_within = "10d"; + keep_daily = 30; + }; + }; +in +{ + options = { + services.borgBackup.enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to enable remote backups. + ''; + }; + services.borgBackup.profiles = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + hash = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Hash bucket and directory names + ''; + }; + rootDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib"; + description = '' + Path to backup + ''; + }; + password = lib.mkOption { + type = lib.types.str; + default = cfg.password; + description = '' + password to use to encrypt data + ''; + }; + directoriesMustExist = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Raise error if backuped directory doesn't exist + ''; + }; + bucket = lib.mkOption { + type = lib.types.str; + description = '' + Bucket to use + ''; + }; + remotes = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + Remotes to use for backup + ''; + }; + includedPaths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = '' + Included paths (subdirs of rootDir) + ''; + }; + excludeFile = lib.mkOption { + type = lib.types.lines; + default = ""; + description = '' + Content to put in exclude file + ''; + }; + ignoredPaths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = '' + List of paths to ignore when checking non-backed-up directories + Can use (POSIX extended) regex + ''; + }; + }; + }); + }; + }; + + config = lib.mkIf config.services.borgBackup.enable { + system.activationScripts.borg_backup = '' + install -m 0700 -o root -g root -d ${varDir} + ''; + secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v: + let + bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket; + in map (remote: [ + (lib.nameValuePair "borg_backup/${remote}_${k}/borgmatic.yaml" { + permissions = "0400"; + text = borgProfile v remote bucket; + }) + (lib.nameValuePair "borg_backup/${remote}_${k}" { + permissions = "0700"; + isDir = true; + }) + ]) v.remotes) config.services.borgBackup.profiles)) // { + "borg_backup/identity" = { + permissions = "0400"; + text = "{{ .ssl_keys.borg_backup }}"; + }; + "borg_backup/ignored_list" = { + permissions = "0400"; + text = let + ignored = map + (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths) + (builtins.attrValues config.services.borgBackup.profiles); + in builtins.concatStringsSep "\n" (lib.flatten ignored); + }; + "borg_backup/backuped_list" = { + permissions = "0400"; + text = let + included = map + (v: map (p: "${v.rootDir}/${p}") v.includedPaths) + (builtins.attrValues config.services.borgBackup.profiles); + in builtins.concatStringsSep "\n" (lib.flatten included); + }; + }; + + programs.ssh.knownHostsFiles = [ + (pkgs.writeText + "borg_backup_known_hosts" + (builtins.concatStringsSep + "\n" + (builtins.filter + (v: v != null) + (builtins.map + (v: v.sshKnownHosts) + (builtins.attrValues cfg.remotes) + ) + ) + ) + ) + ]; + environment.systemPackages = [ pkgs.borgbackup pkgs.borgmatic borg_backup_full_with_ignored borg_backup check_backups ]; + services.cron = { + enable = true; + systemCronJobs = [ + "0 0 * * * root ${borg_backup}/bin/borg_backup 300" + ]; + + }; + + }; +} diff --git a/systems/eldiron/databases/openldap/default.nix b/systems/eldiron/databases/openldap/default.nix index 7cd15da..fcab337 100644 --- a/systems/eldiron/databases/openldap/default.nix +++ b/systems/eldiron/databases/openldap/default.nix @@ -103,6 +103,9 @@ in }; users.users.openldap.extraGroups = [ "keys" ]; networking.firewall.allowedTCPPorts = [ 636 389 ]; + services.borgBackup.profiles.global.includedPaths = [ + "openldap" + ]; security.acme.certs."ldap" = { group = "openldap"; diff --git a/systems/eldiron/duply_backup.nix b/systems/eldiron/duply_backup.nix index 590d125..5143302 100644 --- a/systems/eldiron/duply_backup.nix +++ b/systems/eldiron/duply_backup.nix @@ -3,29 +3,108 @@ let cfg = config.myEnv.backup; varDir = "/var/lib/duply"; - duplyProfile = profile: remote: prefix: '' - GPG_PW="${cfg.password}" - TARGET="${cfg.remotes.${remote}.remote profile.bucket}${prefix}" - export AWS_ACCESS_KEY_ID="${cfg.remotes.${remote}.accessKeyId}" - export AWS_SECRET_ACCESS_KEY="${cfg.remotes.${remote}.secretAccessKey}" + default_action = "pre_bkp_purge_purgeFull_purgeIncr"; + duply_backup_full_with_ignored = pkgs.writeScriptBin "duply_full_with_ignored" '' + #!${pkgs.stdenv.shell} + + export DUPLY_FULL_BACKUP_WITH_IGNORED=yes + if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then + echo "duply_full_with_ignored /path/to/profile" + echo "Does a full backup including directories with .duplicity-ignore" + exit 1 + fi + ${pkgs.duply}/bin/duply "$1" pre_full --force + ''; + duply_backup = pkgs.writeScriptBin "duply_backup" '' + #!${pkgs.stdenv.shell} + + declare -a profiles + profiles=() + ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [ + ''profiles+=("${remote}_${k}")'' + ]) v.remotes) config.services.duplyBackup.profiles))} + + if [ -f "${varDir}/last_backup_profile" ]; then + last_backup=$(cat ${varDir}/last_backup_profile) + for i in "''${!profiles[@]}"; do + if [[ "''${profiles[$i]}" = "$last_backup" ]]; then + break + fi + done + ((i+=1)) + profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}") + fi + + # timeout in minutes + timeout="''${1:-180}" + timeout_timestamp=$(date +%s -d "$timeout minutes") + for profile in "''${profiles[@]}"; do + if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then + break + fi + + touch "${varDir}/$profile.log" + ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/$profile/ ${default_action} --force >> ${varDir}/$profile.log + [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2 + echo "$profile" > ${varDir}/last_backup_profile + done + ''; + + check_backups = pkgs.writeScriptBin "duply_list_not_backuped" '' + #!${pkgs.stdenv.shell} + + do_check() { + local dir="$1" path ignored_path + find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do + if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."backup/backuped_list"}; then + continue + elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."backup/backuped_list"}; then + do_check "$path" + else + while IFS= read -r ignored_path; do + if [[ "$path" =~ ^$ignored_path$ ]]; then + continue 2 + fi + done < ${config.secrets.fullPaths."backup/ignored_list"} + printf '%s\n' "$path" + fi + done + } + + do_check /var/lib + ''; + duplyProfile = profile: remote: bucket: let + remote' = cfg.remotes.${remote}; + in '' + if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then + GPG_PW="${cfg.password}" + fi + TARGET="${remote'.remote bucket}" + ${lib.optionalString (remote'.remote_type == "s3") '' + export AWS_ACCESS_KEY_ID="${remote'.s3AccessKeyId}" + export AWS_SECRET_ACCESS_KEY="${remote'.s3SecretAccessKey}" + ''} + ${lib.optionalString (remote'.remote_type == "rsync") '' + DUPL_PARAMS="$DUPL_PARAMS --ssh-options=-oIdentityFile='${config.secrets.fullPaths."backup/identity"}' " + ''} SOURCE="${profile.rootDir}" - FILENAME=".duplicity-ignore" - DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'" + if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then + FILENAME=".duplicity-ignore" + DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'" + fi VERBOSITY=4 ARCH_DIR="${varDir}/caches" + DUPL_PYTHON_BIN="" - # Do a full backup after 1 month - MAX_FULLBKP_AGE=1M - DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --exclude-other-filesystems --full-if-older-than $MAX_FULLBKP_AGE " - # Backups older than 2months are deleted - MAX_AGE=2M - # Keep 2 full backups - MAX_FULL_BACKUPS=2 - MAX_FULLS_WITH_INCRS=2 + # Do a full backup after 6 month + MAX_FULLBKP_AGE=6M + DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --full-if-older-than $MAX_FULLBKP_AGE " + # Backups older than 1months are deleted + MAX_AGE=1M + # Keep 1 full backup + MAX_FULL_BACKUPS=1 + MAX_FULLS_WITH_INCRS=1 ''; - action = "bkp_purge_purgeFull_purgeIncr"; - varName = k: remoteName: - if remoteName == "eriomem" then k else remoteName + "_" + k; in { options = { @@ -39,26 +118,46 @@ in services.duplyBackup.profiles = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule { options = { + hash = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Hash bucket and directory names + ''; + }; + excludeRootDir = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Exclude root dir in exclusion file + ''; + }; rootDir = lib.mkOption { type = lib.types.path; + default = "/var/lib"; description = '' Path to backup ''; }; bucket = lib.mkOption { type = lib.types.str; - default = "immae-${name}"; description = '' Bucket to use ''; }; remotes = lib.mkOption { type = lib.types.listOf lib.types.str; - default = ["eriomem"]; description = '' Remotes to use for backup ''; }; + includedPaths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = '' + Included paths (subdirs of rootDir) + ''; + }; excludeFile = lib.mkOption { type = lib.types.lines; default = ""; @@ -66,6 +165,14 @@ in Content to put in exclude file ''; }; + ignoredPaths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = '' + List of paths to ignore when checking non-backed-up directories + Can use (POSIX extended) regex + ''; + }; }; }); }; @@ -76,76 +183,91 @@ in install -m 0700 -o root -g root -d ${varDir} ${varDir}/caches ''; secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v: - map (remote: [ - (lib.nameValuePair "backup/${varName k remote}/conf" { + let + bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket; + in map (remote: [ + (lib.nameValuePair "backup/${remote}_${k}/conf" { permissions = "0400"; - text = duplyProfile v remote "${k}/"; + text = duplyProfile v remote bucket; }) - (lib.nameValuePair "backup/${varName k remote}/exclude" { + (lib.nameValuePair "backup/${remote}_${k}/exclude" { permissions = "0400"; - text = v.excludeFile; + text = v.excludeFile + (builtins.concatStringsSep "\n" (map (p: "+ ${v.rootDir}/${p}") v.includedPaths)) + (lib.optionalString v.excludeRootDir '' + + - ** + ''); }) - (lib.nameValuePair "backup/${varName k remote}" { + (lib.nameValuePair "backup/${remote}_${k}/pre" { + keyDependencies = [ + pkgs.bash + pkgs.rsync + ]; permissions = "0500"; + text = let + remote' = cfg.remotes.${remote}; + in '' + #!${pkgs.stdenv.shell} + + ${lib.optionalString (remote'.remote_type == "rsync") '' + # Recreate directory structure before synchronizing + mkdir -p ${varDir}/rsync_remotes/${remote}/${bucket} + ${pkgs.rsync}/bin/rsync -av -e \ + "ssh -p ${remote'.sshRsyncPort} -oIdentityFile=${config.secrets.fullPaths."backup/identity"}" \ + "${varDir}/rsync_remotes/${remote}/" \ + ${remote'.sshRsyncHost}: + ''} + ''; + }) + (lib.nameValuePair "backup/${remote}_${k}" { + permissions = "0700"; isDir = true; }) - ]) v.remotes) config.services.duplyBackup.profiles)); + ]) v.remotes) config.services.duplyBackup.profiles)) // { + "backup/identity" = { + permissions = "0400"; + text = "{{ .ssl_keys.duply_backup }}"; + }; + "backup/ignored_list" = { + permissions = "0400"; + text = let + ignored = map + (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths) + (builtins.attrValues config.services.duplyBackup.profiles); + in builtins.concatStringsSep "\n" (lib.flatten ignored); + }; + "backup/backuped_list" = { + permissions = "0400"; + text = let + included = map + (v: map (p: "${v.rootDir}/${p}") v.includedPaths) + (builtins.attrValues config.services.duplyBackup.profiles); + in builtins.concatStringsSep "\n" (lib.flatten included); + }; + }; + programs.ssh.knownHostsFiles = [ + (pkgs.writeText + "duply_backup_known_hosts" + (builtins.concatStringsSep + "\n" + (builtins.filter + (v: v != null) + (builtins.map + (v: v.sshKnownHosts) + (builtins.attrValues cfg.remotes) + ) + ) + ) + ) + ]; + environment.systemPackages = [ pkgs.duply check_backups duply_backup_full_with_ignored duply_backup ]; services.cron = { enable = true; - systemCronJobs = let - backups = pkgs.writeScript "backups" '' - #!${pkgs.stdenv.shell} - - ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: - map (remote: [ - '' - touch ${varDir}/${varName k remote}.log - ${pkgs.duply}/bin/duply ${config.secrets.fullPaths."backup/${varName k remote}"}/ ${action} --force >> ${varDir}/${varName k remote}.log - [[ $? = 0 ]] || echo -e "Error when doing backup for ${varName k remote}, see above\n---------------------------------------" >&2 - '' - ]) v.remotes - ) config.services.duplyBackup.profiles))} - ''; - in - [ - "0 2 * * * root ${backups}" - ]; + systemCronJobs = [ + "0 0 * * * root ${duply_backup}/bin/duply_backup 90" + ]; }; - security.pki.certificateFiles = [ - (pkgs.fetchurl { - url = "http://downloads.e.eriomem.net/eriomemca.pem"; - sha256 = "1ixx4c6j3m26j8dp9a3dkvxc80v1nr5aqgmawwgs06bskasqkvvh"; - }) - ]; - - myServices.monitoring.fromMasterActivatedPlugins = [ "eriomem" ]; - myServices.monitoring.fromMasterObjects.service = [ - { - service_description = "eriomem backup is up and not full"; - host_name = config.hostEnv.fqdn; - use = "external-service"; - check_command = "check_backup_eriomem"; - - check_interval = 120; - notification_interval = "1440"; - - servicegroups = "webstatus-backup"; - } - - { - service_description = "ovh backup is up and not full"; - host_name = config.hostEnv.fqdn; - use = "external-service"; - check_command = "check_ok"; - - check_interval = 120; - notification_interval = "1440"; - - servicegroups = "webstatus-backup"; - } - ]; }; } diff --git a/systems/eldiron/flake.lock b/systems/eldiron/flake.lock index 27d4d5b..58291c5 100644 --- a/systems/eldiron/flake.lock +++ b/systems/eldiron/flake.lock @@ -129,7 +129,7 @@ "environment": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -141,7 +141,7 @@ "environment_2": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -153,7 +153,7 @@ "environment_3": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -165,7 +165,7 @@ "environment_4": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -177,7 +177,7 @@ "environment_5": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -189,7 +189,7 @@ "environment_6": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -1989,7 +1989,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-ptLDqa3BTCX2orio9YgGsOwYa5bsz2DWn6TrtR2B45w=", + "narHash": "sha256-Deh1qsi1UFskPSAwq2sUGyPeh7hVVHct8hhy4o6fEzE=", "path": "../../flakes/private/chatons", "type": "path" }, @@ -2001,7 +2001,7 @@ "private-environment": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../../flakes/private/environment", "type": "path" }, @@ -2020,7 +2020,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-VhRXb3AAlSKwkq4BfVmoKzkLxEaAFGjYLAFETTZuhjE=", + "narHash": "sha256-bWNhkERypwoog3lphO0xURJ4xt58CZEWKn7So7A5mtM=", "path": "../../flakes/private/milters", "type": "path" }, @@ -2038,7 +2038,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-Lpe56lKWhlMQiQoCgvMJuqJtJ8+szDldjqMAGyt2U3U=", + "narHash": "sha256-VZjf9fXcyeS3LpVW6NvzJpiJuEtJsGlOOfH8XwL8CdI=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -2073,7 +2073,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-/vQ6FGFc53r79yiQrzF0NWTbRd4RKf8QiPSDhmiCciU=", + "narHash": "sha256-fntajNe0urhuR0NbTOQZLTMhtHnd7p6PVuuEf0oAoFg=", "path": "../../flakes/private/opendmarc", "type": "path" }, @@ -2134,7 +2134,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-gjapO6CZFeLMHUlhqBVZu5P+IJzJaPu4pnuTep4ZSuM=", + "narHash": "sha256-uqftr7R3cVYwWuu8Xl6VbPVL2pqapv1bfmMJpq3LnZ4=", "path": "../../flakes/private/ssh", "type": "path" }, @@ -2153,7 +2153,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-+s9C1mPCXRA44AC0Fg+B2uD6UiK0JfUA0F8fhceH0lQ=", + "narHash": "sha256-ufaclDTTnoB7xLOCOY4EretrBp70rSmk0YzcVfglbLA=", "path": "../../flakes/private/system", "type": "path" }, diff --git a/systems/eldiron/ftp.nix b/systems/eldiron/ftp.nix index 6aa1afc..e5bc0f5 100644 --- a/systems/eldiron/ftp.nix +++ b/systems/eldiron/ftp.nix @@ -30,6 +30,10 @@ in }; config = lib.mkIf config.myServices.ftp.enable { + services.borgBackup.profiles.global.ignoredPaths = [ + "ftp/test_ftp" + "proftpd/authorized_keys" + ]; myServices.dns.zones."immae.eu".subdomains.ftp = with config.myServices.dns.helpers; ips servers.eldiron.ips.main; diff --git a/systems/eldiron/gitolite/default.nix b/systems/eldiron/gitolite/default.nix index 1885234..0882c18 100644 --- a/systems/eldiron/gitolite/default.nix +++ b/systems/eldiron/gitolite/default.nix @@ -11,6 +11,22 @@ in { }; config = lib.mkIf cfg.enable { + services.borgBackup.profiles.global.ignoredPaths = [ + "gitolite/.nix-.*" + "gitolite/.ssh" + "gitolite/.vim.*" + "gitolite/.bash_history" + ]; + services.borgBackup.profiles.global.includedPaths = [ + "gitolite/gitolite_ldap_groups.sh" + "gitolite/projects.list" + "gitolite/.gitolite.rc" + "gitolite/.gitolite" + "gitolite/repositories/github" + "gitolite/repositories/testing.git" + "gitolite/repositories/gitolite-admin.git" + + ]; myServices.dns.zones."immae.eu".subdomains.git = with config.myServices.dns.helpers; ips servers.eldiron.ips.main; diff --git a/systems/eldiron/pub/default.nix b/systems/eldiron/pub/default.nix index ca8122a..847e9d0 100644 --- a/systems/eldiron/pub/default.nix +++ b/systems/eldiron/pub/default.nix @@ -40,6 +40,9 @@ in }; config = lib.mkIf config.myServices.pub.enable { + services.borgBackup.profiles.global.ignoredPaths = [ + "pub/.nix-.*" + ]; myServices.dns.zones."immae.eu".subdomains.pub = with config.myServices.dns.helpers; ips servers.eldiron.ips.main; diff --git a/systems/eldiron/websites/cloud/default.nix b/systems/eldiron/websites/cloud/default.nix index e1df883..c859f32 100644 --- a/systems/eldiron/websites/cloud/default.nix +++ b/systems/eldiron/websites/cloud/default.nix @@ -9,6 +9,15 @@ in { }; config = lib.mkIf cfg.enable { + services.borgBackup.profiles.global.includedPaths = [ + "nextcloud/appdata_occ80acffb591" + "nextcloud/files_external" + "nextcloud/config" + #"nextcloud/gpxpod" + "nextcloud/.ocdata" + "nextcloud/.htaccess" + "nextcloud/index.html" + ]; myServices.dns.zones."immae.eu".subdomains.cloud = with config.myServices.dns.helpers; ips servers.eldiron.ips.main; diff --git a/systems/eldiron/websites/commento/default.nix b/systems/eldiron/websites/commento/default.nix index c5131b8..9abc180 100644 --- a/systems/eldiron/websites/commento/default.nix +++ b/systems/eldiron/websites/commento/default.nix @@ -12,6 +12,9 @@ in enable = lib.mkEnableOption "Enable commento website"; }; config = lib.mkIf cfg.enable { + services.borgBackup.profiles.global.includedPaths = [ + "vhost/tools.immae.eu/commento" + ]; myServices.dns.zones."immae.eu".subdomains.commento = with config.myServices.dns.helpers; ips servers.eldiron.ips.main; diff --git a/systems/eldiron/websites/cryptpad/default.nix b/systems/eldiron/websites/cryptpad/default.nix index 4635548..34a51cd 100644 --- a/systems/eldiron/websites/cryptpad/default.nix +++ b/systems/eldiron/websites/cryptpad/default.nix @@ -75,6 +75,9 @@ in inherit domain port; config = configFile; }; + services.borgBackup.profiles.global.includedPaths = [ + "cryptpad/immaeEu" + ]; services.websites.env.tools.modules = [ "proxy_wstunnel" ]; security.acme.certs.eldiron.extraDomainNames = [ domain ]; services.websites.env.tools.vhostConfs.cryptpad = { diff --git a/systems/eldiron/websites/tools/default.nix b/systems/eldiron/websites/tools/default.nix index b591190..46e6a9f 100644 --- a/systems/eldiron/websites/tools/default.nix +++ b/systems/eldiron/websites/tools/default.nix @@ -110,6 +110,21 @@ in { ]; }; + services.borgBackup.profiles.global.ignoredPaths = [ + "duply" + "kanboard" + "ntfy" + ]; + services.borgBackup.profiles.global.includedPaths = [ + "paste" + "dokuwiki/conf" + "dokuwiki/data" + "phpbb" + "shaarli/cache" + "shaarli/pagecache" + "shaarli/tmp" + ]; + myServices.chatonsProperties.services = { adminer = adminer.chatonsProperties; dokuwiki = dokuwiki.chatonsProperties; diff --git a/systems/monitoring-1/flake.lock b/systems/monitoring-1/flake.lock index c69d09a..28d2415 100644 --- a/systems/monitoring-1/flake.lock +++ b/systems/monitoring-1/flake.lock @@ -22,7 +22,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-ptLDqa3BTCX2orio9YgGsOwYa5bsz2DWn6TrtR2B45w=", + "narHash": "sha256-Deh1qsi1UFskPSAwq2sUGyPeh7hVVHct8hhy4o6fEzE=", "path": "../../flakes/private/chatons", "type": "path" }, @@ -74,7 +74,7 @@ "environment": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -86,7 +86,7 @@ "environment_2": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../../flakes/private/environment", "type": "path" }, @@ -98,7 +98,7 @@ "environment_3": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -110,7 +110,7 @@ "environment_4": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -122,7 +122,7 @@ "environment_5": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -259,7 +259,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-1sEwMNWSytfyn4Rh5gUJACwCNgaTLdMDkO9hEr5OB7M=", + "narHash": "sha256-XdgjCex3Izb2hID+EBVj0YsEE5xvc+I416I2fHpi1LE=", "path": "../../flakes/private/mail-relay", "type": "path" }, @@ -277,7 +277,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-Lpe56lKWhlMQiQoCgvMJuqJtJ8+szDldjqMAGyt2U3U=", + "narHash": "sha256-VZjf9fXcyeS3LpVW6NvzJpiJuEtJsGlOOfH8XwL8CdI=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -735,7 +735,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-+s9C1mPCXRA44AC0Fg+B2uD6UiK0JfUA0F8fhceH0lQ=", + "narHash": "sha256-ufaclDTTnoB7xLOCOY4EretrBp70rSmk0YzcVfglbLA=", "path": "../../flakes/private/system", "type": "path" }, diff --git a/systems/monitoring-1/monitoring-master.nix b/systems/monitoring-1/monitoring-master.nix index 09319b8..e9362d9 100644 --- a/systems/monitoring-1/monitoring-master.nix +++ b/systems/monitoring-1/monitoring-master.nix @@ -74,11 +74,11 @@ in group = "naemon"; permissions = "0400"; text = '' - export AWS_ACCESS_KEY_ID="${v.accessKeyId}" - export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}" + export AWS_ACCESS_KEY_ID="${v.s3AccessKeyId}" + export AWS_SECRET_ACCESS_KEY="${v.s3SecretAccessKey}" export BASE_URL="${v.remote "immae-eldiron"}" ''; - }) config.myEnv.backup.remotes; + }) (lib.filterAttrs (k: v: v.remote_type == "s3") config.myEnv.backup.remotes); services.naemon.extraConfig = '' broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live diff --git a/systems/quatresaisons/flake.lock b/systems/quatresaisons/flake.lock index 4cb49da..5534558 100644 --- a/systems/quatresaisons/flake.lock +++ b/systems/quatresaisons/flake.lock @@ -59,7 +59,7 @@ "environment": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../../flakes/private/environment", "type": "path" }, @@ -71,7 +71,7 @@ "environment_2": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -83,7 +83,7 @@ "environment_3": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -239,7 +239,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-Lpe56lKWhlMQiQoCgvMJuqJtJ8+szDldjqMAGyt2U3U=", + "narHash": "sha256-VZjf9fXcyeS3LpVW6NvzJpiJuEtJsGlOOfH8XwL8CdI=", "path": "../../flakes/private/monitoring", "type": "path" }, @@ -712,7 +712,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-+s9C1mPCXRA44AC0Fg+B2uD6UiK0JfUA0F8fhceH0lQ=", + "narHash": "sha256-ufaclDTTnoB7xLOCOY4EretrBp70rSmk0YzcVfglbLA=", "path": "../../flakes/private/system", "type": "path" }, diff --git a/systems/zoldene/flake.lock b/systems/zoldene/flake.lock index 86e91e6..fb95d03 100644 --- a/systems/zoldene/flake.lock +++ b/systems/zoldene/flake.lock @@ -59,7 +59,7 @@ "environment": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../environment", "type": "path" }, @@ -436,7 +436,7 @@ "private-environment": { "locked": { "lastModified": 1, - "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=", + "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=", "path": "../../flakes/private/environment", "type": "path" }, @@ -455,7 +455,7 @@ }, "locked": { "lastModified": 1, - "narHash": "sha256-+s9C1mPCXRA44AC0Fg+B2uD6UiK0JfUA0F8fhceH0lQ=", + "narHash": "sha256-ufaclDTTnoB7xLOCOY4EretrBp70rSmk0YzcVfglbLA=", "path": "../../flakes/private/system", "type": "path" }, -- cgit v1.2.3