Migrate to borg backup

12 files changed, 524 insertions, 91 deletions

diff --git a/systems/eldiron/base.nix b/systems/eldiron/base.nix
index a77a438..5308ddb 100644
--- a/systems/eldiron/base.nix
+++ b/systems/eldiron/base.nix
@@ -118,6 +118,7 @@
     ./buildbot
     ./coturn.nix
     ./dns.nix
+    ./borg_backup.nix
     ./duply_backup.nix
     ./gemini
     ./gitolite
@@ -132,6 +133,19 @@
     ./vpn
   ];
 
+  services.borgBackup.enable = true;
+  services.borgBackup.profiles.global = {
+    bucket = "global";
+    hash = false;
+    remotes = [ "attilax" ];
+    ignoredPaths = [
+      "udev"
+      "portables"
+      "machines"
+      "nixos"
+      "nixos-containers"
+    ];
+  };
   myServices.buildbot.enable = true;
   myServices.databases.enable = true;
   myServices.gitolite.enable = true;
diff --git a/systems/eldiron/borg_backup.nix b/systems/eldiron/borg_backup.nix
new file mode 100644
index 0000000..9956a46
--- /dev/null
+++ b/systems/eldiron/borg_backup.nix
@@ -0,0 +1,241 @@
+{ lib, pkgs, config, name, ... }:
+
+let
+  cfg = config.myEnv.borg_backup;
+  varDir = "/var/lib/borgbackup";
+  borg_args = "--encryption repokey --make-parent-dirs init create prune compact check";
+  borg_backup_full_with_ignored = pkgs.writeScriptBin "borg_full_with_ignored" ''
+    #!${pkgs.stdenv.shell}
+
+    if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then
+      echo "borg_full_with_ignored /path/to/borgmatic.yaml"
+      echo "Does a full backup including directories with .duplicity-ignore"
+      exit 1
+    fi
+    ${pkgs.borgmatic}/bin/borgmatic -c "$1" --override 'storage.archive_name_format="{hostname}-with-ignored-{now:%Y-%m-%dT%H:%M:%S.%f}"' --override 'location.exclude_if_present=[]' ${borg_args}
+  '';
+  borg_backup = pkgs.writeScriptBin "borg_backup" ''
+    #!${pkgs.stdenv.shell}
+
+    declare -a profiles
+    profiles=()
+    ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [
+      ''profiles+=("${remote}_${k}")''
+    ]) v.remotes) config.services.borgBackup.profiles))}
+
+    if [ -f "${varDir}/last_backup_profile" ]; then
+      last_backup=$(cat ${varDir}/last_backup_profile)
+      for i in "''${!profiles[@]}"; do
+        if [[ "''${profiles[$i]}" = "$last_backup" ]]; then
+          break
+        fi
+      done
+      ((i+=1))
+      profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}")
+    fi
+
+    # timeout in minutes
+    timeout="''${1:-180}"
+    timeout_timestamp=$(date +%s -d "$timeout minutes")
+    for profile in "''${profiles[@]}"; do
+      if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then
+        break
+      fi
+
+      touch "${varDir}/$profile.log"
+      ${pkgs.borgmatic}/bin/borgmatic -c "${config.secrets.location}/borg_backup/$profile/borgmatic.yaml" ${borg_args} >> ${varDir}/$profile.log
+      [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2
+      echo "$profile" > ${varDir}/last_backup_profile
+    done
+  '';
+
+  check_backups = pkgs.writeScriptBin "borg_list_not_backuped" ''
+    #!${pkgs.stdenv.shell}
+
+    do_check() {
+      local dir="$1" path ignored_path
+      find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do
+        if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then
+          continue
+        elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."borg_backup/backuped_list"}; then
+          do_check "$path"
+        else
+          while IFS= read -r ignored_path; do
+            if [[ "$path" =~ ^$ignored_path$ ]]; then
+              continue 2
+            fi
+          done < ${config.secrets.fullPaths."borg_backup/ignored_list"}
+          printf '%s\n' "$path"
+        fi
+      done
+    }
+
+    do_check /var/lib
+  '';
+  borgProfile = profile: remote: bucket: builtins.toJSON {
+    location = {
+      source_directories = map (p: "${profile.rootDir}/${p}") profile.includedPaths;
+      repositories = [
+        { path = cfg.remotes.${remote}.remote bucket; label = "backupserver"; }
+      ];
+      one_file_system = false;
+      exclude_if_present = [".duplicity-ignore"];
+      source_directories_must_exist = profile.directoriesMustExist;
+      borgmatic_source_directory = "${varDir}/${profile.bucket}/.borgmatic";
+    };
+    storage = {
+      encryption_passphrase = profile.password;
+      ssh_command = "ssh -i ${config.secrets.fullPaths."borg_backup/identity"}";
+      compression = "zlib";
+      borg_base_directory = "${varDir}/${profile.bucket}";
+    };
+    retention = {
+      keep_within = "10d";
+      keep_daily = 30;
+    };
+  };
+in
+{
+  options = {
+    services.borgBackup.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to enable remote backups.
+      '';
+    };
+    services.borgBackup.profiles = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule {
+        options = {
+          hash = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            description = ''
+              Hash bucket and directory names
+            '';
+          };
+          rootDir = lib.mkOption {
+            type = lib.types.path;
+            default = "/var/lib";
+            description = ''
+              Path to backup
+              '';
+          };
+          password = lib.mkOption {
+            type = lib.types.str;
+            default = cfg.password;
+            description = ''
+              password to use to encrypt data
+            '';
+          };
+          directoriesMustExist = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            description = ''
+              Raise error if backuped directory doesn't exist
+            '';
+          };
+          bucket = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              Bucket to use
+              '';
+          };
+          remotes = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            description = ''
+              Remotes to use for backup
+              '';
+          };
+          includedPaths = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [];
+            description = ''
+              Included paths (subdirs of rootDir)
+            '';
+          };
+          excludeFile = lib.mkOption {
+            type = lib.types.lines;
+            default = "";
+            description = ''
+              Content to put in exclude file
+              '';
+          };
+          ignoredPaths = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [];
+            description = ''
+              List of paths to ignore when checking non-backed-up directories
+              Can use (POSIX extended) regex
+            '';
+          };
+        };
+      });
+    };
+  };
+
+  config = lib.mkIf config.services.borgBackup.enable {
+    system.activationScripts.borg_backup = ''
+      install -m 0700 -o root -g root -d ${varDir}
+      '';
+    secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v:
+      let
+        bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket;
+      in map (remote: [
+        (lib.nameValuePair "borg_backup/${remote}_${k}/borgmatic.yaml" {
+          permissions = "0400";
+          text = borgProfile v remote bucket;
+        })
+        (lib.nameValuePair "borg_backup/${remote}_${k}" {
+          permissions = "0700";
+          isDir = true;
+        })
+    ]) v.remotes) config.services.borgBackup.profiles)) // {
+      "borg_backup/identity" = {
+        permissions = "0400";
+        text = "{{ .ssl_keys.borg_backup }}";
+      };
+      "borg_backup/ignored_list" = {
+        permissions = "0400";
+        text = let
+            ignored = map
+              (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths)
+              (builtins.attrValues config.services.borgBackup.profiles);
+          in builtins.concatStringsSep "\n" (lib.flatten ignored);
+      };
+      "borg_backup/backuped_list" = {
+        permissions = "0400";
+        text = let
+            included = map
+              (v: map (p: "${v.rootDir}/${p}") v.includedPaths)
+              (builtins.attrValues config.services.borgBackup.profiles);
+          in builtins.concatStringsSep "\n" (lib.flatten included);
+      };
+    };
+
+    programs.ssh.knownHostsFiles = [
+      (pkgs.writeText
+        "borg_backup_known_hosts"
+        (builtins.concatStringsSep
+          "\n"
+          (builtins.filter
+            (v: v != null)
+            (builtins.map
+              (v: v.sshKnownHosts)
+              (builtins.attrValues cfg.remotes)
+            )
+          )
+        )
+      )
+    ];
+    environment.systemPackages = [ pkgs.borgbackup pkgs.borgmatic borg_backup_full_with_ignored borg_backup check_backups ];
+    services.cron = {
+      enable = true;
+      systemCronJobs = [
+        "0 0 * * * root ${borg_backup}/bin/borg_backup 300"
+      ];
+
+    };
+
+  };
+}
diff --git a/systems/eldiron/databases/openldap/default.nix b/systems/eldiron/databases/openldap/default.nix
index 7cd15da..fcab337 100644
--- a/systems/eldiron/databases/openldap/default.nix
+++ b/systems/eldiron/databases/openldap/default.nix
@@ -103,6 +103,9 @@ in
     };
     users.users.openldap.extraGroups = [ "keys" ];
     networking.firewall.allowedTCPPorts = [ 636 389 ];
+    services.borgBackup.profiles.global.includedPaths = [
+      "openldap"
+    ];
 
     security.acme.certs."ldap" = {
       group = "openldap";
diff --git a/systems/eldiron/duply_backup.nix b/systems/eldiron/duply_backup.nix
index 590d125..5143302 100644
--- a/systems/eldiron/duply_backup.nix
+++ b/systems/eldiron/duply_backup.nix
@@ -3,29 +3,108 @@
 let
   cfg = config.myEnv.backup;
   varDir = "/var/lib/duply";
-  duplyProfile = profile: remote: prefix: ''
-    GPG_PW="${cfg.password}"
-    TARGET="${cfg.remotes.${remote}.remote profile.bucket}${prefix}"
-    export AWS_ACCESS_KEY_ID="${cfg.remotes.${remote}.accessKeyId}"
-    export AWS_SECRET_ACCESS_KEY="${cfg.remotes.${remote}.secretAccessKey}"
+  default_action = "pre_bkp_purge_purgeFull_purgeIncr";
+  duply_backup_full_with_ignored = pkgs.writeScriptBin "duply_full_with_ignored" ''
+    #!${pkgs.stdenv.shell}
+
+    export DUPLY_FULL_BACKUP_WITH_IGNORED=yes
+    if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then
+      echo "duply_full_with_ignored /path/to/profile"
+      echo "Does a full backup including directories with .duplicity-ignore"
+      exit 1
+    fi
+    ${pkgs.duply}/bin/duply "$1" pre_full --force
+  '';
+  duply_backup = pkgs.writeScriptBin "duply_backup" ''
+    #!${pkgs.stdenv.shell}
+
+    declare -a profiles
+    profiles=()
+    ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [
+      ''profiles+=("${remote}_${k}")''
+    ]) v.remotes) config.services.duplyBackup.profiles))}
+
+    if [ -f "${varDir}/last_backup_profile" ]; then
+      last_backup=$(cat ${varDir}/last_backup_profile)
+      for i in "''${!profiles[@]}"; do
+        if [[ "''${profiles[$i]}" = "$last_backup" ]]; then
+          break
+        fi
+      done
+      ((i+=1))
+      profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}")
+    fi
+
+    # timeout in minutes
+    timeout="''${1:-180}"
+    timeout_timestamp=$(date +%s -d "$timeout minutes")
+    for profile in "''${profiles[@]}"; do
+      if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then
+        break
+      fi
+
+      touch "${varDir}/$profile.log"
+      ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/$profile/ ${default_action} --force >> ${varDir}/$profile.log
+      [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2
+      echo "$profile" > ${varDir}/last_backup_profile
+    done
+  '';
+
+  check_backups = pkgs.writeScriptBin "duply_list_not_backuped" ''
+    #!${pkgs.stdenv.shell}
+
+    do_check() {
+      local dir="$1" path ignored_path
+      find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do
+        if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."backup/backuped_list"}; then
+          continue
+        elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."backup/backuped_list"}; then
+          do_check "$path"
+        else
+          while IFS= read -r ignored_path; do
+            if [[ "$path" =~ ^$ignored_path$ ]]; then
+              continue 2
+            fi
+          done < ${config.secrets.fullPaths."backup/ignored_list"}
+          printf '%s\n' "$path"
+        fi
+      done
+    }
+
+    do_check /var/lib
+  '';
+  duplyProfile = profile: remote: bucket: let
+    remote' = cfg.remotes.${remote};
+  in ''
+    if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then
+      GPG_PW="${cfg.password}"
+    fi
+    TARGET="${remote'.remote bucket}"
+    ${lib.optionalString (remote'.remote_type == "s3") ''
+    export AWS_ACCESS_KEY_ID="${remote'.s3AccessKeyId}"
+    export AWS_SECRET_ACCESS_KEY="${remote'.s3SecretAccessKey}"
+    ''}
+    ${lib.optionalString (remote'.remote_type == "rsync") ''
+    DUPL_PARAMS="$DUPL_PARAMS --ssh-options=-oIdentityFile='${config.secrets.fullPaths."backup/identity"}' "
+    ''}
     SOURCE="${profile.rootDir}"
-    FILENAME=".duplicity-ignore"
-    DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'"
+    if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then
+      FILENAME=".duplicity-ignore"
+      DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'"
+    fi
     VERBOSITY=4
     ARCH_DIR="${varDir}/caches"
+    DUPL_PYTHON_BIN=""
 
-    # Do a full backup after 1 month
-    MAX_FULLBKP_AGE=1M
-    DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --exclude-other-filesystems --full-if-older-than $MAX_FULLBKP_AGE "
-    # Backups older than 2months are deleted
-    MAX_AGE=2M
-    # Keep 2 full backups
-    MAX_FULL_BACKUPS=2
-    MAX_FULLS_WITH_INCRS=2
+    # Do a full backup after 6 month
+    MAX_FULLBKP_AGE=6M
+    DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --full-if-older-than $MAX_FULLBKP_AGE "
+    # Backups older than 1months are deleted
+    MAX_AGE=1M
+    # Keep 1 full backup
+    MAX_FULL_BACKUPS=1
+    MAX_FULLS_WITH_INCRS=1
   '';
-  action = "bkp_purge_purgeFull_purgeIncr";
-  varName = k: remoteName:
-    if remoteName == "eriomem" then k else remoteName + "_" + k;
 in
 {
   options = {
@@ -39,26 +118,46 @@ in
     services.duplyBackup.profiles = lib.mkOption {
       type = lib.types.attrsOf (lib.types.submodule {
         options = {
+          hash = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            description = ''
+              Hash bucket and directory names
+            '';
+          };
+          excludeRootDir = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+            description = ''
+              Exclude root dir in exclusion file
+            '';
+          };
           rootDir = lib.mkOption {
             type = lib.types.path;
+            default = "/var/lib";
             description = ''
               Path to backup
               '';
           };
           bucket = lib.mkOption {
             type = lib.types.str;
-            default = "immae-${name}";
             description = ''
               Bucket to use
               '';
           };
           remotes = lib.mkOption {
             type = lib.types.listOf lib.types.str;
-            default = ["eriomem"];
             description = ''
               Remotes to use for backup
               '';
           };
+          includedPaths = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [];
+            description = ''
+              Included paths (subdirs of rootDir)
+            '';
+          };
           excludeFile = lib.mkOption {
             type = lib.types.lines;
             default = "";
@@ -66,6 +165,14 @@ in
               Content to put in exclude file
               '';
           };
+          ignoredPaths = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [];
+            description = ''
+              List of paths to ignore when checking non-backed-up directories
+              Can use (POSIX extended) regex
+            '';
+          };
         };
       });
     };
@@ -76,76 +183,91 @@ in
       install -m 0700 -o root -g root -d ${varDir} ${varDir}/caches
       '';
     secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v:
-      map (remote: [
-        (lib.nameValuePair "backup/${varName k remote}/conf" {
+      let
+        bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket;
+      in map (remote: [
+        (lib.nameValuePair "backup/${remote}_${k}/conf" {
           permissions = "0400";
-          text = duplyProfile v remote "${k}/";
+          text = duplyProfile v remote bucket;
         })
-        (lib.nameValuePair "backup/${varName k remote}/exclude" {
+        (lib.nameValuePair "backup/${remote}_${k}/exclude" {
           permissions = "0400";
-          text = v.excludeFile;
+          text = v.excludeFile + (builtins.concatStringsSep "\n" (map (p: "+ ${v.rootDir}/${p}") v.includedPaths)) + (lib.optionalString v.excludeRootDir ''
+
+            - **
+          '');
         })
-        (lib.nameValuePair "backup/${varName k remote}" {
+        (lib.nameValuePair "backup/${remote}_${k}/pre" {
+          keyDependencies = [
+            pkgs.bash
+            pkgs.rsync
+          ];
           permissions = "0500";
+          text = let
+            remote' = cfg.remotes.${remote};
+          in ''
+            #!${pkgs.stdenv.shell}
+
+            ${lib.optionalString (remote'.remote_type == "rsync") ''
+              # Recreate directory structure before synchronizing
+              mkdir -p ${varDir}/rsync_remotes/${remote}/${bucket}
+              ${pkgs.rsync}/bin/rsync -av -e \
+                "ssh -p ${remote'.sshRsyncPort} -oIdentityFile=${config.secrets.fullPaths."backup/identity"}" \
+                "${varDir}/rsync_remotes/${remote}/" \
+                ${remote'.sshRsyncHost}:
+            ''}
+          '';
+        })
+        (lib.nameValuePair "backup/${remote}_${k}" {
+          permissions = "0700";
           isDir = true;
         })
-    ]) v.remotes) config.services.duplyBackup.profiles));
+    ]) v.remotes) config.services.duplyBackup.profiles)) // {
+      "backup/identity" = {
+        permissions = "0400";
+        text = "{{ .ssl_keys.duply_backup }}";
+      };
+      "backup/ignored_list" = {
+        permissions = "0400";
+        text = let
+            ignored = map
+              (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths)
+              (builtins.attrValues config.services.duplyBackup.profiles);
+          in builtins.concatStringsSep "\n" (lib.flatten ignored);
+      };
+      "backup/backuped_list" = {
+        permissions = "0400";
+        text = let
+            included = map
+              (v: map (p: "${v.rootDir}/${p}") v.includedPaths)
+              (builtins.attrValues config.services.duplyBackup.profiles);
+          in builtins.concatStringsSep "\n" (lib.flatten included);
+      };
+    };
 
+    programs.ssh.knownHostsFiles = [
+      (pkgs.writeText
+        "duply_backup_known_hosts"
+        (builtins.concatStringsSep
+          "\n"
+          (builtins.filter
+            (v: v != null)
+            (builtins.map
+              (v: v.sshKnownHosts)
+              (builtins.attrValues cfg.remotes)
+            )
+          )
+        )
+      )
+    ];
+    environment.systemPackages = [ pkgs.duply check_backups duply_backup_full_with_ignored duply_backup ];
     services.cron = {
       enable = true;
-      systemCronJobs = let
-        backups = pkgs.writeScript "backups" ''
-          #!${pkgs.stdenv.shell}
-
-          ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v:
-            map (remote: [
-            ''
-              touch ${varDir}/${varName k remote}.log
-              ${pkgs.duply}/bin/duply ${config.secrets.fullPaths."backup/${varName k remote}"}/ ${action} --force >> ${varDir}/${varName k remote}.log
-              [[ $? = 0 ]] || echo -e "Error when doing backup for ${varName k remote}, see above\n---------------------------------------" >&2
-            ''
-            ]) v.remotes
-          ) config.services.duplyBackup.profiles))}
-        '';
-      in
-        [
-          "0 2 * * * root ${backups}"
-        ];
+      systemCronJobs = [
+        "0 0 * * * root ${duply_backup}/bin/duply_backup 90"
+      ];
 
     };
 
-    security.pki.certificateFiles = [
-      (pkgs.fetchurl {
-        url = "http://downloads.e.eriomem.net/eriomemca.pem";
-        sha256 = "1ixx4c6j3m26j8dp9a3dkvxc80v1nr5aqgmawwgs06bskasqkvvh";
-      })
-    ];
-
-    myServices.monitoring.fromMasterActivatedPlugins = [ "eriomem" ];
-    myServices.monitoring.fromMasterObjects.service = [
-      {
-        service_description = "eriomem backup is up and not full";
-        host_name = config.hostEnv.fqdn;
-        use = "external-service";
-        check_command = "check_backup_eriomem";
-
-        check_interval = 120;
-        notification_interval = "1440";
-
-        servicegroups = "webstatus-backup";
-      }
-
-      {
-        service_description = "ovh backup is up and not full";
-        host_name = config.hostEnv.fqdn;
-        use = "external-service";
-        check_command = "check_ok";
-
-        check_interval = 120;
-        notification_interval = "1440";
-
-        servicegroups = "webstatus-backup";
-      }
-    ];
   };
 }
diff --git a/systems/eldiron/flake.lock b/systems/eldiron/flake.lock
index 27d4d5b..58291c5 100644
--- a/systems/eldiron/flake.lock
+++ b/systems/eldiron/flake.lock
@@ -129,7 +129,7 @@
     "environment": {
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=",
+        "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=",
         "path": "../environment",
         "type": "path"
       },
@@ -141,7 +141,7 @@
     "environment_2": {
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=",
+        "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=",
         "path": "../environment",
         "type": "path"
       },
@@ -153,7 +153,7 @@
     "environment_3": {
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=",
+        "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=",
         "path": "../environment",
         "type": "path"
       },
@@ -165,7 +165,7 @@
     "environment_4": {
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=",
+        "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=",
         "path": "../environment",
         "type": "path"
       },
@@ -177,7 +177,7 @@
     "environment_5": {
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=",
+        "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=",
         "path": "../environment",
         "type": "path"
       },
@@ -189,7 +189,7 @@
     "environment_6": {
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=",
+        "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=",
         "path": "../environment",
         "type": "path"
       },
@@ -1989,7 +1989,7 @@
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-ptLDqa3BTCX2orio9YgGsOwYa5bsz2DWn6TrtR2B45w=",
+        "narHash": "sha256-Deh1qsi1UFskPSAwq2sUGyPeh7hVVHct8hhy4o6fEzE=",
         "path": "../../flakes/private/chatons",
         "type": "path"
       },
@@ -2001,7 +2001,7 @@
     "private-environment": {
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-VO82m/95IcX3xxJ63wcLh3hXzXDRFKUohYil/18pBSY=",
+        "narHash": "sha256-Kj3j/3B8V8IHbeSZ3ho33C7ktOcTle2h6dKEWWfVuvU=",
         "path": "../../flakes/private/environment",
         "type": "path"
       },
@@ -2020,7 +2020,7 @@
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-VhRXb3AAlSKwkq4BfVmoKzkLxEaAFGjYLAFETTZuhjE=",
+        "narHash": "sha256-bWNhkERypwoog3lphO0xURJ4xt58CZEWKn7So7A5mtM=",
         "path": "../../flakes/private/milters",
         "type": "path"
       },
@@ -2038,7 +2038,7 @@
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-Lpe56lKWhlMQiQoCgvMJuqJtJ8+szDldjqMAGyt2U3U=",
+        "narHash": "sha256-VZjf9fXcyeS3LpVW6NvzJpiJuEtJsGlOOfH8XwL8CdI=",
         "path": "../../flakes/private/monitoring",
         "type": "path"
       },
@@ -2073,7 +2073,7 @@
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-/vQ6FGFc53r79yiQrzF0NWTbRd4RKf8QiPSDhmiCciU=",
+        "narHash": "sha256-fntajNe0urhuR0NbTOQZLTMhtHnd7p6PVuuEf0oAoFg=",
         "path": "../../flakes/private/opendmarc",
         "type": "path"
       },
@@ -2134,7 +2134,7 @@
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-gjapO6CZFeLMHUlhqBVZu5P+IJzJaPu4pnuTep4ZSuM=",
+        "narHash": "sha256-uqftr7R3cVYwWuu8Xl6VbPVL2pqapv1bfmMJpq3LnZ4=",
         "path": "../../flakes/private/ssh",
         "type": "path"
       },
@@ -2153,7 +2153,7 @@
       },
       "locked": {
         "lastModified": 1,
-        "narHash": "sha256-+s9C1mPCXRA44AC0Fg+B2uD6UiK0JfUA0F8fhceH0lQ=",
+        "narHash": "sha256-ufaclDTTnoB7xLOCOY4EretrBp70rSmk0YzcVfglbLA=",
         "path": "../../flakes/private/system",
         "type": "path"
       },
diff --git a/systems/eldiron/ftp.nix b/systems/eldiron/ftp.nix
index 6aa1afc..e5bc0f5 100644
--- a/systems/eldiron/ftp.nix
+++ b/systems/eldiron/ftp.nix
@@ -30,6 +30,10 @@ in
   };
 
   config = lib.mkIf config.myServices.ftp.enable {
+    services.borgBackup.profiles.global.ignoredPaths = [
+      "ftp/test_ftp"
+      "proftpd/authorized_keys"
+    ];
     myServices.dns.zones."immae.eu".subdomains.ftp =
       with config.myServices.dns.helpers; ips servers.eldiron.ips.main;
 
diff --git a/systems/eldiron/gitolite/default.nix b/systems/eldiron/gitolite/default.nix
index 1885234..0882c18 100644
--- a/systems/eldiron/gitolite/default.nix
+++ b/systems/eldiron/gitolite/default.nix
@@ -11,6 +11,22 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
+    services.borgBackup.profiles.global.ignoredPaths = [
+      "gitolite/.nix-.*"
+      "gitolite/.ssh"
+      "gitolite/.vim.*"
+      "gitolite/.bash_history"
+    ];
+    services.borgBackup.profiles.global.includedPaths = [
+      "gitolite/gitolite_ldap_groups.sh"
+      "gitolite/projects.list"
+      "gitolite/.gitolite.rc"
+      "gitolite/.gitolite"
+      "gitolite/repositories/github"
+      "gitolite/repositories/testing.git"
+      "gitolite/repositories/gitolite-admin.git"
+
+    ];
     myServices.dns.zones."immae.eu".subdomains.git =
       with config.myServices.dns.helpers; ips servers.eldiron.ips.main;
 
diff --git a/systems/eldiron/pub/default.nix b/systems/eldiron/pub/default.nix
index ca8122a..847e9d0 100644
--- a/systems/eldiron/pub/default.nix
+++ b/systems/eldiron/pub/default.nix
@@ -40,6 +40,9 @@ in
   };
 
   config = lib.mkIf config.myServices.pub.enable {
+    services.borgBackup.profiles.global.ignoredPaths = [
+      "pub/.nix-.*"
+    ];
     myServices.dns.zones."immae.eu".subdomains.pub =
       with config.myServices.dns.helpers; ips servers.eldiron.ips.main;
 
diff --git a/systems/eldiron/websites/cloud/default.nix b/systems/eldiron/websites/cloud/default.nix
index e1df883..c859f32 100644
--- a/systems/eldiron/websites/cloud/default.nix
+++ b/systems/eldiron/websites/cloud/default.nix
@@ -9,6 +9,15 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
+    services.borgBackup.profiles.global.includedPaths = [
+      "nextcloud/appdata_occ80acffb591"
+      "nextcloud/files_external"
+      "nextcloud/config"
+      #"nextcloud/gpxpod"
+      "nextcloud/.ocdata"
+      "nextcloud/.htaccess"
+      "nextcloud/index.html"
+    ];
     myServices.dns.zones."immae.eu".subdomains.cloud =
       with config.myServices.dns.helpers; ips servers.eldiron.ips.main;
 
diff --git a/systems/eldiron/websites/commento/default.nix b/systems/eldiron/websites/commento/default.nix
index c5131b8..9abc180 100644
--- a/systems/eldiron/websites/commento/default.nix
+++ b/systems/eldiron/websites/commento/default.nix
@@ -12,6 +12,9 @@ in
     enable = lib.mkEnableOption "Enable commento website";
   };
   config = lib.mkIf cfg.enable {
+    services.borgBackup.profiles.global.includedPaths = [
+      "vhost/tools.immae.eu/commento"
+    ];
     myServices.dns.zones."immae.eu".subdomains.commento =
       with config.myServices.dns.helpers; ips servers.eldiron.ips.main;
 
diff --git a/systems/eldiron/websites/cryptpad/default.nix b/systems/eldiron/websites/cryptpad/default.nix
index 4635548..34a51cd 100644
--- a/systems/eldiron/websites/cryptpad/default.nix
+++ b/systems/eldiron/websites/cryptpad/default.nix
@@ -75,6 +75,9 @@ in
       inherit domain port;
       config = configFile;
     };
+    services.borgBackup.profiles.global.includedPaths = [
+      "cryptpad/immaeEu"
+    ];
     services.websites.env.tools.modules = [ "proxy_wstunnel" ];
     security.acme.certs.eldiron.extraDomainNames = [ domain ];
     services.websites.env.tools.vhostConfs.cryptpad = {
diff --git a/systems/eldiron/websites/tools/default.nix b/systems/eldiron/websites/tools/default.nix
index b591190..46e6a9f 100644
--- a/systems/eldiron/websites/tools/default.nix
+++ b/systems/eldiron/websites/tools/default.nix
@@ -110,6 +110,21 @@ in {
         ];
       };
 
+    services.borgBackup.profiles.global.ignoredPaths = [
+      "duply"
+      "kanboard"
+      "ntfy"
+    ];
+    services.borgBackup.profiles.global.includedPaths = [
+      "paste"
+      "dokuwiki/conf"
+      "dokuwiki/data"
+      "phpbb"
+      "shaarli/cache"
+      "shaarli/pagecache"
+      "shaarli/tmp"
+    ];
+
     myServices.chatonsProperties.services = {
       adminer = adminer.chatonsProperties;
       dokuwiki = dokuwiki.chatonsProperties;