From bd5c5d4e23ebd3863a960976767ed4a83dfd07fe Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Fri, 15 Oct 2021 00:59:34 +0200 Subject: [PATCH] Move backups to flake --- flakes/rsync_backup/flake.nix | 245 ++++++++++++++++++++++++++ modules/default.nix | 2 +- modules/private/system/backup-2.nix | 20 ++- modules/rsync_backup/default.nix | 262 ---------------------------- 4 files changed, 264 insertions(+), 265 deletions(-) create mode 100644 flakes/rsync_backup/flake.nix delete mode 100644 modules/rsync_backup/default.nix diff --git a/flakes/rsync_backup/flake.nix b/flakes/rsync_backup/flake.nix new file mode 100644 index 0000000..6d359e5 --- /dev/null +++ b/flakes/rsync_backup/flake.nix @@ -0,0 +1,245 @@ +{ + description = "Rsync backups"; + + outputs = { self }: { + nixosModule = { lib, pkgs, config, ... }: { + options.services.rsyncBackup = { + mountpoint = lib.mkOption { + type = lib.types.path; + description = "Path to the base folder for backups"; + }; + profiles = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + keep = lib.mkOption { + type = lib.types.int; + default = 7; + description = '' + Number of backups to keep + ''; + }; + check_command = lib.mkOption { + type = lib.types.str; + default = "backup"; + description = '' + command to check if backup needs to be done + ''; + }; + login = lib.mkOption { + type = lib.types.str; + description = '' + login to connect to + ''; + }; + host = lib.mkOption { + type = lib.types.str; + description = '' + host to connect to + ''; + }; + port = lib.mkOption { + type = lib.types.str; + default = "22"; + description = '' + port to connect to + ''; + }; + host_key = lib.mkOption { + type = lib.types.str; + description = '' + Host key to use as known host + ''; + }; + host_key_type = lib.mkOption { + type = lib.types.str; + description = '' + Host key type + ''; + }; + parts = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + remote_folder = lib.mkOption { + type = lib.types.path; + description = '' + Path to backup + ''; + }; + exclude_from = lib.mkOption { + type = lib.types.listOf lib.types.path; + default = []; + description = '' + Paths to exclude from the backup + ''; + }; + files_from = lib.mkOption { + type = lib.types.listOf lib.types.path; + default = []; + description = '' + Paths to take for the backup + (if empty: whole folder minus exclude_from) + ''; + }; + args = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + additional arguments for rsync + ''; + }; + }; + }); + description = '' + folders to backup in the host + ''; + }; + }; + }); + default = {}; + description = '' + Profiles to backup + ''; + }; + ssh_key_public = lib.mkOption { + type = lib.types.str; + description = "Public key for the backup"; + }; + ssh_key_private = lib.mkOption { + type = lib.types.str; + description = "Private key for the backup"; + }; + }; + + config = + let + cfg = config.services.rsyncBackup; + in + lib.mkIf (builtins.length (builtins.attrNames cfg.profiles) > 0) { + users.users.backup = { + isSystemUser = true; + uid = config.ids.uids.backup; + group = "backup"; + extraGroups = [ "keys" ]; + }; + + users.groups.backup = { + gid = config.ids.gids.backup; + }; + + services.cron.systemCronJobs = let + ssh_key = cfg.ssh_key_private; + backup_head = '' + #!${pkgs.stdenv.shell} + EXCL_FROM=`mktemp` + FILES_FROM=`mktemp` + TMP_STDERR=`mktemp` + + on_exit() { + if [ -s "$TMP_STDERR" ]; then + cat "$TMP_STDERR" + fi + rm -f $TMP_STDERR $EXCL_FROM $FILES_FROM + } + + trap "on_exit" EXIT + + exec 2> "$TMP_STDERR" + exec < /dev/null + + set -e + ''; + backup_profile_head = name: profile: '' + ##### ${name} ##### + PORT="${profile.port}" + DEST="${profile.login}@${profile.host}" + BASE="${cfg.mountpoint}/${name}" + OLD_BAK_BASE=$BASE/older/j + BAK_BASE=''${OLD_BAK_BASE}0 + RSYNC_OUTPUT=$BASE/rsync_output + NBR=${builtins.toString profile.keep} + + if ! ssh \ + -o PreferredAuthentications=publickey \ + -o StrictHostKeyChecking=yes \ + -o ClearAllForwardings=yes \ + -o UserKnownHostsFile=/dev/null \ + -o CheckHostIP=no \ + -p $PORT \ + -i ${ssh_key} \ + $DEST ${profile.check_command}; then + echo "Fichier de verrouillage backup sur $DEST ou impossible de se connecter" >&2 + skip=$DEST + fi + + rm -rf ''${OLD_BAK_BASE}''${NBR} + for j in `seq -w $(($NBR-1)) -1 0`; do + [ ! -d ''${OLD_BAK_BASE}$j ] && continue + mv ''${OLD_BAK_BASE}$j ''${OLD_BAK_BASE}$(($j+1)) + done + mkdir $BAK_BASE + mv $RSYNC_OUTPUT $BAK_BASE + mkdir $RSYNC_OUTPUT + + if [ "$skip" != "$DEST" ]; then + ''; + backup_profile_tail = name: profile: '' + ssh -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -i ${ssh_key} -p $PORT $DEST sh -c "date > .cache/last_backup" + fi # [ "$skip" != "$DEST" ] + ##### End ${name} ##### + ''; + + backup_part = profile_name: part_name: part: '' + ### ${profile_name} ${part_name} ### + LOCAL="${part_name}" + REMOTE="${part.remote_folder}" + + if [ ! -d "$BASE/$LOCAL" ]; then + mkdir $BASE/$LOCAL + fi + cd $BASE/$LOCAL + cat > $EXCL_FROM < $FILES_FROM < 0) "\n --exclude-from=$EXCL_FROM \\"}${ + lib.optionalString (builtins.length part.files_from > 0) "\n --files-from=$FILES_FROM \\"} + $DEST:$REMOTE . > $OUT || true + ### End ${profile_name} ${part_name} ### + ''; + backup_profile = name: profile: builtins.concatStringsSep "\n" ( + [(backup_profile_head name profile)] + ++ lib.mapAttrsToList (backup_part name) profile.parts + ++ [(backup_profile_tail name profile)]); + + backup = pkgs.writeScript "backup.sh" (builtins.concatStringsSep "\n" ([ + backup_head + ] ++ lib.mapAttrsToList backup_profile cfg.profiles)); + in [ + '' + 25 3,15 * * * backup ${backup} + '' + ]; + + programs.ssh.knownHosts = lib.attrsets.mapAttrs' (name: profile: lib.attrsets.nameValuePair name { + hostNames = [ profile.host ]; + publicKey = "${profile.host_key_type} ${profile.host_key}"; + }) cfg.profiles; + + system.activationScripts.rsyncBackup = { + deps = [ "users" ]; + text = builtins.concatStringsSep "\n" (map (v: '' + install -m 0700 -o backup -g backup -d ${cfg.mountpoint}/${v} ${cfg.mountpoint}/${v}/older ${cfg.mountpoint}/${v}/rsync_output + '') (builtins.attrNames cfg.profiles) + ); + }; + }; + }; + }; +} diff --git a/modules/default.nix b/modules/default.nix index 630e8f5..b6ac68a 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -20,7 +20,7 @@ in openarc = flakeLib.withNarKeyCompat flakeCompat ../flakes/openarc "nixosModule"; duplyBackup = ./duply_backup; - rsyncBackup = ./rsync_backup; + rsyncBackup = flakeLib.withNarKeyCompat flakeCompat ../flakes/rsync_backup "nixosModule"; naemon = ./naemon; php-application = ./websites/php-application.nix; diff --git a/modules/private/system/backup-2.nix b/modules/private/system/backup-2.nix index 1f226c0..181f455 100644 --- a/modules/private/system/backup-2.nix +++ b/modules/private/system/backup-2.nix @@ -7,6 +7,22 @@ }; # ssh-keyscan backup-2 | nix-shell -p ssh-to-age --run ssh-to-age secrets.ageKeys = [ "age1kk3nr27qu42j28mcfdag5lhq0zu2pky7gfanvne8l4z2ctevjpgskmw0sr" ]; + secrets.keys = [ + { + dest = "rsync_backup/identity"; + user = "backup"; + group = "backup"; + permissions = "0400"; + text = config.myEnv.rsync_backup.ssh_key.private; + } + { + dest = "rsync_backup/identity.pub"; + user = "backup"; + group = "backup"; + permissions = "0444"; + text = config.myEnv.rsync_backup.ssh_key.public; + } + ]; boot.kernelPackages = pkgs.linuxPackages_latest; myEnv = import ../../../nixops/secrets/environment.nix; @@ -54,8 +70,8 @@ services.rsyncBackup = { mountpoint = "/backup2"; profiles = config.myEnv.rsync_backup.profiles; - ssh_key_public = config.myEnv.rsync_backup.ssh_key.public; - ssh_key_private = config.myEnv.rsync_backup.ssh_key.private; + ssh_key_public = config.secrets.fullPaths."rsync_backup/identity.pub"; + ssh_key_private = config.secrets.fullPaths."rsync_backup/identity"; }; myServices.mailRelay.enable = true; diff --git a/modules/rsync_backup/default.nix b/modules/rsync_backup/default.nix deleted file mode 100644 index f0df5a1..0000000 --- a/modules/rsync_backup/default.nix +++ /dev/null @@ -1,262 +0,0 @@ -{ lib, pkgs, config, ... }: -let - partModule = lib.types.submodule { - options = { - remote_folder = lib.mkOption { - type = lib.types.path; - description = '' - Path to backup - ''; - }; - exclude_from = lib.mkOption { - type = lib.types.listOf lib.types.path; - default = []; - description = '' - Paths to exclude from the backup - ''; - }; - files_from = lib.mkOption { - type = lib.types.listOf lib.types.path; - default = []; - description = '' - Paths to take for the backup - (if empty: whole folder minus exclude_from) - ''; - }; - args = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - description = '' - additional arguments for rsync - ''; - }; - }; - }; - profileModule = lib.types.submodule { - options = { - keep = lib.mkOption { - type = lib.types.int; - default = 7; - description = '' - Number of backups to keep - ''; - }; - check_command = lib.mkOption { - type = lib.types.str; - default = "backup"; - description = '' - command to check if backup needs to be done - ''; - }; - login = lib.mkOption { - type = lib.types.str; - description = '' - login to connect to - ''; - }; - host = lib.mkOption { - type = lib.types.str; - description = '' - host to connect to - ''; - }; - port = lib.mkOption { - type = lib.types.str; - default = "22"; - description = '' - port to connect to - ''; - }; - host_key = lib.mkOption { - type = lib.types.str; - description = '' - Host key to use as known host - ''; - }; - host_key_type = lib.mkOption { - type = lib.types.str; - description = '' - Host key type - ''; - }; - parts = lib.mkOption { - type = lib.types.attrsOf partModule; - description = '' - folders to backup in the host - ''; - }; - }; - }; - cfg = config.services.rsyncBackup; - - ssh_key = config.secrets.fullPaths."rsync_backup/identity"; - - backup_head = '' - #!${pkgs.stdenv.shell} - EXCL_FROM=`mktemp` - FILES_FROM=`mktemp` - TMP_STDERR=`mktemp` - - on_exit() { - if [ -s "$TMP_STDERR" ]; then - cat "$TMP_STDERR" - fi - rm -f $TMP_STDERR $EXCL_FROM $FILES_FROM - } - - trap "on_exit" EXIT - - exec 2> "$TMP_STDERR" - exec < /dev/null - - set -e - ''; - - backup_profile = name: profile: builtins.concatStringsSep "\n" ( - [(backup_profile_head name profile)] - ++ lib.mapAttrsToList (backup_part name) profile.parts - ++ [(backup_profile_tail name profile)]); - - backup_profile_head = name: profile: '' - ##### ${name} ##### - PORT="${profile.port}" - DEST="${profile.login}@${profile.host}" - BASE="${cfg.mountpoint}/${name}" - OLD_BAK_BASE=$BASE/older/j - BAK_BASE=''${OLD_BAK_BASE}0 - RSYNC_OUTPUT=$BASE/rsync_output - NBR=${builtins.toString profile.keep} - - if ! ssh \ - -o PreferredAuthentications=publickey \ - -o StrictHostKeyChecking=yes \ - -o ClearAllForwardings=yes \ - -o UserKnownHostsFile=/dev/null \ - -o CheckHostIP=no \ - -p $PORT \ - -i ${ssh_key} \ - $DEST ${profile.check_command}; then - echo "Fichier de verrouillage backup sur $DEST ou impossible de se connecter" >&2 - skip=$DEST - fi - - rm -rf ''${OLD_BAK_BASE}''${NBR} - for j in `seq -w $(($NBR-1)) -1 0`; do - [ ! -d ''${OLD_BAK_BASE}$j ] && continue - mv ''${OLD_BAK_BASE}$j ''${OLD_BAK_BASE}$(($j+1)) - done - mkdir $BAK_BASE - mv $RSYNC_OUTPUT $BAK_BASE - mkdir $RSYNC_OUTPUT - - if [ "$skip" != "$DEST" ]; then - ''; - - backup_profile_tail = name: profile: '' - ssh -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -i ${ssh_key} -p $PORT $DEST sh -c "date > .cache/last_backup" - fi # [ "$skip" != "$DEST" ] - ##### End ${name} ##### - ''; - - backup_part = profile_name: part_name: part: '' - ### ${profile_name} ${part_name} ### - LOCAL="${part_name}" - REMOTE="${part.remote_folder}" - - if [ ! -d "$BASE/$LOCAL" ]; then - mkdir $BASE/$LOCAL - fi - cd $BASE/$LOCAL - cat > $EXCL_FROM < $FILES_FROM < 0) "\n --exclude-from=$EXCL_FROM \\"}${ - lib.optionalString (builtins.length part.files_from > 0) "\n --files-from=$FILES_FROM \\"} - $DEST:$REMOTE . > $OUT || true - ### End ${profile_name} ${part_name} ### - ''; -in -{ - options.services.rsyncBackup = { - mountpoint = lib.mkOption { - type = lib.types.path; - description = "Path to the base folder for backups"; - }; - profiles = lib.mkOption { - type = lib.types.attrsOf profileModule; - default = {}; - description = '' - Profiles to backup - ''; - }; - ssh_key_public = lib.mkOption { - type = lib.types.str; - description = "Public key for the backup"; - }; - ssh_key_private = lib.mkOption { - type = lib.types.str; - description = "Private key for the backup"; - }; - }; - - config = lib.mkIf (builtins.length (builtins.attrNames cfg.profiles) > 0) { - users.users.backup = { - isSystemUser = true; - uid = config.ids.uids.backup; - group = "backup"; - extraGroups = [ "keys" ]; - }; - - users.groups.backup = { - gid = config.ids.gids.backup; - }; - - services.cron.systemCronJobs = let - backup = pkgs.writeScript "backup.sh" (builtins.concatStringsSep "\n" ([ - backup_head - ] ++ lib.mapAttrsToList backup_profile cfg.profiles)); - in [ - '' - 25 3,15 * * * backup ${backup} - '' - ]; - - programs.ssh.knownHosts = lib.attrsets.mapAttrs' (name: profile: lib.attrsets.nameValuePair name { - hostNames = [ profile.host ]; - publicKey = "${profile.host_key_type} ${profile.host_key}"; - }) cfg.profiles; - - system.activationScripts.rsyncBackup = { - deps = [ "users" ]; - text = builtins.concatStringsSep "\n" (map (v: '' - install -m 0700 -o backup -g backup -d ${cfg.mountpoint}/${v} ${cfg.mountpoint}/${v}/older ${cfg.mountpoint}/${v}/rsync_output - '') (builtins.attrNames cfg.profiles) - ); - }; - - secrets.keys = [ - { - dest = "rsync_backup/identity"; - user = "backup"; - group = "backup"; - permissions = "0400"; - text = cfg.ssh_key_private; - } - { - dest = "rsync_backup/identity.pub"; - user = "backup"; - group = "backup"; - permissions = "0444"; - text = cfg.ssh_key_public; - } - ]; - }; -} -- 2.41.0