diff options
Diffstat (limited to 'systems/eldiron/duply_backup.nix')
-rw-r--r-- | systems/eldiron/duply_backup.nix | 278 |
1 files changed, 200 insertions, 78 deletions
diff --git a/systems/eldiron/duply_backup.nix b/systems/eldiron/duply_backup.nix index 590d125..5143302 100644 --- a/systems/eldiron/duply_backup.nix +++ b/systems/eldiron/duply_backup.nix | |||
@@ -3,29 +3,108 @@ | |||
3 | let | 3 | let |
4 | cfg = config.myEnv.backup; | 4 | cfg = config.myEnv.backup; |
5 | varDir = "/var/lib/duply"; | 5 | varDir = "/var/lib/duply"; |
6 | duplyProfile = profile: remote: prefix: '' | 6 | default_action = "pre_bkp_purge_purgeFull_purgeIncr"; |
7 | GPG_PW="${cfg.password}" | 7 | duply_backup_full_with_ignored = pkgs.writeScriptBin "duply_full_with_ignored" '' |
8 | TARGET="${cfg.remotes.${remote}.remote profile.bucket}${prefix}" | 8 | #!${pkgs.stdenv.shell} |
9 | export AWS_ACCESS_KEY_ID="${cfg.remotes.${remote}.accessKeyId}" | 9 | |
10 | export AWS_SECRET_ACCESS_KEY="${cfg.remotes.${remote}.secretAccessKey}" | 10 | export DUPLY_FULL_BACKUP_WITH_IGNORED=yes |
11 | if [ -z "$1" -o "$1" = "-h" -o "$1" = "--help" ]; then | ||
12 | echo "duply_full_with_ignored /path/to/profile" | ||
13 | echo "Does a full backup including directories with .duplicity-ignore" | ||
14 | exit 1 | ||
15 | fi | ||
16 | ${pkgs.duply}/bin/duply "$1" pre_full --force | ||
17 | ''; | ||
18 | duply_backup = pkgs.writeScriptBin "duply_backup" '' | ||
19 | #!${pkgs.stdenv.shell} | ||
20 | |||
21 | declare -a profiles | ||
22 | profiles=() | ||
23 | ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: map (remote: [ | ||
24 | ''profiles+=("${remote}_${k}")'' | ||
25 | ]) v.remotes) config.services.duplyBackup.profiles))} | ||
26 | |||
27 | if [ -f "${varDir}/last_backup_profile" ]; then | ||
28 | last_backup=$(cat ${varDir}/last_backup_profile) | ||
29 | for i in "''${!profiles[@]}"; do | ||
30 | if [[ "''${profiles[$i]}" = "$last_backup" ]]; then | ||
31 | break | ||
32 | fi | ||
33 | done | ||
34 | ((i+=1)) | ||
35 | profiles=("''${profiles[@]:$i}" "''${profiles[@]:0:$i}") | ||
36 | fi | ||
37 | |||
38 | # timeout in minutes | ||
39 | timeout="''${1:-180}" | ||
40 | timeout_timestamp=$(date +%s -d "$timeout minutes") | ||
41 | for profile in "''${profiles[@]}"; do | ||
42 | if [ $(date +%s -d "now") -ge "$timeout_timestamp" ]; then | ||
43 | break | ||
44 | fi | ||
45 | |||
46 | touch "${varDir}/$profile.log" | ||
47 | ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/$profile/ ${default_action} --force >> ${varDir}/$profile.log | ||
48 | [[ $? = 0 ]] || echo -e "Error when doing backup for $profile, see above or logs in ${varDir}/$profile.log\n---------------------------------------" >&2 | ||
49 | echo "$profile" > ${varDir}/last_backup_profile | ||
50 | done | ||
51 | ''; | ||
52 | |||
53 | check_backups = pkgs.writeScriptBin "duply_list_not_backuped" '' | ||
54 | #!${pkgs.stdenv.shell} | ||
55 | |||
56 | do_check() { | ||
57 | local dir="$1" path ignored_path | ||
58 | find "$dir" -mindepth 1 -maxdepth 1 | while IFS= read -r path; do | ||
59 | if ${pkgs.gnugrep}/bin/grep -qFx "$path" ${config.secrets.fullPaths."backup/backuped_list"}; then | ||
60 | continue | ||
61 | elif ${pkgs.gnugrep}/bin/grep -q "^$path/" ${config.secrets.fullPaths."backup/backuped_list"}; then | ||
62 | do_check "$path" | ||
63 | else | ||
64 | while IFS= read -r ignored_path; do | ||
65 | if [[ "$path" =~ ^$ignored_path$ ]]; then | ||
66 | continue 2 | ||
67 | fi | ||
68 | done < ${config.secrets.fullPaths."backup/ignored_list"} | ||
69 | printf '%s\n' "$path" | ||
70 | fi | ||
71 | done | ||
72 | } | ||
73 | |||
74 | do_check /var/lib | ||
75 | ''; | ||
76 | duplyProfile = profile: remote: bucket: let | ||
77 | remote' = cfg.remotes.${remote}; | ||
78 | in '' | ||
79 | if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then | ||
80 | GPG_PW="${cfg.password}" | ||
81 | fi | ||
82 | TARGET="${remote'.remote bucket}" | ||
83 | ${lib.optionalString (remote'.remote_type == "s3") '' | ||
84 | export AWS_ACCESS_KEY_ID="${remote'.s3AccessKeyId}" | ||
85 | export AWS_SECRET_ACCESS_KEY="${remote'.s3SecretAccessKey}" | ||
86 | ''} | ||
87 | ${lib.optionalString (remote'.remote_type == "rsync") '' | ||
88 | DUPL_PARAMS="$DUPL_PARAMS --ssh-options=-oIdentityFile='${config.secrets.fullPaths."backup/identity"}' " | ||
89 | ''} | ||
11 | SOURCE="${profile.rootDir}" | 90 | SOURCE="${profile.rootDir}" |
12 | FILENAME=".duplicity-ignore" | 91 | if [ -z "$DUPLY_FULL_BACKUP_WITH_IGNORED" ]; then |
13 | DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'" | 92 | FILENAME=".duplicity-ignore" |
93 | DUPL_PARAMS="$DUPL_PARAMS --exclude-if-present '$FILENAME'" | ||
94 | fi | ||
14 | VERBOSITY=4 | 95 | VERBOSITY=4 |
15 | ARCH_DIR="${varDir}/caches" | 96 | ARCH_DIR="${varDir}/caches" |
97 | DUPL_PYTHON_BIN="" | ||
16 | 98 | ||
17 | # Do a full backup after 1 month | 99 | # Do a full backup after 6 month |
18 | MAX_FULLBKP_AGE=1M | 100 | MAX_FULLBKP_AGE=6M |
19 | DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --exclude-other-filesystems --full-if-older-than $MAX_FULLBKP_AGE " | 101 | DUPL_PARAMS="$DUPL_PARAMS --allow-source-mismatch --full-if-older-than $MAX_FULLBKP_AGE " |
20 | # Backups older than 2months are deleted | 102 | # Backups older than 1months are deleted |
21 | MAX_AGE=2M | 103 | MAX_AGE=1M |
22 | # Keep 2 full backups | 104 | # Keep 1 full backup |
23 | MAX_FULL_BACKUPS=2 | 105 | MAX_FULL_BACKUPS=1 |
24 | MAX_FULLS_WITH_INCRS=2 | 106 | MAX_FULLS_WITH_INCRS=1 |
25 | ''; | 107 | ''; |
26 | action = "bkp_purge_purgeFull_purgeIncr"; | ||
27 | varName = k: remoteName: | ||
28 | if remoteName == "eriomem" then k else remoteName + "_" + k; | ||
29 | in | 108 | in |
30 | { | 109 | { |
31 | options = { | 110 | options = { |
@@ -39,26 +118,46 @@ in | |||
39 | services.duplyBackup.profiles = lib.mkOption { | 118 | services.duplyBackup.profiles = lib.mkOption { |
40 | type = lib.types.attrsOf (lib.types.submodule { | 119 | type = lib.types.attrsOf (lib.types.submodule { |
41 | options = { | 120 | options = { |
121 | hash = lib.mkOption { | ||
122 | type = lib.types.bool; | ||
123 | default = true; | ||
124 | description = '' | ||
125 | Hash bucket and directory names | ||
126 | ''; | ||
127 | }; | ||
128 | excludeRootDir = lib.mkOption { | ||
129 | type = lib.types.bool; | ||
130 | default = true; | ||
131 | description = '' | ||
132 | Exclude root dir in exclusion file | ||
133 | ''; | ||
134 | }; | ||
42 | rootDir = lib.mkOption { | 135 | rootDir = lib.mkOption { |
43 | type = lib.types.path; | 136 | type = lib.types.path; |
137 | default = "/var/lib"; | ||
44 | description = '' | 138 | description = '' |
45 | Path to backup | 139 | Path to backup |
46 | ''; | 140 | ''; |
47 | }; | 141 | }; |
48 | bucket = lib.mkOption { | 142 | bucket = lib.mkOption { |
49 | type = lib.types.str; | 143 | type = lib.types.str; |
50 | default = "immae-${name}"; | ||
51 | description = '' | 144 | description = '' |
52 | Bucket to use | 145 | Bucket to use |
53 | ''; | 146 | ''; |
54 | }; | 147 | }; |
55 | remotes = lib.mkOption { | 148 | remotes = lib.mkOption { |
56 | type = lib.types.listOf lib.types.str; | 149 | type = lib.types.listOf lib.types.str; |
57 | default = ["eriomem"]; | ||
58 | description = '' | 150 | description = '' |
59 | Remotes to use for backup | 151 | Remotes to use for backup |
60 | ''; | 152 | ''; |
61 | }; | 153 | }; |
154 | includedPaths = lib.mkOption { | ||
155 | type = lib.types.listOf lib.types.str; | ||
156 | default = []; | ||
157 | description = '' | ||
158 | Included paths (subdirs of rootDir) | ||
159 | ''; | ||
160 | }; | ||
62 | excludeFile = lib.mkOption { | 161 | excludeFile = lib.mkOption { |
63 | type = lib.types.lines; | 162 | type = lib.types.lines; |
64 | default = ""; | 163 | default = ""; |
@@ -66,6 +165,14 @@ in | |||
66 | Content to put in exclude file | 165 | Content to put in exclude file |
67 | ''; | 166 | ''; |
68 | }; | 167 | }; |
168 | ignoredPaths = lib.mkOption { | ||
169 | type = lib.types.listOf lib.types.str; | ||
170 | default = []; | ||
171 | description = '' | ||
172 | List of paths to ignore when checking non-backed-up directories | ||
173 | Can use (POSIX extended) regex | ||
174 | ''; | ||
175 | }; | ||
69 | }; | 176 | }; |
70 | }); | 177 | }); |
71 | }; | 178 | }; |
@@ -76,76 +183,91 @@ in | |||
76 | install -m 0700 -o root -g root -d ${varDir} ${varDir}/caches | 183 | install -m 0700 -o root -g root -d ${varDir} ${varDir}/caches |
77 | ''; | 184 | ''; |
78 | secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v: | 185 | secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (k: v: |
79 | map (remote: [ | 186 | let |
80 | (lib.nameValuePair "backup/${varName k remote}/conf" { | 187 | bucket = if v.hash or true then builtins.hashString "sha256" v.bucket else v.bucket; |
188 | in map (remote: [ | ||
189 | (lib.nameValuePair "backup/${remote}_${k}/conf" { | ||
81 | permissions = "0400"; | 190 | permissions = "0400"; |
82 | text = duplyProfile v remote "${k}/"; | 191 | text = duplyProfile v remote bucket; |
83 | }) | 192 | }) |
84 | (lib.nameValuePair "backup/${varName k remote}/exclude" { | 193 | (lib.nameValuePair "backup/${remote}_${k}/exclude" { |
85 | permissions = "0400"; | 194 | permissions = "0400"; |
86 | text = v.excludeFile; | 195 | text = v.excludeFile + (builtins.concatStringsSep "\n" (map (p: "+ ${v.rootDir}/${p}") v.includedPaths)) + (lib.optionalString v.excludeRootDir '' |
196 | |||
197 | - ** | ||
198 | ''); | ||
87 | }) | 199 | }) |
88 | (lib.nameValuePair "backup/${varName k remote}" { | 200 | (lib.nameValuePair "backup/${remote}_${k}/pre" { |
201 | keyDependencies = [ | ||
202 | pkgs.bash | ||
203 | pkgs.rsync | ||
204 | ]; | ||
89 | permissions = "0500"; | 205 | permissions = "0500"; |
206 | text = let | ||
207 | remote' = cfg.remotes.${remote}; | ||
208 | in '' | ||
209 | #!${pkgs.stdenv.shell} | ||
210 | |||
211 | ${lib.optionalString (remote'.remote_type == "rsync") '' | ||
212 | # Recreate directory structure before synchronizing | ||
213 | mkdir -p ${varDir}/rsync_remotes/${remote}/${bucket} | ||
214 | ${pkgs.rsync}/bin/rsync -av -e \ | ||
215 | "ssh -p ${remote'.sshRsyncPort} -oIdentityFile=${config.secrets.fullPaths."backup/identity"}" \ | ||
216 | "${varDir}/rsync_remotes/${remote}/" \ | ||
217 | ${remote'.sshRsyncHost}: | ||
218 | ''} | ||
219 | ''; | ||
220 | }) | ||
221 | (lib.nameValuePair "backup/${remote}_${k}" { | ||
222 | permissions = "0700"; | ||
90 | isDir = true; | 223 | isDir = true; |
91 | }) | 224 | }) |
92 | ]) v.remotes) config.services.duplyBackup.profiles)); | 225 | ]) v.remotes) config.services.duplyBackup.profiles)) // { |
226 | "backup/identity" = { | ||
227 | permissions = "0400"; | ||
228 | text = "{{ .ssl_keys.duply_backup }}"; | ||
229 | }; | ||
230 | "backup/ignored_list" = { | ||
231 | permissions = "0400"; | ||
232 | text = let | ||
233 | ignored = map | ||
234 | (v: map (p: "${v.rootDir}/${p}") v.ignoredPaths) | ||
235 | (builtins.attrValues config.services.duplyBackup.profiles); | ||
236 | in builtins.concatStringsSep "\n" (lib.flatten ignored); | ||
237 | }; | ||
238 | "backup/backuped_list" = { | ||
239 | permissions = "0400"; | ||
240 | text = let | ||
241 | included = map | ||
242 | (v: map (p: "${v.rootDir}/${p}") v.includedPaths) | ||
243 | (builtins.attrValues config.services.duplyBackup.profiles); | ||
244 | in builtins.concatStringsSep "\n" (lib.flatten included); | ||
245 | }; | ||
246 | }; | ||
93 | 247 | ||
248 | programs.ssh.knownHostsFiles = [ | ||
249 | (pkgs.writeText | ||
250 | "duply_backup_known_hosts" | ||
251 | (builtins.concatStringsSep | ||
252 | "\n" | ||
253 | (builtins.filter | ||
254 | (v: v != null) | ||
255 | (builtins.map | ||
256 | (v: v.sshKnownHosts) | ||
257 | (builtins.attrValues cfg.remotes) | ||
258 | ) | ||
259 | ) | ||
260 | ) | ||
261 | ) | ||
262 | ]; | ||
263 | environment.systemPackages = [ pkgs.duply check_backups duply_backup_full_with_ignored duply_backup ]; | ||
94 | services.cron = { | 264 | services.cron = { |
95 | enable = true; | 265 | enable = true; |
96 | systemCronJobs = let | 266 | systemCronJobs = [ |
97 | backups = pkgs.writeScript "backups" '' | 267 | "0 0 * * * root ${duply_backup}/bin/duply_backup 90" |
98 | #!${pkgs.stdenv.shell} | 268 | ]; |
99 | |||
100 | ${builtins.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList (k: v: | ||
101 | map (remote: [ | ||
102 | '' | ||
103 | touch ${varDir}/${varName k remote}.log | ||
104 | ${pkgs.duply}/bin/duply ${config.secrets.fullPaths."backup/${varName k remote}"}/ ${action} --force >> ${varDir}/${varName k remote}.log | ||
105 | [[ $? = 0 ]] || echo -e "Error when doing backup for ${varName k remote}, see above\n---------------------------------------" >&2 | ||
106 | '' | ||
107 | ]) v.remotes | ||
108 | ) config.services.duplyBackup.profiles))} | ||
109 | ''; | ||
110 | in | ||
111 | [ | ||
112 | "0 2 * * * root ${backups}" | ||
113 | ]; | ||
114 | 269 | ||
115 | }; | 270 | }; |
116 | 271 | ||
117 | security.pki.certificateFiles = [ | ||
118 | (pkgs.fetchurl { | ||
119 | url = "http://downloads.e.eriomem.net/eriomemca.pem"; | ||
120 | sha256 = "1ixx4c6j3m26j8dp9a3dkvxc80v1nr5aqgmawwgs06bskasqkvvh"; | ||
121 | }) | ||
122 | ]; | ||
123 | |||
124 | myServices.monitoring.fromMasterActivatedPlugins = [ "eriomem" ]; | ||
125 | myServices.monitoring.fromMasterObjects.service = [ | ||
126 | { | ||
127 | service_description = "eriomem backup is up and not full"; | ||
128 | host_name = config.hostEnv.fqdn; | ||
129 | use = "external-service"; | ||
130 | check_command = "check_backup_eriomem"; | ||
131 | |||
132 | check_interval = 120; | ||
133 | notification_interval = "1440"; | ||
134 | |||
135 | servicegroups = "webstatus-backup"; | ||
136 | } | ||
137 | |||
138 | { | ||
139 | service_description = "ovh backup is up and not full"; | ||
140 | host_name = config.hostEnv.fqdn; | ||
141 | use = "external-service"; | ||
142 | check_command = "check_ok"; | ||
143 | |||
144 | check_interval = 120; | ||
145 | notification_interval = "1440"; | ||
146 | |||
147 | servicegroups = "webstatus-backup"; | ||
148 | } | ||
149 | ]; | ||
150 | }; | 272 | }; |
151 | } | 273 | } |