aboutsummaryrefslogtreecommitdiff
path: root/modules
diff options
context:
space:
mode:
authorIsmaël Bouya <ismael.bouya@normalesup.org>2020-06-18 06:47:08 +0200
committerIsmaël Bouya <ismael.bouya@normalesup.org>2020-06-18 06:47:08 +0200
commit171d8e1a8861e5844f6cb8d1623b93b0e86aabea (patch)
tree67c4d21e885e9a7b863329c6ed29742c7b490f4d /modules
parenta97118c489a59d723538292214efaa10dfcb96df (diff)
downloadNix-171d8e1a8861e5844f6cb8d1623b93b0e86aabea.tar.gz
Nix-171d8e1a8861e5844f6cb8d1623b93b0e86aabea.tar.zst
Nix-171d8e1a8861e5844f6cb8d1623b93b0e86aabea.zip
Add monitoring for duply backup
Diffstat (limited to 'modules')
-rw-r--r--modules/duply_backup/default.nix1
-rw-r--r--modules/private/monitoring/default.nix21
-rw-r--r--modules/private/monitoring/objects_common.nix1
-rw-r--r--modules/private/monitoring/objects_monitoring-1.nix24
-rwxr-xr-xmodules/private/monitoring/plugins/check_eriomem_age66
-rw-r--r--modules/private/monitoring/to_objects.nix10
-rw-r--r--modules/private/system/monitoring-1.nix7
7 files changed, 125 insertions, 5 deletions
diff --git a/modules/duply_backup/default.nix b/modules/duply_backup/default.nix
index 1e115be..bce4d65 100644
--- a/modules/duply_backup/default.nix
+++ b/modules/duply_backup/default.nix
@@ -82,6 +82,7 @@ in
82 '' 82 ''
83 touch ${varDir}/${k}.log 83 touch ${varDir}/${k}.log
84 ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/${k}/ ${action} --force >> ${varDir}/${k}.log 84 ${pkgs.duply}/bin/duply ${config.secrets.location}/backup/${k}/ ${action} --force >> ${varDir}/${k}.log
85 [[ $? = 0 ]] || echo -e "Error when doing backup for ${k}, see above\n---------------------------------------" >&2
85 '' 86 ''
86 ) config.services.duplyBackup.profiles)} 87 ) config.services.duplyBackup.profiles)}
87 ''; 88 '';
diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index 8ae0b30..316c2dd 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -58,6 +58,9 @@ let
58 wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [ 58 wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
59 pkgs.s3cmd pkgs.python3 59 pkgs.s3cmd pkgs.python3
60 ]} 60 ]}
61 wrapProgram $out/check_eriomem_age --prefix PATH : ${lib.makeBinPath [
62 pkgs.duplicity
63 ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
61 wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [ 64 wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
62 pkgs.mailutils 65 pkgs.mailutils
63 ]} 66 ]}
@@ -136,7 +139,7 @@ let
136 lib.attrsets.optionalAttrs 139 lib.attrsets.optionalAttrs
137 (builtins.pathExists specific_file) 140 (builtins.pathExists specific_file)
138 (pkgs.callPackage specific_file { 141 (pkgs.callPackage specific_file {
139 inherit config emailCheck; 142 inherit config nodes emailCheck;
140 hostFQDN = config.hostEnv.fqdn; 143 hostFQDN = config.hostEnv.fqdn;
141 hostName = name; 144 hostName = name;
142 }); 145 });
@@ -232,10 +235,22 @@ in
232 dest = "naemon/id_rsa"; 235 dest = "naemon/id_rsa";
233 user = "naemon"; 236 user = "naemon";
234 group = "naemon"; 237 group = "naemon";
235 premissions = "0400"; 238 permissions = "0400";
236 text = config.myEnv.monitoring.ssh_secret_key; 239 text = config.myEnv.monitoring.ssh_secret_key;
237 } 240 }
238 ]; 241 ] ++ lib.optional cfg.master (
242 {
243 dest = "eriomem_access_key";
244 user = "naemon";
245 group = "naemon";
246 permissions = "0400";
247 text = ''
248 export AWS_ACCESS_KEY_ID="${config.myEnv.backup.accessKeyId}"
249 export AWS_SECRET_ACCESS_KEY="${config.myEnv.backup.secretAccessKey}"
250 export BASE_URL="${config.myEnv.backup.remote}"
251 '';
252 }
253 );
239 # needed since extraResource is not in the closure 254 # needed since extraResource is not in the closure
240 systemd.services.naemon.path = [ myplugins ]; 255 systemd.services.naemon.path = [ myplugins ];
241 services.naemon = { 256 services.naemon = {
diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix
index 2585c38..c0a17e6 100644
--- a/modules/private/monitoring/objects_common.nix
+++ b/modules/private/monitoring/objects_common.nix
@@ -94,6 +94,7 @@ in
94 check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$"; 94 check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$";
95 check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$"; 95 check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$";
96 check_eriomem = "$USER2$/check_eriomem $USER208$"; 96 check_eriomem = "$USER2$/check_eriomem $USER208$";
97 check_eriomem_age = "$USER2$/check_eriomem_age $ARG1$";
97 check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$"; 98 check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$";
98 check_ftp_database = "$USER2$/check_ftp_database"; 99 check_ftp_database = "$USER2$/check_ftp_database";
99 check_git = "$USER2$/check_git $USER203$"; 100 check_git = "$USER2$/check_git $USER203$";
diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix
index ec6fdce..02870ed 100644
--- a/modules/private/monitoring/objects_monitoring-1.nix
+++ b/modules/private/monitoring/objects_monitoring-1.nix
@@ -1,4 +1,23 @@
1{ config, pkgs, hostFQDN, emailCheck, ... }: 1{ config, pkgs, nodes, hostFQDN, emailCheck, ... }:
2let
3 to_eriomem_age_dependency = name: {
4 dependent_host_name = "eldiron.immae.eu";
5 host_name = "eldiron.immae.eu";
6 dependent_service_description = "Eriomem backup for ${name} is not too old";
7 service_description = "Eriomem backup is up and not full";
8 execution_failure_criteria = "u";
9 notification_failure_criteria = "u";
10 };
11 to_eriomem_age = name: {
12 service_description = "Eriomem backup for ${name} is not too old";
13 host_name = "eldiron.immae.eu";
14 use = "external-service";
15 check_command = ["check_eriomem_age" name];
16
17 check_interval = "120";
18 notification_interval = "120";
19 };
20in
2{ 21{
3 host = { 22 host = {
4 # Dummy host for testing 23 # Dummy host for testing
@@ -654,7 +673,7 @@
654 _webstatus_name = "LDAP"; 673 _webstatus_name = "LDAP";
655 _webstatus_url = "ldap.immae.eu"; 674 _webstatus_url = "ldap.immae.eu";
656 } 675 }
657 ]; 676 ] ++ map to_eriomem_age (builtins.attrNames nodes.eldiron.config.services.duplyBackup.profiles);
658 contact = { 677 contact = {
659 telio-tortay = config.myEnv.monitoring.contacts.telio-tortay // { 678 telio-tortay = config.myEnv.monitoring.contacts.telio-tortay // {
660 use = "generic-contact"; 679 use = "generic-contact";
@@ -664,4 +683,5 @@
664 contactgroup = { 683 contactgroup = {
665 telio-tortay = { alias = "Telio Tortay"; members = "immae"; }; 684 telio-tortay = { alias = "Telio Tortay"; members = "immae"; };
666 }; 685 };
686 servicedependency = map to_eriomem_age_dependency (builtins.attrNames nodes.eldiron.config.services.duplyBackup.profiles);
667} 687}
diff --git a/modules/private/monitoring/plugins/check_eriomem_age b/modules/private/monitoring/plugins/check_eriomem_age
new file mode 100755
index 0000000..4d03b82
--- /dev/null
+++ b/modules/private/monitoring/plugins/check_eriomem_age
@@ -0,0 +1,66 @@
1#!/usr/bin/env bash
2
3set -euo pipefail
4
5source $SECRETS_PATH
6export HOME=$(mktemp -d)
7
8trap "rm -rf $HOME" EXIT
9folder=$1
10
11parse_date() {
12 d=$1
13 echo $d | sed -e "s/^\(....\)\(..\)\(..\)T\(..\)\(..\)\(..\)/\1-\2-\3T\4:\5:\6/"
14}
15
16output=$(duplicity collection-status --log-fd 2 "$BASE_URL$folder" 2>&1 > /dev/null)
17
18output=$(echo "$output" | grep -v "^\.")
19
20last_full=$(parse_date $(echo "$output" | grep "^ full " | cut -d' ' -f3 | sort | tail -n1))
21last_bkp=$(parse_date $(echo "$output" | grep -E "^ (full|inc) " | cut -d' ' -f3 | sort | tail -n1))
22orphaned_sets=$(echo "$output" | grep "^orphaned-sets-num" | cut -d' ' -f2)
23incomplete_sets=$(echo "$output" | grep "^incomplete-sets-num" | cut -d' ' -f2)
24
25if [[ -z "$last_full" || -z "$last_bkp" || -z "$orphaned_sets" || -z "$incomplete_sets" ]]; then
26 echo "duply-backup $folder UNKNOWN - impossible to parse result"
27 exit 3
28fi
29
30last_full_age=$(( ($(date "+%s") - $(date -d "$last_full" "+%s")) / (60*60*24) ))
31last_bkp_age=$(( ($(date "+%s") - $(date -d "$last_bkp" "+%s")) / (60*60) ))
32
33PERFS="orphan=$orphaned_sets;1;;0; incomplete=$incomplete_sets;1;;0; age=${last_bkp_age}h;30;48;0; full_age=${last_full_age}d;35;45;0;"
34
35
36WARNINGS=""
37ERRORS=""
38if [[ "$incomplete_sets" -gt 0 ]]; then
39 WARNINGS="$WARNINGS - Incomplete sets is $incomplete_sets"
40fi
41
42if [[ "$orphaned_sets" -gt 0 ]]; then
43 WARNINGS="$WARNINGS - Orphaned sets is $orphaned_sets"
44fi
45
46if [[ "$last_full_age" -gt 45 ]]; then
47 ERRORS="$ERRORS - Last full backup is too old $last_full"
48elif [[ "$last_full_age" -gt 35 ]]; then
49 WARNINGS="$WARNINGS - Last full backup is getting old $last_full"
50fi
51
52if [[ "$last_bkp_age" -gt 48 ]]; then
53 ERRORS="$ERRORS - Last backup is too old $last_bkp"
54elif [[ "$last_bkp_age" -gt 30 ]]; then
55 WARNINGS="$WARNINGS - Last backup is getting old $last_bkp"
56fi
57
58if [[ -n "$ERRORS" ]]; then
59 echo "duply-backup $folder CRITICAL$ERRORS$WARNINGS | $PERFS"
60 exit 2
61elif [[ -n "$WARNINGS" ]]; then
62 echo "duply-backup $folder WARNING$WARNINGS | $PERFS"
63 exit 1
64else
65 echo "duply-backup $folder OK | $PERFS"
66fi
diff --git a/modules/private/monitoring/to_objects.nix b/modules/private/monitoring/to_objects.nix
index 7b4b523..12721d2 100644
--- a/modules/private/monitoring/to_objects.nix
+++ b/modules/private/monitoring/to_objects.nix
@@ -38,7 +38,15 @@ let
38 ) v)} 38 ) v)}
39 } 39 }
40 ''; 40 '';
41 toOtherNoName = keyname: v: ''
42 define ${keyname} {
43 ${builtins.concatStringsSep "\n" (mapAttrsToList (kk: vv:
44 " ${pad 30 kk} ${vv}"
45 ) v)}
46 }
47 '';
41 toOthers = keyname: a: builtins.concatStringsSep "\n" (mapAttrsToList (toOther keyname) a); 48 toOthers = keyname: a: builtins.concatStringsSep "\n" (mapAttrsToList (toOther keyname) a);
49 toOthersArray = keyname: a: builtins.concatStringsSep "\n" (map (toOtherNoName keyname) a);
42 50
43 toTemplate = keyname: k: v: '' 51 toTemplate = keyname: k: v: ''
44 define ${keyname} { 52 define ${keyname} {
@@ -61,6 +69,8 @@ let
61 then toTemplates v 69 then toTemplates v
62 else if builtins.elem keyname ["hostgroup" "host" "contactgroup" "contact" "timeperiod" "servicegroup"] 70 else if builtins.elem keyname ["hostgroup" "host" "contactgroup" "contact" "timeperiod" "servicegroup"]
63 then toOthers keyname v 71 then toOthers keyname v
72 else if builtins.elem keyname ["servicedependency"]
73 then toOthersArray keyname v
64 else ""; 74 else "";
65 toObjects = v: builtins.concatStringsSep "\n" (mapAttrsToList toObjects' v); 75 toObjects = v: builtins.concatStringsSep "\n" (mapAttrsToList toObjects' v);
66in 76in
diff --git a/modules/private/system/monitoring-1.nix b/modules/private/system/monitoring-1.nix
index 7581c01..c87c784 100644
--- a/modules/private/system/monitoring-1.nix
+++ b/modules/private/system/monitoring-1.nix
@@ -31,6 +31,13 @@
31 }; 31 };
32 myServices.mailRelay.enable = true; 32 myServices.mailRelay.enable = true;
33 33
34 security.pki.certificateFiles = [
35 (pkgs.fetchurl {
36 url = "http://downloads.e.eriomem.net/eriomemca.pem";
37 sha256 = "1ixx4c6j3m26j8dp9a3dkvxc80v1nr5aqgmawwgs06bskasqkvvh";
38 })
39 ];
40
34 # This value determines the NixOS release with which your system is 41 # This value determines the NixOS release with which your system is
35 # to be compatible, in order to avoid breaking some software such as 42 # to be compatible, in order to avoid breaking some software such as
36 # database servers. You should change this only after NixOS release 43 # database servers. You should change this only after NixOS release