aboutsummaryrefslogtreecommitdiff
path: root/flakes/private/monitoring
diff options
context:
space:
mode:
authorIsmaël Bouya <ismael.bouya@normalesup.org>2023-10-04 01:35:06 +0200
committerIsmaël Bouya <ismael.bouya@normalesup.org>2023-10-04 02:11:48 +0200
commit1a64deeb894dc95e2645a75771732c6cc53a79ad (patch)
tree1b9df4838f894577a09b9b260151756272efeb53 /flakes/private/monitoring
parentfa25ffd4583cc362075cd5e1b4130f33306103f0 (diff)
downloadNix-1a64deeb894dc95e2645a75771732c6cc53a79ad.tar.gz
Nix-1a64deeb894dc95e2645a75771732c6cc53a79ad.tar.zst
Nix-1a64deeb894dc95e2645a75771732c6cc53a79ad.zip
Squash changes containing private information
There were a lot of changes since the previous commit, but a lot of them contained personnal information about users. All thos changes got stashed into a single commit (history is kept in a different place) and private information was moved in a separate private repository
Diffstat (limited to 'flakes/private/monitoring')
-rw-r--r--flakes/private/monitoring/flake.lock67
-rw-r--r--flakes/private/monitoring/flake.nix267
-rw-r--r--flakes/private/monitoring/myplugins.nix400
-rw-r--r--flakes/private/monitoring/nagios-cli.cfg68
-rw-r--r--flakes/private/monitoring/objects_common.nix227
-rwxr-xr-xflakes/private/monitoring/plugins/check_backup_age66
-rwxr-xr-xflakes/private/monitoring/plugins/check_bandwidth122
-rwxr-xr-xflakes/private/monitoring/plugins/check_command113
-rwxr-xr-xflakes/private/monitoring/plugins/check_emails121
-rwxr-xr-xflakes/private/monitoring/plugins/check_eriomem83
-rwxr-xr-xflakes/private/monitoring/plugins/check_ftp_database11
-rwxr-xr-xflakes/private/monitoring/plugins/check_git81
-rwxr-xr-xflakes/private/monitoring/plugins/check_imap_connection52
-rwxr-xr-xflakes/private/monitoring/plugins/check_last_file_date28
-rwxr-xr-xflakes/private/monitoring/plugins/check_mem.sh31
-rwxr-xr-xflakes/private/monitoring/plugins/check_mysql_replication41
-rwxr-xr-xflakes/private/monitoring/plugins/check_openldap_replication54
-rwxr-xr-xflakes/private/monitoring/plugins/check_ovh_sms25
-rwxr-xr-xflakes/private/monitoring/plugins/check_postgres_database_count32
-rwxr-xr-xflakes/private/monitoring/plugins/check_postgres_replication35
-rwxr-xr-xflakes/private/monitoring/plugins/check_redis_replication38
-rwxr-xr-xflakes/private/monitoring/plugins/check_zfs_snapshot325
-rwxr-xr-xflakes/private/monitoring/plugins/notify_by_apprise31
-rwxr-xr-xflakes/private/monitoring/plugins/notify_by_email29
-rwxr-xr-xflakes/private/monitoring/plugins/notify_by_slack46
-rwxr-xr-xflakes/private/monitoring/plugins/send_nrdp.sh57
-rwxr-xr-xflakes/private/monitoring/send_mails15
-rw-r--r--flakes/private/monitoring/to_objects.nix77
28 files changed, 2542 insertions, 0 deletions
diff --git a/flakes/private/monitoring/flake.lock b/flakes/private/monitoring/flake.lock
new file mode 100644
index 0000000..2e156fb
--- /dev/null
+++ b/flakes/private/monitoring/flake.lock
@@ -0,0 +1,67 @@
1{
2 "nodes": {
3 "environment": {
4 "locked": {
5 "lastModified": 1,
6 "narHash": "sha256-rMKbM7fHqWQbI7y59BsPG8KwoDj2jyrvN2niPWB24uE=",
7 "path": "../environment",
8 "type": "path"
9 },
10 "original": {
11 "path": "../environment",
12 "type": "path"
13 }
14 },
15 "naemon": {
16 "locked": {
17 "lastModified": 1,
18 "narHash": "sha256-6le57WLKj1HXdhe4cgYO6N0Z9nJZC+plQY8HhOwzEIk=",
19 "path": "../../naemon",
20 "type": "path"
21 },
22 "original": {
23 "path": "../../naemon",
24 "type": "path"
25 }
26 },
27 "nixpkgs-lib": {
28 "locked": {
29 "dir": "lib",
30 "lastModified": 1691269286,
31 "narHash": "sha256-7cPTz1bPhwq8smt9rHDcFtJsd1tFDcBukzj5jOXqjfk=",
32 "owner": "NixOS",
33 "repo": "nixpkgs",
34 "rev": "85d4248a4f5aa6bc55dd2cea8131bb68b2d43804",
35 "type": "github"
36 },
37 "original": {
38 "dir": "lib",
39 "owner": "NixOS",
40 "repo": "nixpkgs",
41 "type": "github"
42 }
43 },
44 "root": {
45 "inputs": {
46 "environment": "environment",
47 "naemon": "naemon",
48 "nixpkgs-lib": "nixpkgs-lib",
49 "secrets": "secrets"
50 }
51 },
52 "secrets": {
53 "locked": {
54 "lastModified": 1,
55 "narHash": "sha256-5AakznhrJFmwCD7lr4JEh55MtdAJL6WA/YuBks6ISSE=",
56 "path": "../../secrets",
57 "type": "path"
58 },
59 "original": {
60 "path": "../../secrets",
61 "type": "path"
62 }
63 }
64 },
65 "root": "root",
66 "version": 7
67}
diff --git a/flakes/private/monitoring/flake.nix b/flakes/private/monitoring/flake.nix
new file mode 100644
index 0000000..b7c3997
--- /dev/null
+++ b/flakes/private/monitoring/flake.nix
@@ -0,0 +1,267 @@
1{
2 inputs = {
3 environment.url = "path:../environment";
4 secrets.url = "path:../../secrets";
5 naemon.url = "path:../../naemon";
6 nixpkgs-lib.url = "github:NixOS/nixpkgs?dir=lib";
7 };
8 outputs = { self, environment, nixpkgs-lib, secrets, naemon }: {
9 nagios-cli-config = ./nagios-cli.cfg;
10 lib = rec {
11 expandedObject = kind: object: objects:
12 if object ? "use"
13 then expandedObject kind objects.templates.${kind}.${object.use} objects // object
14 else object;
15
16 objectsCommon = import ./objects_common.nix;
17 toObjects = import ./to_objects.nix { inherit (nixpkgs-lib) lib; };
18
19 toMasterPassiveObject = svcTemplate: freshnessThresholdMultiplier: objects:
20 {
21 service = with nixpkgs-lib.lib; map (s:
22 {
23 host_name = (expandedObject "service" s objects).host_name;
24 use = svcTemplate;
25 retry_interval = "1";
26 freshness_threshold = let
27 fs = expandedObject "service" s objects;
28 in if builtins.isInt fs.check_interval
29 then builtins.ceil (freshnessThresholdMultiplier * 60 * fs.check_interval)
30 else fs.check_interval;
31 }
32 // filterAttrs (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_") s
33 // mapAttrs'
34 (n: nameValuePair (removePrefix "__passive_" n))
35 (filterAttrs (k: _: hasPrefix "__passive_" k) s)
36 ) objects.service;
37 host = objects.host;
38 };
39
40 emailCheck = allCfg: host: hostFQDN: let
41 cfg = allCfg."${host}";
42 reverseTargets = builtins.attrNames (nixpkgs-lib.lib.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
43 to_email = cfg': host':
44 let sep = if nixpkgs-lib.lib.hasInfix "+" cfg'.mail_address then "_" else "+";
45 in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
46 mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
47 mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
48 command = if cfg.local
49 then
50 [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
51 else
52 [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
53 in
54 {
55 service_description = "${hostFQDN} email service is active";
56 use = "mail-service";
57 host_name = hostFQDN;
58 servicegroups = "webstatus-email";
59 check_command = command;
60 };
61 };
62 nixosModule = self.nixosModules.monitoring;
63 nixosModules.monitoring = { config, pkgs, lib, ... }:
64 let
65 cfg = config.myServices.monitoring;
66 allPluginsConfig = import ./myplugins.nix {
67 inherit pkgs lib config;
68 sudo = "/run/wrappers/bin/sudo";
69 };
70 mypluginsConfig = lib.mapAttrs (n: v:
71 if builtins.isFunction v
72 then v (cfg.pluginsArgs."${n}" or {})
73 else v
74 ) (lib.getAttrs cfg.activatedPlugins allPluginsConfig);
75 myplugins = let
76 mypluginsChunk = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: v.chunk or "") mypluginsConfig);
77 in pkgs.runCommand "buildplugins" {
78 buildInputs = [ pkgs.makeWrapper pkgs.perl ];
79 } ''
80 mkdir $out
81 ${mypluginsChunk}
82 '';
83 objectsModule = with lib.types; submodule {
84 options = {
85 command = lib.mkOption {
86 type = attrsOf str;
87 default = {};
88 description = "Command definitions";
89 };
90
91 host = lib.mkOption {
92 type = attrsOf (attrsOf str);
93 default = {};
94 description = "Host definitions";
95 };
96 hostgroup = lib.mkOption {
97 type = attrsOf (attrsOf str);
98 default = {};
99 description = "Host group definitions";
100 };
101 hostdependency = lib.mkOption {
102 type = listOf (attrsOf str);
103 default = [];
104 description = "Host dependency definitions";
105 };
106
107 service = lib.mkOption {
108 type = listOf (attrsOf (oneOf [ str (listOf str) int ]));
109 # str -> string
110 # listOf str -> list to be concatenated with "!"
111 # int -> toString
112 default = [];
113 description = "Service definitions";
114 };
115 servicegroup = lib.mkOption {
116 type = attrsOf (attrsOf str);
117 default = {};
118 description = "Service group definitions";
119 };
120 servicedependency = lib.mkOption {
121 type = listOf (attrsOf str);
122 default = [];
123 description = "Service dependency definitions";
124 };
125
126 contact = lib.mkOption {
127 type = attrsOf (attrsOf str);
128 default = {};
129 description = "Contact definitions";
130 };
131 contactgroup = lib.mkOption {
132 type = attrsOf (attrsOf str);
133 default = {};
134 description = "Contact group definitions";
135 };
136
137 timeperiod = lib.mkOption {
138 type = attrsOf (attrsOf str);
139 default = {};
140 description = "Time period definitions";
141 };
142
143 templates = lib.mkOption {
144 description = "Template definitions";
145 default = {};
146 type = submodule {
147 options = {
148 service = lib.mkOption { type = attrsOf (attrsOf (either str int)); default = {}; };
149 contact = lib.mkOption { type = attrsOf (attrsOf str); default = {}; };
150 host = lib.mkOption { type = attrsOf (attrsOf str); default = {}; };
151 };
152 };
153 };
154 };
155 };
156 in
157 {
158 options = {
159 myServices.monitoring = {
160 enable = lib.mkOption {
161 type = lib.types.bool;
162 default = false;
163 description = ''
164 Whether to enable monitoring.
165 '';
166 };
167 master = lib.mkOption {
168 type = lib.types.bool;
169 default = false;
170 description = ''
171 This instance is the master instance
172 '';
173 };
174 pluginsArgs = lib.mkOption {
175 default = {};
176 description = "Arguments to pass to the naemon plugin configuration";
177 type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified);
178 };
179 activatedPlugins = lib.mkOption {
180 default = [];
181 description = "List of naemon plugins to activate";
182 type = lib.types.listOf (lib.types.enum (builtins.attrNames allPluginsConfig));
183 };
184 fromMasterActivatedPlugins = lib.mkOption {
185 default = [];
186 description = "List of naemon plugins to activate from master";
187 type = lib.types.listOf (lib.types.str);
188 };
189 resources = lib.mkOption {
190 default = {};
191 description = "List of additionnal resources elements";
192 type = lib.types.attrsOf (lib.types.str);
193 };
194 objects = lib.mkOption {
195 default = {};
196 description = "Object definitions";
197 type = objectsModule;
198 };
199 fromMasterObjects = lib.mkOption {
200 default = {};
201 description = "Object definitions of checks that should be executed from master";
202 type = objectsModule;
203 };
204 };
205 };
206
207 imports = [
208 environment.nixosModule
209 secrets.nixosModule
210 naemon.nixosModule
211 ];
212 config = lib.mkIf cfg.enable {
213 myServices.monitoring.objects.command =
214 lib.foldr (v: o: o // (v.commands or {})) {} (builtins.attrValues mypluginsConfig);
215
216 security.sudo.extraRules = let
217 pluginsSudo = lib.lists.remove null (lib.mapAttrsToList (k: v:
218 if (v ? sudo)
219 then ({ users = [ "naemon" ]; } // (v.sudo myplugins))
220 else null) mypluginsConfig);
221 in pluginsSudo;
222
223 environment.etc.cnagios.source = "${pkgs.cnagios}/share/doc/cnagios";
224 environment.systemPackages = let
225 nagios-cli = pkgs.writeScriptBin "nagios-cli" ''
226 #!${pkgs.stdenv.shell}
227 sudo -u naemon ${pkgs.nagios-cli}/bin/nagios-cli -c ${self.nagios-cli-config}
228 '';
229 in [
230 pkgs.cnagios
231 nagios-cli
232 ];
233 secrets.keys = {
234 "naemon/id_rsa" = {
235 user = "naemon";
236 group = "naemon";
237 permissions = "0400";
238 text = config.myEnv.monitoring.ssh_secret_key;
239 };
240 "naemon/resources.cfg".keyDependencies = [ myplugins ];
241 };
242 services.naemon = {
243 enable = true;
244 extraConfig = ''
245 use_syslog=1
246 log_initial_states=1
247 date_format=iso8601
248 admin_email=${config.myEnv.monitoring.email}
249 '' + lib.optionalString (!cfg.master) ''
250 obsess_over_services=1
251 ocsp_command=notify-master
252 '';
253 extraResource = let
254 resources = [cfg.resources or {}] ++ (lib.mapAttrsToList (k: v: v.resources or {}) mypluginsConfig);
255 joined = lib.zipAttrsWith (n: v: if builtins.length (lib.unique v) == 1 then builtins.head v else abort "Non-unique resources names") resources;
256 joinedStr = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "$" + "${k}$=${v}") joined);
257 in ''
258 $USER2$=${myplugins}
259 ${joinedStr}
260 '';
261 objectDefs =
262 self.lib.toObjects cfg.objects;
263 };
264 };
265 };
266 };
267}
diff --git a/flakes/private/monitoring/myplugins.nix b/flakes/private/monitoring/myplugins.nix
new file mode 100644
index 0000000..35730bb
--- /dev/null
+++ b/flakes/private/monitoring/myplugins.nix
@@ -0,0 +1,400 @@
1{ sudo, pkgs, lib, config }:
2let
3 cfg = config.myServices.monitoring;
4in
5{
6 notify-secondary = {
7 resources = {
8 USER200 = config.myEnv.monitoring.status_url;
9 USER201 = config.myEnv.monitoring.status_token;
10 };
11 commands = {
12 notify-master = "$USER2$/send_nrdp.sh -u \"$USER200$\" -t \"$USER201$\" -H \"$HOSTADDRESS$\" -s \"$SERVICEDESC$\" -S \"$SERVICESTATEID$\" -o \"$SERVICEOUTPUT$ | $SERVICEPERFDATA$\"";
13 };
14 chunk = ''
15 cp ${./plugins}/send_nrdp.sh $out
16 patchShebangs $out/send_nrdp.sh
17 wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
18 pkgs.curl pkgs.jq
19 ]}
20 '';
21 };
22 notify-primary = {
23 resources = {
24 USER210 = config.myEnv.monitoring.apprise_urls;
25 };
26 commands = {
27 # $OVE is to force naemon to run via shell instead of execve which fails here
28 notify-host-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" HOSTSTATE=\"$HOSTSTATE$\" HOSTOUTPUT=\"$HOSTOUTPUT$\" $USER2$/notify_by_email host \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE";
29 # $OVE is to force naemon to run via shell instead of execve which fails here
30 notify-service-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE";
31 notify-host-by-apprise = "HOST=\"$HOSTALIAS$\" NOTIFICATIONTYPE=\"$NOTIFICATIONTYPE$\" HOSTSTATE=\"$HOSTSTATE$\" HOSTOUTPUT=\"$HOSTOUTPUT$\" $USER2$/notify_by_apprise host \"$ARG1$\"";
32 notify-service-by-apprise = "HOST=\"$HOSTALIAS$\" NOTIFICATIONTYPE=\"$NOTIFICATIONTYPE$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_apprise service \"$ARG1$\"";
33 };
34 chunk = ''
35 cp ${./plugins}/{notify_by_email,notify_by_apprise} $out
36 patchShebangs $out/{notify_by_email,notify_by_apprise}
37 wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
38 pkgs.mailutils
39 ]}
40 wrapProgram $out/notify_by_apprise --prefix PATH : ${lib.makeBinPath [
41 pkgs.apprise
42 ]}
43 '';
44 };
45 bandwidth = {
46 commands = {
47 check_local_bandwidth = "$USER2$/check_bandwidth -i=$ARG1$ -w $ARG2$ -c $ARG3$";
48 };
49 chunk = ''
50 cp ${./plugins}/check_bandwidth $out/
51 patchShebangs $out/check_bandwidth
52 wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
53 pkgs.iproute pkgs.bc
54 ]}
55 '';
56 };
57 command = {
58 commands = {
59 check_command_match = "$USER2$/check_command -c \"$ARG1$\" -C \"$ARG2$\" $ARG3$";
60 check_command_output = "$USER2$/check_command -c \"$ARG1$\" -s 0 -o \"$ARG2$\" $ARG3$";
61 check_command_status = "$USER2$/check_command -c \"$ARG1$\" -s \"$ARG2$\" $ARG3$";
62 };
63 chunk = ''
64 cp ${./plugins}/check_command $out/
65 patchShebangs $out/check_command
66 wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
67 '';
68 };
69 dns = {
70 commands = {
71 check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$";
72 check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$";
73 };
74 };
75 mdadm = {
76 commands = {
77 check_mdadm = "$USER2$/check_command -c \"${pkgs.mdadm}/bin/mdadm --monitor --scan -1\" -s 0 -o \"^$\" -r root";
78 };
79 sudo = _: {
80 commands = [
81 { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
82 ];
83 runAs = "root";
84 };
85 };
86 postfix = {
87 commands = {
88 check_mailq = "$USER1$/check_mailq -s -w 1 -c 2";
89 };
90 sudo = _: {
91 commands = [
92 { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
93 ];
94 runAs = "root";
95 };
96 };
97 emails = {
98 resources = {
99 USER203 = config.secrets.fullPaths."naemon/id_rsa";
100 };
101 commands = {
102 check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$";
103 check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$";
104 };
105 chunk = let
106 send_mails = pkgs.runCommand "send_mails" {
107 buildInputs = [ pkgs.makeWrapper ];
108 } ''
109 mkdir -p $out/bin
110 cp ${./send_mails} $out/bin/send_mails
111 patchShebangs $out
112 wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
113 pkgs.mailutils
114 ]}
115 '';
116 in ''
117 cp ${./plugins}/check_emails $out/
118 patchShebangs $out/check_emails
119 wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
120 pkgs.openssh send_mails
121 ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
122 pkgs.perlPackages.TimeDate
123 ]}
124 '';
125 };
126 eriomem = {
127 resources = {
128 USER208 = builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys);
129 };
130 commands = {
131 check_backup_eriomem = "$USER2$/check_eriomem $USER208$";
132 check_backup_eriomem_age = "$USER2$/check_backup_eriomem_age $ARG1$";
133 };
134 chunk = ''
135 cp ${./plugins}/check_eriomem $out/
136 patchShebangs $out/check_eriomem
137 wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
138 pkgs.s3cmd pkgs.python38
139 ]}
140 cp ${./plugins}/check_backup_age $out/check_backup_eriomem_age
141 patchShebangs $out/check_backup_eriomem_age
142 wrapProgram $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
143 pkgs.duplicity
144 ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
145 '';
146 };
147 file_date = {
148 commands = {
149 check_last_file_date = "${sudo} -u \"$ARG3$\" $USER2$/check_last_file_date \"$ARG1$\" \"$ARG2$\"";
150 };
151 chunk = ''
152 cp ${./plugins}/check_last_file_date $out/
153 patchShebangs $out/check_last_file_date
154 '';
155 sudo = myplugins: {
156 commands = [
157 { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
158 ];
159 runAs = "ALL";
160 };
161 };
162 ftp = {
163 commands = {
164 check_ftp_database = "$USER2$/check_ftp_database";
165 };
166 chunk = ''
167 cp ${./plugins}/check_ftp_database $out/
168 patchShebangs $out/check_ftp_database
169 wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
170 pkgs.lftp
171 ]}
172 '';
173 };
174 git = {
175 resources = {
176 USER203 = config.secrets.fullPaths."naemon/id_rsa";
177 };
178 commands = {
179 check_git = "$USER2$/check_git $USER203$";
180 };
181 chunk = ''
182 cp ${./plugins}/check_git $out/
183 patchShebangs $out/check_git
184 wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [
185 pkgs.git pkgs.openssh
186 ]}
187 '';
188 };
189 http = {
190 resources = {
191 USER202 = config.myEnv.monitoring.http_user_password;
192 };
193 commands = {
194 check_http = "$USER1$/check_http --sni -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\"";
195 check_https = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\"";
196 check_https_4 = "$USER1$/check_http -4 --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\"";
197 check_https_6 = "$USER1$/check_http -6 --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\"";
198 check_https_auth = "$USER1$/check_http --sni --ssl -a \"$USER202$\" -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\"";
199 check_https_certificate = "$USER1$/check_http --sni --ssl -H \"$ARG1$\" -C 21,15";
200 check_https_code = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -e \"$ARG3$\" -r \"$ARG4$\"";
201 };
202 };
203 imap = {
204 resources = {
205 USER204 = config.myEnv.monitoring.imap_login;
206 USER205 = config.myEnv.monitoring.imap_password;
207 };
208 commands = {
209 check_imap_connection = "$USER2$/check_imap_connection -u \"$USER204$\" -p \"$USER205$\" -H \"imap.immae.eu:143\"";
210 };
211 chunk = ''
212 cp ${./plugins}/check_imap_connection $out/
213 patchShebangs $out/check_imap_connection
214 wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [
215 pkgs.openssl
216 ]}
217 '';
218 };
219 megaraid = let
220 megacli = pkgs.megacli.overrideAttrs(old: { meta = old.meta // { license = null; }; });
221 in {
222 commands = {
223 check_megaraid = "$USER2$/check_megaraid_sas --sudo";
224 };
225 chunk = let
226 megaCliPlugin = pkgs.runCommand "megaCliPlugin" {
227 plugin = pkgs.fetchurl {
228 name = "check_megaraid_sas";
229 url = "https://exchange.nagios.org/components/com_mtree/attachment.php?link_id=6381&cf_id=24";
230 sha256 = "0yf60p4c0hb4q3fng9fc14qc89bqm0f1sijayzygadaqcl44jx4p";
231 };
232 } ''
233 mkdir $out
234 cp $plugin $out/check_megaraid_sas
235 chmod +x $out/check_megaraid_sas
236 patchShebangs $out
237 substituteInPlace $out/check_megaraid_sas --replace /usr/sbin/MegaCli ${megacli}/bin/MegaCli64
238 substituteInPlace $out/check_megaraid_sas --replace 'sudo $megacli' '${sudo} $megacli'
239 sed -i -e "s/use utils qw(%ERRORS);/my %ERRORS = ('OK' => 0, 'WARNING' => 1, 'CRITICAL' => 2, 'UNKNOWN' => 3);/" $out/check_megaraid_sas
240 '';
241 in ''
242 cp ${megaCliPlugin}/check_megaraid_sas $out/
243 patchShebangs $out/check_megaraid_sas
244 '';
245 sudo = _: {
246 commands = [
247 { command = "${megacli}/bin/MegaCli64"; options = [ "NOPASSWD" ]; }
248 ];
249 runAs = "root";
250 };
251 };
252 memory = {
253 commands = {
254 check_memory = "$USER2$/check_mem.sh -w $ARG1$ -c $ARG2$";
255 };
256 chunk = ''
257 cp ${./plugins}/check_mem.sh $out/
258 patchShebangs $out/check_mem.sh
259 wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
260 pkgs.gnugrep pkgs.gawk pkgs.procps
261 ]}
262 '';
263 };
264 mysql = {
265 commands = {
266 check_mysql_replication = "${sudo} -u mysql $USER2$/check_mysql_replication \"$ARG1$\" \"$ARG2$\"";
267 };
268 chunk = ''
269 cp ${./plugins}/check_mysql_replication $out/
270 patchShebangs $out/check_mysql_replication
271 wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [
272 pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb
273 ]}
274 '';
275 sudo = myplugins: {
276 commands = [
277 { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; }
278 ];
279 runAs = "mysql";
280 };
281 };
282 openldap = {
283 commands = {
284 check_openldap_replication = "${sudo} -u openldap $USER2$/check_openldap_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\" \"$ARG4$\" \"$ARG5$\"";
285 };
286 chunk = ''
287 cp ${./plugins}/check_openldap_replication $out/
288 patchShebangs $out/check_openldap_replication
289 wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
290 pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
291 ]}
292 '';
293 sudo = myplugins: {
294 commands = [
295 { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; }
296 ];
297 runAs = "openldap";
298 };
299 };
300 ovh = {
301 resources = {
302 USER209 = builtins.concatStringsSep "," [
303 config.myEnv.monitoring.ovh_sms.endpoint
304 config.myEnv.monitoring.ovh_sms.application_key
305 config.myEnv.monitoring.ovh_sms.application_secret
306 config.myEnv.monitoring.ovh_sms.consumer_key
307 config.myEnv.monitoring.ovh_sms.account
308 ];
309 };
310 commands = {
311 check_backup_ovh_age = "$USER2$/check_backup_ovh_age $ARG1$";
312 check_ovh_sms = "$USER2$/check_ovh_sms \"$USER209$\"";
313 };
314 chunk = ''
315 cp ${./plugins}/check_backup_age $out/check_backup_ovh_age
316 patchShebangs $out/check_backup_ovh_age
317 wrapProgram $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
318 pkgs.duplicity
319 ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
320 cp ${./plugins}/check_ovh_sms $out/
321 patchShebangs $out/check_ovh_sms
322 wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
323 (pkgs.python38.withPackages (ps: [ps.ovh]))
324 ]}
325 '';
326 };
327 postgresql = { package }: {
328 commands = {
329 check_postgresql_replication = "${sudo} -u postgres $USER2$/check_postgres_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\"";
330 check_postgresql_database_count = "$USER2$/check_postgres_database_count \"$ARG1$\" \"$ARG2$\" \"$ARG3$\"";
331 };
332 chunk = ''
333 cp ${./plugins}/check_postgres_replication $out/
334 patchShebangs $out/check_postgres_replication
335 wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
336 package
337 ]}
338 cp ${./plugins}/check_postgres_database_count $out/
339 patchShebangs $out/check_postgres_database_count
340 wrapProgram $out/check_postgres_database_count --prefix PATH : ${lib.makeBinPath [
341 package
342 ]}
343 '';
344
345 sudo = myplugins: {
346 commands = [
347 { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
348 ];
349 runAs = "postgres";
350 };
351 };
352 redis = {
353 commands = {
354 check_redis_replication = "${sudo} -u redis $USER2$/check_redis_replication \"$ARG1$\"";
355 };
356 chunk = ''
357 cp ${./plugins}/check_redis_replication $out/
358 patchShebangs $out/check_redis_replication
359 wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [
360 pkgs.gnugrep pkgs.coreutils pkgs.redis
361 ]}
362 '';
363 sudo = myplugins: {
364 commands = [
365 { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; }
366 ];
367 runAs = "redis";
368 };
369 };
370 tcp = {
371 commands = {
372 check_tcp = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -e \"$ARG2$\" -Mcrit";
373 check_tcp_ssl = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -S -D 21,15";
374 };
375 };
376 zfs = {
377 commands = {
378 check_zfs = "$USER2$/check_zpool.sh -p ALL -w 80 -c 90";
379 check_zfs_snapshot = "$USER2$/check_zfs_snapshot -d $ARG1$ -c 18000 -w 14400";
380 };
381 chunk = let
382 zfsPlugin = pkgs.fetchurl {
383 url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh";
384 sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf";
385 };
386 in ''
387 cp ${zfsPlugin} $out/check_zpool.sh
388 chmod +x $out/check_zpool.sh
389 patchShebangs $out/check_zpool.sh
390 wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [
391 pkgs.which pkgs.zfs pkgs.gawk
392 ]}
393 cp ${./plugins}/check_zfs_snapshot $out
394 patchShebangs $out/check_zfs_snapshot
395 wrapProgram $out/check_zfs_snapshot --prefix PATH : ${lib.makeBinPath [
396 pkgs.zfs pkgs.coreutils pkgs.gawk pkgs.gnugrep
397 ]}
398 '';
399 };
400}
diff --git a/flakes/private/monitoring/nagios-cli.cfg b/flakes/private/monitoring/nagios-cli.cfg
new file mode 100644
index 0000000..7bd30cb
--- /dev/null
+++ b/flakes/private/monitoring/nagios-cli.cfg
@@ -0,0 +1,68 @@
1# -*- coding: utf-8; -*-
2
3[cli]
4history = /var/lib/naemon/nagios_cli_history
5
6[ui]
7color = 1
8prompt = naemon %s>
9prompt_separator = " → "
10
11[nagios]
12log = /var/log/naemon
13command_file = /run/naemon/naemon.cmd
14log_file = %(log)s/naemon.log
15object_cache_file = /var/lib/naemon/objects.cache
16status_file = /var/lib/naemon/status.dat
17
18[object]
19host.status =
20 host_name
21 current_state
22 plugin_output
23 is_flapping
24 last_check
25 last_time_down
26 last_state_change
27 check_period
28 notification_period
29 current_attempt
30 max_attempts
31service.status =
32 host_name
33 service_description
34 current_state
35 is_flapping
36 plugin_output
37 last_time_down
38 last_state_change
39 last_check
40 next_check
41 check_interval
42 check_latency
43 check_period
44 notification_period
45 current_attempt
46 max_attempts
47
48[string]
49level.ok = ↑ OK
50level.warning = ! WARNING
51level.critical = ↓ CRITICAL
52level.unknown = ↕ UNKNOWN
53
54[color]
55error = bold_red
56
57prompt = normal
58prompt.object = bold
59
60host.host_name = bold
61host.plugin_output = bold
62service.plugin_output = bold
63
64level.ok = bold_green
65level.warning = bold_yellow
66level.critical = bold_red
67level.unknown = bold_magenta
68
diff --git a/flakes/private/monitoring/objects_common.nix b/flakes/private/monitoring/objects_common.nix
new file mode 100644
index 0000000..1da7764
--- /dev/null
+++ b/flakes/private/monitoring/objects_common.nix
@@ -0,0 +1,227 @@
1{ hostFQDN
2, hostName
3, interface ? "eth0"
4, processWarn ? "250"
5, processAlert ? "400"
6, loadWarn ? "0.9"
7, load5Warn ? loadWarn
8, load15Warn ? load5Warn
9, loadAlert ? "1.0"
10, load5Alert ? loadAlert
11, load15Alert ? load5Alert
12, master
13, ...
14}:
15{
16 host = {
17 "${hostFQDN}" = {
18 alias = hostFQDN;
19 address = hostFQDN;
20 use = "linux-server";
21 hostgroups = "webstatus-hosts";
22 _webstatus_name = hostName;
23 _webstatus_vhost = "status.immae.eu";
24 };
25 };
26 service = [
27 {
28 service_description = "Size on root partition";
29 use = "local-service";
30 check_command = ["check_local_disk" "20%" "10%" "/"];
31 __passive_servicegroups = "webstatus-resources";
32 }
33 {
34 service_description = "Total number of process";
35 use = "local-service";
36 check_command = [
37 "check_local_procs"
38 processWarn
39 processAlert
40 "RSZDT"
41 ];
42 __passive_servicegroups = "webstatus-resources";
43 }
44 {
45 service_description = "Network bandwidth";
46 use = "local-service";
47 check_interval = 2;
48 max_check_attempts = "20";
49 retry_interval = "2";
50 check_command = [
51 "check_local_bandwidth"
52 interface
53 "20480" # kb/s
54 "51200" # kb/s
55 ];
56 __passive_servicegroups = "webstatus-resources";
57 }
58 {
59 service_description = "Average load";
60 use = "local-service";
61 check_command = [
62 "check_local_load"
63 "${loadWarn},${load5Warn},${load15Warn}"
64 "${loadAlert},${load5Alert},${load15Alert}"
65 ];
66 __passive_servicegroups = "webstatus-resources";
67 }
68 {
69 service_description = "Swap usage";
70 use = "local-service";
71 check_command = ["check_local_swap" "20" "10"];
72 __passive_servicegroups = "webstatus-resources";
73 }
74 {
75 service_description = "Memory usage";
76 use = "local-service";
77 check_command = ["check_memory" "80" "90"];
78 __passive_servicegroups = "webstatus-resources";
79 }
80 {
81 service_description = "NTP is activated and working";
82 use = "local-service";
83 check_command = ["check_ntp"];
84 __passive_servicegroups = "webstatus-resources";
85 }
86 ];
87 command = {
88 check_local_disk = "$USER1$/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$";
89 check_local_procs = "$USER1$/check_procs -w $ARG1$ -c $ARG2$ -s $ARG3$";
90 check_local_load = "$USER1$/check_load -r -w $ARG1$ -c $ARG2$";
91 check_local_swap = "$USER1$/check_swap -n ok -w $ARG1$ -c $ARG2$";
92 check_ntp = "$USER1$/check_ntp_time -t 30 -q -H 0.arch.pool.ntp.org";
93 check_smtp = "$USER1$/check_smtp -H $HOSTADDRESS$ -p 25 -S -D 21,15";
94
95 check_host_alive = "$USER1$/check_ping -H $HOSTADDRESS$ -w 3000.0,80% -c 5000.0,100% -p 5";
96 check_ok = "$USER1$/check_dummy 0 \"Dummy OK\"";
97 check_critical = "$USER1$/check_dummy 2 \"Dummy CRITICAL\"";
98 };
99 timeperiod = {
100 "24x7" = {
101 alias = "24 Hours A Day, 7 Days A Week";
102 monday = "00:00-24:00";
103 tuesday = "00:00-24:00";
104 wednesday = "00:00-24:00";
105 thursday = "00:00-24:00";
106 friday = "00:00-24:00";
107 saturday = "00:00-24:00";
108 sunday = "00:00-24:00";
109 };
110 };
111 servicegroup = {
112 webstatus-webapps = { alias = "Web applications"; };
113 webstatus-websites = { alias = "Personal websites"; };
114 webstatus-ssl = { alias = "SSL certificates"; };
115 webstatus-dns = { alias = "DNS resolution"; };
116 webstatus-remote-services = { alias = "Other remote services"; };
117 webstatus-local-services = { alias = "Other local services"; };
118 webstatus-email = { alias = "E-mail services"; };
119 webstatus-resources = { alias = "Local resources"; };
120 webstatus-databases = { alias = "Databases resources"; };
121 webstatus-backup = { alias = "Backup resources"; };
122 };
123 hostgroup = {
124 webstatus-hosts = { alias = "Hosts"; };
125 };
126 contactgroup = {
127 admins = { alias = "Naemon Administrators"; };
128 };
129 templates = {
130 service = {
131 generic-service = {
132 active_checks_enabled = "1";
133 check_freshness = "0";
134 check_interval = 10;
135 check_period = "24x7";
136 contact_groups = "admins";
137 event_handler_enabled = "1";
138 flap_detection_enabled = "1";
139 is_volatile = "0";
140 max_check_attempts = "3";
141 notification_interval = "60";
142 notification_options = "w,u,c,r,f,s";
143 notification_period = "24x7";
144 notifications_enabled = if master then "1" else "0";
145 obsess_over_service = "1";
146 passive_checks_enabled = "1";
147 process_perf_data = "1";
148 retain_nonstatus_information = "1";
149 retain_status_information = "1";
150 retry_interval = "2";
151 _webstatus_namespace = "immae";
152 };
153 local-service = {
154 use = "generic-service";
155 host_name = hostFQDN;
156 check_interval = 5;
157 max_check_attempts = "4";
158 retry_interval = "1";
159 servicegroups = "webstatus-resources";
160 };
161 external-service = {
162 use = "generic-service";
163 check_interval = 5;
164 max_check_attempts = "4";
165 retry_interval = "1";
166 };
167 web-service = {
168 use = "generic-service";
169 check_interval = 20;
170 max_check_attempts = "2";
171 retry_interval = "1";
172 };
173 external-web-service = {
174 use = "generic-service";
175 check_interval = 20;
176 max_check_attempts = "2";
177 retry_interval = "1";
178 };
179 mail-service = {
180 use = "generic-service";
181 check_interval = 15;
182 max_check_attempts = "1";
183 retry_interval = "1";
184 };
185 dns-service = {
186 use = "generic-service";
187 check_interval = 120;
188 notification_interval = "120";
189 max_check_attempts = "5";
190 retry_interval = "5";
191 };
192 };
193 # No contact, we go through master
194 contact = {
195 generic-contact = {
196 host_notification_commands = "notify-host-by-email";
197 host_notification_options = "d,u,r,f,s";
198 host_notification_period = "24x7";
199 service_notification_commands = "notify-service-by-email";
200 service_notification_options = "w,u,c,r,f,s";
201 service_notification_period = "24x7";
202 };
203 };
204 host = {
205 generic-host = {
206 event_handler_enabled = "1";
207 flap_detection_enabled = "1";
208 notification_period = "24x7";
209 notifications_enabled = "1";
210 process_perf_data = "1";
211 retain_nonstatus_information = "1";
212 retain_status_information = "1";
213 };
214 linux-server = {
215 check_command = "check_host_alive";
216 check_interval = "5";
217 check_period = "24x7";
218 contact_groups = "admins";
219 max_check_attempts = "10";
220 notification_interval = "120";
221 notification_options = "d,u,r,f";
222 retry_interval = "1";
223 _webstatus_namespace = "immae";
224 };
225 };
226 };
227}
diff --git a/flakes/private/monitoring/plugins/check_backup_age b/flakes/private/monitoring/plugins/check_backup_age
new file mode 100755
index 0000000..d873bdc
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_backup_age
@@ -0,0 +1,66 @@
1#!/usr/bin/env bash
2
3set -euo pipefail
4
5source $SECRETS_PATH
6export HOME=$(mktemp -d)
7
8trap "rm -rf $HOME" EXIT
9folder=$1
10
11parse_date() {
12 d=$1
13 echo $d | sed -e "s/^\(....\)\(..\)\(..\)T\(..\)\(..\)\(..\)/\1-\2-\3T\4:\5:\6/"
14}
15
16output=$(duplicity collection-status --log-fd 2 "$BASE_URL$folder" 2>&1 > /dev/null)
17
18output=$(echo "$output" | grep -v "^\.")
19
20last_full=$(parse_date "$(echo "$output" | grep "^ full " | cut -d' ' -f3 | sort | tail -n1)")
21last_bkp=$(parse_date "$(echo "$output" | grep -E "^ (full|inc) " | cut -d' ' -f3 | sort | tail -n1)")
22orphaned_sets=$(echo "$output" | grep "^orphaned-sets-num" | cut -d' ' -f2)
23incomplete_sets=$(echo "$output" | grep "^incomplete-sets-num" | cut -d' ' -f2)
24
25if [[ -z "$last_full" || -z "$last_bkp" || -z "$orphaned_sets" || -z "$incomplete_sets" ]]; then
26 echo "duply-backup $folder UNKNOWN - impossible to parse result"
27 exit 3
28fi
29
30last_full_age=$(( ($(date "+%s") - $(date -d "$last_full" "+%s")) / (60*60*24) ))
31last_bkp_age=$(( ($(date "+%s") - $(date -d "$last_bkp" "+%s")) / (60*60) ))
32
33PERFS="orphan=$orphaned_sets;1;;0; incomplete=$incomplete_sets;1;;0; age=${last_bkp_age}h;30;48;0; full_age=${last_full_age}d;35;45;0;"
34
35
36WARNINGS=""
37ERRORS=""
38if [[ "$incomplete_sets" -gt 0 ]]; then
39 WARNINGS="$WARNINGS - Incomplete sets is $incomplete_sets"
40fi
41
42if [[ "$orphaned_sets" -gt 0 ]]; then
43 WARNINGS="$WARNINGS - Orphaned sets is $orphaned_sets"
44fi
45
46if [[ "$last_full_age" -gt 45 ]]; then
47 ERRORS="$ERRORS - Last full backup is too old $last_full"
48elif [[ "$last_full_age" -gt 35 ]]; then
49 WARNINGS="$WARNINGS - Last full backup is getting old $last_full"
50fi
51
52if [[ "$last_bkp_age" -gt 48 ]]; then
53 ERRORS="$ERRORS - Last backup is too old $last_bkp"
54elif [[ "$last_bkp_age" -gt 30 ]]; then
55 WARNINGS="$WARNINGS - Last backup is getting old $last_bkp"
56fi
57
58if [[ -n "$ERRORS" ]]; then
59 echo "duply-backup $folder CRITICAL$ERRORS$WARNINGS | $PERFS"
60 exit 2
61elif [[ -n "$WARNINGS" ]]; then
62 echo "duply-backup $folder WARNING$WARNINGS | $PERFS"
63 exit 1
64else
65 echo "duply-backup $folder OK | $PERFS"
66fi
diff --git a/flakes/private/monitoring/plugins/check_bandwidth b/flakes/private/monitoring/plugins/check_bandwidth
new file mode 100755
index 0000000..21d01f5
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_bandwidth
@@ -0,0 +1,122 @@
1#!/bin/bash
2
3# ============================== SUMMARY =====================================
4#Author : Ken Roulamellah
5#Date : 19/07/2018
6#Version : 1.0
7# Licence : GPL
8# ===================== INFORMATION ABOUT THIS PLUGIN ========================
9#
10# This plugin checks the average RX and TX bandwidth utilisation. It use
11# kbytes as measure unite.
12#
13# ========================== START OF PROGRAM CODE ===========================
14
15STATE_OK=0
16STATE_WARNING=1
17STATE_CRITICAL=2
18STATE_UNKNOWN=3
19
20interface=$( ip route | grep default | awk '{print $5}' | head -n1)
21function print_usage()
22{
23 echo "Usage :"
24 echo "$0 [ -i=INTERFACE] [ -ct=COUNT ] -w WARNING -c CRITICAL"
25 echo "This script calculate the average bandwith usage."
26 echo "Default values | interface: ${interface}, counter: 10"
27}
28
29counter=10
30warning=-1
31critical=-1
32
33sum_rx=0
34sum_tx=0
35avg_rx=
36avg_tx=
37i=
38
39
40if [[ $# -lt 4 ]];
41then
42 echo "Error: Arguments are missing"
43 print_usage
44 exit $STATE_UNKNOWN
45fi
46
47while [[ $# -gt 0 ]]; do
48 case "$1" in
49 -i=*)
50 interface="$(cut -d'=' -f2 <<<"$1")"
51 shift
52 ;;
53 -ct=*)
54 counter="$(cut -d'=' -f2 <<<"$1")"
55 shift
56 ;;
57 -w)
58 warning=$2
59 shift 2
60 ;;
61 -c)
62 critical=$2
63 shift 2
64 ;;
65 *)
66 printf "\nError: Invalid option '$1'"
67 print_usage
68 exit $STATE_UNKNOWN
69 ;;
70 esac
71done
72
73if [ $warning -lt 0 ] || [ $critical -lt 0 ];
74then
75 echo "Error: You need to specify a warning and critical treshold"
76 print_usage
77 exit $STATE_UNKNOWN
78fi
79
80grep -q "up" /sys/class/net/$interface/operstate || exec echo "$interface: no such device or down"
81
82read rx <"/sys/class/net/$interface/statistics/rx_bytes"
83read tx <"/sys/class/net/$interface/statistics/tx_bytes"
84
85i=$counter
86while [ $i -gt 0 ]; do
87 sleep 1
88 read newrx <"/sys/class/net/$interface/statistics/rx_bytes"
89 read newtx <"/sys/class/net/$interface/statistics/tx_bytes"
90
91 #echo "old rx :$rx"
92 #echo "new rx :$newrx"
93 rx_cal=$(bc <<< "scale=2; ($newrx-$rx) / 1000")
94 tx_cal=$(bc <<< "scale=2; ($newtx-$tx) / 1000")
95
96 sum_rx=$(bc <<< "scale=2;$sum_rx+$rx_cal")
97 sum_tx=$(bc <<< "scale=2;$sum_tx+$tx_cal")
98
99 #echo "$interface {rx: $rx_cal ko/s, tx: $tx_cal ko/s}"
100 rx=$newrx
101 tx=$newtx
102 ((i --))
103done
104
105avg_rx=$(bc <<< "scale=2;$sum_rx/$counter")
106avg_tx=$(bc <<< "scale=2;$sum_tx/$counter")
107
108#echo "$avg_rx"
109#echo "$avg_tx"
110
111
112if [ $(bc <<< "$avg_rx > $critical || $avg_tx > $critical") -eq 1 ]; then
113 echo "$interface CRITICAL - AVG_RX: $avg_rx kb/s, AVG_TX: $avg_tx kb/s | RX="$avg_rx"kbps;0;0;0; TX="$avg_tx"kbps;0;0;0;"
114 exit $STATE_CRITICAL
115elif [ $(bc <<< "$avg_rx > $warning || $avg_tx > $warning") -eq 1 ]; then
116 echo "$interface WARNING - AVG_RX: $avg_rx kb/s, AVG_TX: $avg_tx kb/s | RX="$avg_rx"kbps;0;0;0; TX="$avg_tx"kbps;0;0;0;"
117 exit $STATE_WARNING
118else
119 echo "$interface - OK AVG_RX: $avg_rx kb/s, AVG_TX: $avg_tx kb/s | RX="$avg_rx"kbps;0;0;0; TX="$avg_tx"kbps;0;0;0;"
120 exit $STATE_OK
121fi
122exit 3
diff --git a/flakes/private/monitoring/plugins/check_command b/flakes/private/monitoring/plugins/check_command
new file mode 100755
index 0000000..2b546c1
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_command
@@ -0,0 +1,113 @@
1#!/usr/bin/env perl
2
3use strict;
4use Getopt::Std;
5$| = 1;
6
7my %opts;
8getopts('hr:C:c:s:o:', \%opts);
9
10my $STATE_OK = 0;
11my $STATE_WARNING = 1;
12my $STATE_CRITICAL = 2;
13my $STATE_UNKNOWN = 3;
14
15if ($opts{'h'} || scalar(%opts) == 0) {
16 &print_help();
17 exit($STATE_OK);
18}
19
20my $command = $opts{'c'};
21if ($command eq '') {
22 print "You must provide a command to check.\n";
23 exit($STATE_UNKNOWN);
24}
25
26my $expected_output = $opts{'o'};
27my $expected_status = $opts{'s'};
28my $other_command = $opts{'C'};
29
30if ($other_command eq '' and $expected_status eq '' and $expected_output eq '') {
31 $expected_status = 0;
32}
33
34my $cmd = $command . ' 2>&1';
35my $other_cmd;
36if ($other_command ne '') {
37 $other_cmd = $other_command . ' 2>&1';
38}
39
40my $run_as;
41if ($opts{'r'}) {
42 $run_as = $opts{'r'};
43 $cmd = "sudo -u $run_as -n $cmd";
44
45 if ($other_command ne '') {
46 $other_cmd = "sudo -u $run_as -n $other_cmd";
47 }
48
49}
50
51my $cmd_result = `$cmd`;
52my $other_cmd_result;
53if ($other_command ne '') {
54 $other_cmd_result = `$other_cmd`;
55 chomp($other_cmd_result);
56}
57
58chomp($cmd_result);
59if ($cmd_result =~ /sudo/i) {
60 print "$command CRITICAL - No sudo right to run the command | result=1;;;;\n";
61 exit($STATE_UNKNOWN);
62} elsif ($expected_status ne '') {
63 if ($? != $expected_status) {
64 print "$command CRITICAL - Response status $? | result=1;;;;\n";
65 exit($STATE_CRITICAL);
66 } else {
67 print "$command OK - Response status $? | result=0;;;;\n";
68 exit($STATE_OK);
69 }
70} elsif ($other_command ne '') {
71 if ($cmd_result ne $other_cmd_result) {
72 print "$command CRITICAL - Expected output not matching other command output | result=1;;;;\n";
73 exit($STATE_CRITICAL);
74 } else {
75 print "$command OK - Expected output matching other command output | result=0;;;;\n";
76 exit($STATE_OK);
77 }
78} else {
79 if ($cmd_result !~ /$expected_output/) {
80 print "$command CRITICAL - Expected output not matching | result=1;;;;\n";
81 exit($STATE_CRITICAL);
82 } else {
83 print "$command OK - Expected output matching | result=0;;;;\n";
84 exit($STATE_OK);
85 }
86}
87
88sub print_help() {
89 print << "EOF";
90Check whether the given command responds as expected. One of -o -C or -s must be selected.
91
92Options:
93-h
94 Print detailed help screen
95
96-c
97 command to run (required)
98
99-C
100 other command to compare output
101
102-r user
103 Run as user via sudo.
104
105-s
106 status code to check
107
108-o
109 output to check
110
111EOF
112}
113
diff --git a/flakes/private/monitoring/plugins/check_emails b/flakes/private/monitoring/plugins/check_emails
new file mode 100755
index 0000000..534e5a5
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_emails
@@ -0,0 +1,121 @@
1#!/usr/bin/env perl
2
3use strict;
4use Getopt::Std;
5use File::Basename;
6use Date::Parse;
7use POSIX qw(strftime);
8
9$| = 1;
10
11my %opts;
12getopts('hH:l:s:p:f:i:n:r:', \%opts);
13
14my $STATE_OK = 0;
15my $STATE_WARNING = 1;
16my $STATE_CRITICAL = 2;
17my $STATE_UNKNOWN = 3;
18
19if ($opts{'h'} || scalar(%opts) == 0) {
20 &print_help();
21 exit($STATE_OK);
22}
23
24my $port = $opts{'p'};
25my $host = $opts{'H'};
26my $login = $opts{'l'};
27if ($login ne '') {
28 $login = "$login@";
29}
30
31my $identity = $opts{'i'};
32my $local_directory = $opts{'n'};
33my $return_path = $opts{'r'};
34
35my @emails_to_send = split(/,/, $opts{'s'});
36my @emails_to_expect = split(/,/, $opts{'f'});
37
38my $cmd_result;
39if ($local_directory ne '') {
40 if (@emails_to_expect and ! -d $local_directory) {
41 print "Emails $host UNKNOWN - Could not find local directory";
42 exit($STATE_UNKNOWN);
43 }
44 $cmd_result = `send_mails $local_directory $return_path @emails_to_send 2>&1`;
45} else {
46 $cmd_result = `ssh -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -p $port -i $identity $login$host send_mails @emails_to_send 2>&1`;
47
48 if ($cmd_result =~ /Host key verification failed./) {
49 print "Emails $host UNKNOWN - Could not connect to host with ssh key\n";
50 exit($STATE_UNKNOWN);
51 }
52}
53
54my @lines = split(/\n/, $cmd_result);
55
56my %found_emails;
57
58foreach my $line (@lines) {
59 my @split_line = split(/;/, $line, 2);
60 $found_emails{$split_line[0]} = $split_line[1];
61}
62
63my $output = "";
64my $old = 0;
65foreach my $email_from (@emails_to_expect) {
66 my @email_split = split(/:/, $email_from);
67 my $email = $email_split[0];
68 my $from = $email_split[1];
69
70 if ( exists $found_emails{$email} ) {
71 my $email_date = str2time($found_emails{$email});
72 my $current_date = strftime "%s", localtime;
73
74 if ($current_date - $email_date > 60*30) {
75 $output = "$output$email ($found_emails{$email} from $from) ";
76 }
77 $old = ($current_date - $email_date) > $old ? ($current_date - $email_date) : $old;
78 } else {
79 $output = "$output$email (missing) "
80 }
81}
82
83if ($output ne '') {
84 print "Emails $host CRITICAL - expecting emails: $output | timestamp=${old}s;;;;\n";
85 exit($STATE_CRITICAL);
86} else {
87 print "Emails $host OK | timestamp=${old}s;;;;\n";
88 exit($STATE_OK);
89}
90
91sub print_help() {
92 print << "EOF";
93Check sent emails
94
95Options:
96-h
97 Print detailed help screen
98
99-H
100 Host to check
101
102-l
103 Login
104
105-i
106 Identity file
107
108-n
109 Don’t use ssh, pass that directory to script
110
111-r
112 Return path for local e-mails
113
114-s
115 Comma separated list of emails to send from the host.
116
117-f
118 Comma separated list of emails to expect on the host.
119EOF
120}
121
diff --git a/flakes/private/monitoring/plugins/check_eriomem b/flakes/private/monitoring/plugins/check_eriomem
new file mode 100755
index 0000000..880b88a
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_eriomem
@@ -0,0 +1,83 @@
1#!/usr/bin/env python
2import os
3import sys
4import getopt
5import signal
6from subprocess import Popen, PIPE
7
8STATE_OK = 0
9STATE_WARNING = 1
10STATE_CRITICAL = 2
11STATE_UNKNOWN = 3
12
13keys = sys.argv[1].split(",")
14
15def to_args(k):
16 access, secret = k.split(":", 1)
17 return [
18 "s3cmd",
19 '-c=/dev/null',
20 '--no-check-certificate',
21 '--access_key={}'.format(access),
22 '--secret_key={}'.format(secret),
23 '--host=e.eriomem.net',
24 '--host-bucket=%(bucket)s.e.eriomem.net',
25 'du'
26 ]
27
28max_size = 1024*1024*1024*1024
29warning_percent = 99.75
30critical_percent = 99.95
31
32def output(code, msg):
33 print(msg)
34 sys.exit(code)
35
36def main():
37 def handler(signum, frame):
38 raise IOError
39 signal.signal(signal.SIGALRM, handler)
40 signal.alarm(60)
41
42 try:
43 ps = [Popen(to_args(a), stdout=PIPE, stderr=PIPE) for a in keys]
44 outs = [p.communicate() for p in ps]
45 rets = [p.wait() for p in ps]
46 except IOError:
47 for p in ps:
48 os.kill(p.pid, signal.SIGTERM)
49 output(STATE_UNKNOWN,
50 "Eriomem UNKNOWN - Command timeout after 60 seconds!")
51
52 signal.alarm(0)
53
54 if sum(rets) == 0:
55 usages = [int(out[0].decode().split("\n")[-2].split()[0]) for out in outs]
56 usage = sum(usages)
57 use_percent = 100 * usage / max_size
58 if use_percent > critical_percent:
59 output(STATE_CRITICAL,
60 "Eriomem CRITICAL - bucket usage: %s (%s%%);| size=%s;;;;" %
61 (sizeof_fmt(usage), use_percent, sizeof_fmt(usage)))
62 elif use_percent > warning_percent:
63 output(STATE_WARNING,
64 "Eriomem WARNING - bucket usage: %s (%s%%);| size=%s;;;;" %
65 (sizeof_fmt(usage), use_percent, sizeof_fmt(usage)))
66 else:
67 output(STATE_OK,
68 "Eriomem OK - bucket usage: %s (%d%%);| size=%s;;;;" %
69 (sizeof_fmt(usage), use_percent, sizeof_fmt(usage)))
70 else:
71 messages = "\n".join([out[0].decode() + out[1].decode() for out in outs])
72 output(STATE_UNKNOWN,
73 "Eriomem UNKNOWN - Error in command")
74
75def sizeof_fmt(num):
76 for unit in ['','ko','Mo','Go','To','Po','Eo','Zo']:
77 if abs(num) < 1024.0:
78 return "%3.1f%s" % (num, unit)
79 num /= 1024.0
80 return "%.1f%s%s" % (num, 'Yo')
81
82if __name__ == '__main__':
83 main()
diff --git a/flakes/private/monitoring/plugins/check_ftp_database b/flakes/private/monitoring/plugins/check_ftp_database
new file mode 100755
index 0000000..f9cf579
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_ftp_database
@@ -0,0 +1,11 @@
1#!/usr/bin/env bash
2
3OUT=$(echo "ls" | lftp -u test_ftp,test_ftp eldiron.immae.eu | grep it_works | wc -l)
4
5if [ "$OUT" -eq 1 ]; then
6 echo "ftp connection OK - access to ftp is working | ftp=1;;;;"
7 exit 0
8else
9 echo "ftp connection CRITICAL - no access to ftp | ftp=0;;;;"
10 exit 2
11fi
diff --git a/flakes/private/monitoring/plugins/check_git b/flakes/private/monitoring/plugins/check_git
new file mode 100755
index 0000000..e8fbb29
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_git
@@ -0,0 +1,81 @@
1#!/usr/bin/env bash
2
3SSH_KEY="$1"
4
5TMPDIR=$(mktemp -d)
6
7if [ ! -d "$TMPDIR" ]; then
8 echo "gitolite UNKNOWN - impossible to create temp dir"
9 exit 3
10fi
11
12trap "rm -rf $TMPDIR" EXIT
13
14ERRORS=""
15OUTPUT=""
16PERFS=""
17
18cd "$TMPDIR"
19OUT=$(git clone -q git://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1)
20ERR=$?
21if [ -n "$OUT" ]; then
22OUTPUT="$OUTPUT
23$OUT"
24fi
25if [ "$ERR" != 0 ]; then
26 PERFS="$PERFS git=0;;;;"
27 ERRORS="$ERRORS git://"
28else
29 PERFS="$PERFS git=1;;;;"
30fi
31rm -rf Monitor
32
33OUT=$(git clone -q http://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1)
34ERR=$?
35if [ -n "$OUT" ]; then
36OUTPUT="$OUTPUT
37$OUT"
38fi
39if [ "$ERR" != 0 ]; then
40 ERRORS="$ERRORS http://"
41 PERFS="$PERFS http=0;;;;"
42else
43 PERFS="$PERFS http=1;;;;"
44fi
45rm -rf Monitor
46
47OUT=$(git clone -q https://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1)
48ERR=$?
49if [ -n "$OUT" ]; then
50OUTPUT="$OUTPUT
51$OUT"
52fi
53if [ "$ERR" != 0 ]; then
54 ERRORS="$ERRORS https://"
55 PERFS="$PERFS https=0;;;;"
56else
57 PERFS="$PERFS https=1;;;;"
58fi
59rm -rf Monitor
60
61OUT=$(GIT_SSH_COMMAND="ssh -i $SSH_KEY -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no" git clone -q gitolite@git.immae.eu:perso/Immae/Projets/Ruby/Monitor 2>&1)
62ERR=$?
63if [ -n "$OUT" ]; then
64OUTPUT="$OUTPUT
65$OUT"
66fi
67if [ "$ERR" != 0 ]; then
68 ERRORS="$ERRORS ssh"
69 PERFS="$PERFS ssh=0;;;;"
70else
71 PERFS="$PERFS ssh=1;;;;"
72fi
73rm -rf Monitor
74
75if [ -n "$ERRORS" ]; then
76 echo "gitolite CRITICAL - impossible to clone via$ERRORS | $PERFS"
77 exit 2
78else
79 echo "gitolite OK - ssh, git, http and https work | $PERFS"
80 exit 0
81fi
diff --git a/flakes/private/monitoring/plugins/check_imap_connection b/flakes/private/monitoring/plugins/check_imap_connection
new file mode 100755
index 0000000..c1ab0dd
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_imap_connection
@@ -0,0 +1,52 @@
1#!/usr/bin/env perl
2
3use strict;
4use Getopt::Std;
5$| = 1;
6
7my %opts;
8getopts('h:u:p:H:', \%opts);
9
10my $STATE_OK = 0;
11my $STATE_WARNING = 1;
12my $STATE_CRITICAL = 2;
13my $STATE_UNKNOWN = 3;
14
15if ($opts{'h'} || !$opts{'u'} || !$opts{'p'} || !$opts{'H'}) {
16 &print_help();
17 exit($STATE_UNKNOWN);
18}
19
20my $user = $opts{'u'};
21my $password = $opts{'p'};
22my $host = $opts{'H'};
23
24my $cmd_result = `(echo "a login $user $password"; echo "b logout") | openssl s_client -quiet -ign_eof -connect $host -starttls imap 2>&1`;
25my $expected_result = "a OK Logged in";
26
27chomp($cmd_result);
28if ($cmd_result !~ /$expected_result/) {
29 print "IMAP CRITICAL - Unable to connect via imaps | imap=0;;;;\n";
30 exit($STATE_CRITICAL);
31} else {
32 print "IMAP OK - imaps connected successfully | imap=1;;;;\n";
33 exit($STATE_OK);
34}
35
36sub print_help() {
37 print << "EOF";
38Check whether imap works via ssl and is able to connect its database.
39
40Options:
41-h
42 Print detailed help screen
43-u
44 User to log in as
45-p
46 Password to log in
47-H
48 Host to log in to
49
50EOF
51}
52
diff --git a/flakes/private/monitoring/plugins/check_last_file_date b/flakes/private/monitoring/plugins/check_last_file_date
new file mode 100755
index 0000000..f51a258
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_last_file_date
@@ -0,0 +1,28 @@
1#!/bin/bash
2
3STATE_OK=0
4STATE_WARNING=1
5STATE_CRITICAL=2
6STATE_UNKNOWN=3
7
8base_path=$1
9hours=$2
10
11last_date=$(find $base_path -mindepth 1 -maxdepth 1 -printf "%T@\n" 2>/dev/null | sort | tail -n 1)
12
13if [ -z "$last_date" ]; then
14 echo "UNKNOWN: Could not read folder"
15 exit $STATE_UNKNOWN
16else
17 LC_ALL=C last_date=$(printf "%.*f" 0 $last_date)
18 LC_ALL=C age=$(( $(date "+%s") - $last_date))
19 max_age=$(( $hours * 60 * 60 ))
20 min_date=$(date -d "$hours hours ago" "+%s")
21 if [ "$min_date" -lt "$last_date" ]; then
22 echo "OK: Last file $(date -d @$last_date) | age=${age}s;;$max_age;;"
23 exit $STATE_OK
24 else
25 echo "CRITICAL: Last file $(date -d @$last_date) | age=${age}s;;$max_age;;"
26 exit $STATE_CRITICAL
27 fi
28fi
diff --git a/flakes/private/monitoring/plugins/check_mem.sh b/flakes/private/monitoring/plugins/check_mem.sh
new file mode 100755
index 0000000..3a29040
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_mem.sh
@@ -0,0 +1,31 @@
1#!/usr/bin/env bash
2
3if [ "$1" = "-w" ] && [ "$2" -gt "0" ] && [ "$3" = "-c" ] && [ "$4" -gt "0" ]; then
4 FreeM=`free -m -w`
5 memTotal_m=`echo "$FreeM" |grep Mem |awk '{print $2}'`
6 memUsed_m=`echo "$FreeM" |grep Mem |awk '{print $3}'`
7 memFree_m=`echo "$FreeM" |grep Mem |awk '{print $4}'`
8 memShared_m=`echo "$FreeM" |grep Mem |awk '{print $5}'`
9 memBuffer_m=`echo "$FreeM" |grep Mem |awk '{print $6}'`
10 memCache_m=`echo "$FreeM" |grep Mem |awk '{print $7}'`
11 memAvailable_m=`echo "$FreeM" |grep Mem |awk '{print $8}'`
12 memUsedPrc=`echo $((($memUsed_m-$memBuffer_m-$memCache_m)*100/$memTotal_m))||cut -d. -f1`
13 if [ "$memUsedPrc" -ge "$4" ]; then
14 echo "Memory: CRITICAL Total: $memTotal_m MB - Used/Buffer/Cache: $memUsed_m MB - $memUsedPrc% used!|TOTAL=$memTotal_m;;;; USED=$memUsed_m;;;; SHARED=$memShared_m;;;; CACHE=$memCache_m;;;; AVAILABLE=$memAvailable_m;;;; BUFFER=$memBuffer_m;;;;"
15 exit 2
16 elif [ "$memUsedPrc" -ge "$2" ]; then
17 echo "Memory: WARNING Total: $memTotal_m MB - Used/Buffer/Cache: $memUsed_m MB - $memUsedPrc% used!|TOTAL=$memTotal_m;;;; USED=$memUsed_m;;;; SHARED=$memShared_m;;;; CACHE=$memCache_m;;;; AVAILABLE=$memAvailable_m;;;; BUFFER=$memBuffer_m;;;;"
18 exit 1
19 else
20 echo "Memory: OK Total: $memTotal_m MB - Used/Buffer/Cache: $memUsed_m MB - $memUsedPrc% used|TOTAL=$memTotal_m;;;; USED=$memUsed_m;;;; SHARED=$memShared_m;;;; CACHE=$memCache_m;;;; AVAILABLE=$memAvailable_m;;;; BUFFER=$memBuffer_m;;;;"
21 exit 0
22 fi
23else # If inputs are not as expected, print help.
24 sName="`echo $0|awk -F '/' '{print $NF}'`"
25 echo -e "\n\n\t\t### $sName Version 2.0###\n"
26 echo -e "# Usage:\t$sName -w <warnlevel> -c <critlevel>"
27 echo -e "\t\t= warnlevel and critlevel is percentage value without %\n"
28 echo "# EXAMPLE:\t/usr/lib64/nagios/plugins/$sName -w 80 -c 90"
29 echo -e "\nCopyright (C) 2012 Lukasz Gogolin (lukasz.gogolin@gmail.com), improved by Nestor 2015\n\n"
30 exit
31fi
diff --git a/flakes/private/monitoring/plugins/check_mysql_replication b/flakes/private/monitoring/plugins/check_mysql_replication
new file mode 100755
index 0000000..1ee5de1
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_mysql_replication
@@ -0,0 +1,41 @@
1#!/bin/bash
2
3STATE_OK=0
4STATE_WARNING=1
5STATE_CRITICAL=2
6STATE_UNKNOWN=3
7
8socket=$1
9config_file=$2
10info=$(mysql --defaults-file=${config_file} -S $socket -e "show slave status" --vertical)
11exit_code=$?
12
13lag=$(echo "$info" | grep "\bSeconds_Behind_Master\b" | cut -d':' -f2 | sed -e "s/\s//g")
14
15IO_running=$(echo "$info" | grep "\bSlave_IO_Running\b" | cut -d':' -f2 | sed -e "s/\s//g")
16SQL_running=$(echo "$info" | grep "\bSlave_SQL_Running\b" | cut -d':' -f2 | sed -e "s/\s//g")
17
18if [[ $exit_code -ne 0 ]]; then
19 echo "UNKNOWN - Impossible to run mysql command"
20 exit $STATE_UNKNOWN
21elif [[ -z "$lag" ]]; then
22 echo "UNKNOWN - No replication found for mysql"
23 exit $STATE_UNKNOWN
24elif [[ "$IO_running" != "Yes" || "$SQL_running" != "Yes" ]]; then
25 echo "UNKNOWN - Replication is not running"
26 exit $STATE_UNKNOWN
27else
28 output="Replication lag for mysql is ${lag}s"
29 LC_ALL=C lag=$(printf "%.*f" 0 $lag)
30
31 if [[ $lag -lt 5 ]]; then
32 echo "OK - $output | time=${lag}s;5;10;;"
33 exit $STATE_OK
34 elif [[ $lag -lt 10 ]]; then
35 echo "WARNING - $output | time=${lag}s;5;10;;"
36 exit $STATE_WARNING
37 else
38 echo "CRITICAL - $output | time=${lag}s;5;10;;"
39 exit $STATE_CRITICAL
40 fi
41fi
diff --git a/flakes/private/monitoring/plugins/check_openldap_replication b/flakes/private/monitoring/plugins/check_openldap_replication
new file mode 100755
index 0000000..7136ad5
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_openldap_replication
@@ -0,0 +1,54 @@
1#!/bin/bash
2
3STATE_OK=0
4STATE_WARNING=1
5STATE_CRITICAL=2
6STATE_UNKNOWN=3
7
8distant_host="$1"
9replication_dn="$2"
10replication_pw="$3"
11base="$4"
12config="$5"
13
14to_date() {
15 i="$1"
16 i=$(echo "$i" | grep contextCSN | cut -d":" -f2 | sed -e "s/\s//g")
17 i=$(echo "$i" | cut -d"#" -f1)
18 i=$(echo "$i" | cut -d"." -f1)
19 echo "$i"
20}
21
22# ldap
23remote_ldap=$(ldapsearch -H $distant_host -D "$replication_dn" -y "$replication_pw" -b "$base" -s base -LLL contextCSN )
24exit_code_remote=$?
25remote_ldap=$(to_date "$remote_ldap")
26
27# slapcat
28local_ldap=$(slapcat -b "$base" -f "$config" -a "(entryDN=$base)")
29exit_code_local=$?
30local_ldap=$(to_date "$local_ldap")
31
32offset=$(($remote_ldap - $local_ldap))
33
34if [[ $exit_code_remote -ne 0 || $exit_code_local -ne 0 ]]; then
35 echo "UNKNOWN - Impossible to run ldap command"
36 exit $STATE_UNKNOWN
37elif [[ -z "$offset" ]]; then
38 echo "UNKNOWN - No replication found"
39 exit $STATE_UNKNOWN
40else
41 output="Replication lag for openldap is ${offset}s"
42 LC_ALL=C lag=$(printf "%.*f" 0 $lag)
43
44 if [[ $offset -lt 5 ]]; then
45 echo "OK - $output | time=${offset}s;5;10;;"
46 exit $STATE_OK
47 elif [[ $offset -lt 10 ]]; then
48 echo "WARNING - $output | time=${offset}s;5;10;;"
49 exit $STATE_WARNING
50 else
51 echo "CRITICAL - $output | time=${offset}s;5;10;;"
52 exit $STATE_CRITICAL
53 fi
54fi
diff --git a/flakes/private/monitoring/plugins/check_ovh_sms b/flakes/private/monitoring/plugins/check_ovh_sms
new file mode 100755
index 0000000..caf279c
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_ovh_sms
@@ -0,0 +1,25 @@
1#!/usr/bin/env python
2
3import sys
4try:
5 import ovh
6
7 [endpoint, application_key, application_secret, consumer_key, account] = sys.argv[1].split(",")
8 client = ovh.Client(
9 endpoint=endpoint,
10 application_key=application_key,
11 application_secret=application_secret,
12 consumer_key=consumer_key,
13 )
14
15 result = client.get('/sms/{}'.format(account))["creditsLeft"]
16
17 if result < 20:
18 print("SMS OVH Critical - Not enough sms left ({})|SMS={};;;;".format(result, result))
19 sys.exit(2)
20 else:
21 print("SMS OVH Ok - Enough sms left ({})|SMS={};;;;".format(result, result))
22 sys.exit(0)
23except Exception:
24 print("SMS OVH UNKNOWN - Error during script")
25 sys.exit(3)
diff --git a/flakes/private/monitoring/plugins/check_postgres_database_count b/flakes/private/monitoring/plugins/check_postgres_database_count
new file mode 100755
index 0000000..43bdd8c
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_postgres_database_count
@@ -0,0 +1,32 @@
1#!/bin/bash
2
3STATE_OK=0
4STATE_WARNING=1
5STATE_CRITICAL=2
6STATE_UNKNOWN=3
7
8host=$1
9port=$2
10min=$3
11
12count=$(psql -h $host -p $port -A -q -c '\t' -c 'select count(datname) from pg_catalog.pg_database' postgres 2>&1)
13exit_code=$?
14
15if [[ $exit_code -ne 0 ]]; then
16 echo "UNKNOWN - Impossible to run psql command: $count"
17 exit $STATE_UNKNOWN
18elif [[ -z "$count" ]]; then
19 echo "UNKNOWN - No database found"
20 exit $STATE_UNKNOWN
21else
22 output="Database count is $count"
23 LC_ALL=C count=$(printf "%.*f" 0 $count)
24
25 if [[ $count -gt $min ]]; then
26 echo "OK - $output | count=${count};$min;$min;0;"
27 exit $STATE_OK
28 else
29 echo "CRITICAL - $output | count=${count};$min;$min;0;"
30 exit $STATE_CRITICAL
31 fi
32fi
diff --git a/flakes/private/monitoring/plugins/check_postgres_replication b/flakes/private/monitoring/plugins/check_postgres_replication
new file mode 100755
index 0000000..ff257a3
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_postgres_replication
@@ -0,0 +1,35 @@
1#!/bin/bash
2
3STATE_OK=0
4STATE_WARNING=1
5STATE_CRITICAL=2
6STATE_UNKNOWN=3
7
8user=$1
9host=$2
10port=$3
11
12lag=$(psql -h $host -p $port -A -t -c "SELECT COALESCE(EXTRACT(EPOCH FROM replay_lag),0) FROM pg_stat_replication WHERE usename='$user'" 2>/dev/null)
13exit_code=$?
14
15if [[ $exit_code -ne 0 ]]; then
16 echo "UNKNOWN - Impossible to run psql command"
17 exit $STATE_UNKNOWN
18elif [[ -z "$lag" ]]; then
19 echo "UNKNOWN - No replication found for $user"
20 exit $STATE_UNKNOWN
21else
22 output="Replication lag for $user is ${lag}s"
23 LC_ALL=C lag=$(printf "%.*f" 0 $lag)
24
25 if [[ $lag -lt 5 ]]; then
26 echo "OK - $output | time=${lag}s;5;10;0;"
27 exit $STATE_OK
28 elif [[ $lag -lt 10 ]]; then
29 echo "WARNING - $output | time=${lag}s;5;10;0;"
30 exit $STATE_WARNING
31 else
32 echo "CRITICAL - $output | time=${lag}s;5;10;0;"
33 exit $STATE_CRITICAL
34 fi
35fi
diff --git a/flakes/private/monitoring/plugins/check_redis_replication b/flakes/private/monitoring/plugins/check_redis_replication
new file mode 100755
index 0000000..6dbe4c4
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_redis_replication
@@ -0,0 +1,38 @@
1#!/bin/bash
2
3STATE_OK=0
4STATE_WARNING=1
5STATE_CRITICAL=2
6STATE_UNKNOWN=3
7
8socket=$1
9
10info=$(redis-cli -s $socket info replication)
11lag=$(echo "$info" | grep master_last_io_seconds_ago | cut -d":" -f2 | sed -e "s/\s//g")
12slave_offset=$(echo "$info" | grep slave_repl_offset | cut -d":" -f2 | sed -e "s/\s//g")
13master_offset=$(echo "$info" | grep master_repl_offset | cut -d":" -f2 | sed -e "s/\s//g")
14offset=$(($master_offset - $slave_offset))
15
16exit_code=$?
17
18if [[ $exit_code -ne 0 ]]; then
19 echo "UNKNOWN - Impossible to run redis command"
20 exit $STATE_UNKNOWN
21elif [[ -z "$lag" ]]; then
22 echo "UNKNOWN - No replication found"
23 exit $STATE_UNKNOWN
24else
25 output="Replication lag for redis is ${lag}s and offset is ${offset}"
26 LC_ALL=C lag=$(printf "%.*f" 0 $lag)
27
28 if [[ $lag -lt 5 && $offset -lt 5 ]]; then
29 echo "OK - $output | time=${lag}s;5;10;0; offset=${offset};5;10;0;"
30 exit $STATE_OK
31 elif [[ $lag -lt 10 && $offset -lt 10 ]]; then
32 echo "WARNING - $output | time=${lag}s;5;10;0; offset=${offset};5;10;0;"
33 exit $STATE_WARNING
34 else
35 echo "CRITICAL - $output | time=${lag}s;5;10;0; offset=${offset};5;10;0;"
36 exit $STATE_CRITICAL
37 fi
38fi
diff --git a/flakes/private/monitoring/plugins/check_zfs_snapshot b/flakes/private/monitoring/plugins/check_zfs_snapshot
new file mode 100755
index 0000000..56f8c4f
--- /dev/null
+++ b/flakes/private/monitoring/plugins/check_zfs_snapshot
@@ -0,0 +1,325 @@
1#! /bin/sh
2
3OS=$(uname)
4
5# MIT License
6#
7# Copyright (c) 2016 Josef Friedrich <josef@friedrich.rocks>
8#
9# Permission is hereby granted, free of charge, to any person obtaining
10# a copy of this software and associated documentation files (the
11# "Software"), to deal in the Software without restriction, including
12# without limitation the rights to use, copy, modify, merge, publish,
13# distribute, sublicense, and/or sell copies of the Software, and to
14# permit persons to whom the Software is furnished to do so, subject to
15# the following conditions:
16#
17# The above copyright notice and this permission notice shall be
18# included in all copies or substantial portions of the Software.
19#
20# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
23# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
24# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
28########################################################################
29# Date functions
30########################################################################
31
32# This date function must be placed on the top of this file because
33# they are used in some global variables.
34
35# to_year ###
36
37##
38# Get the four digit year integer from now.
39#
40# Return:
41# The current 4 digit year.
42##
43_now_to_year() {
44 date +%Y
45}
46
47##
48# Convert a date in the format YYYY-MM-DD to a four digit year integer.
49#
50# Parameters:
51# a date in the format YYYY-MM-DD
52#
53# Return:
54# four digit year integer
55##
56_date_to_year() {
57 local OPTIONS
58 if [ "$OS" = 'Linux' ]; then
59 OPTIONS="--date $1"
60 # FreeBSD, Darwin
61 else
62 OPTIONS="-j -f %Y-%m-%d $1"
63 fi
64 date $OPTIONS +%Y
65}
66
67# to_datetime ###
68
69##
70# Convert a UNIX timestamp to a datetime string.
71#
72# Parameters:
73# UNIX timestamp
74#
75# Return:
76# %Y-%m-%d.%H:%M:%S
77##
78_timestamp_to_datetime() {
79 local OPTIONS
80 if [ "$OS" = 'Linux' ]; then
81 OPTIONS="--date @$1"
82 # FreeBSD, Darwin
83 else
84 OPTIONS="-j -f %s $1"
85 fi
86 date $OPTIONS +%Y-%m-%d.%H:%M:%S
87}
88
89# to_timestamp ###
90
91##
92# Get the current UNIX timestamp.
93#
94# Return:
95# %current UNIX timestamp
96##
97_now_to_timestamp() {
98 date +%s
99}
100
101PROJECT_PAGES='https://github.com/Josef-Friedrich/check_zfs_snapshot
102https://exchange.icinga.com/joseffriedrich/check_zfs_snapshot
103https://exchange.nagios.org/directory/Plugins/System-Metrics/File-System/check_zfs_snapshot/details'
104
105VERSION=1.2
106FIRST_RELEASE=2016-09-08
107SHORT_DESCRIPTION="Monitoring plugin to check how long ago the last \
108snapshot of a ZFS dataset was created."
109USAGE="check_zfs_snapshot v$VERSION
110Copyright (c) $(_date_to_year $FIRST_RELEASE)-$(_now_to_year) \
111Josef Friedrich <josef@friedrich.rocks>
112
113$SHORT_DESCRIPTION
114
115
116Usage: check_zfs_snapshot <options>
117
118Options:
119 -c, --critical=OPT_CRITICAL
120 Interval in seconds for critical state.
121 -d, --dataset=OPT_DATASET
122 The ZFS dataset to check.
123 -h, --help
124 Show this help.
125 -s, --short-description
126 Show a short description of the command.
127 -v, --version
128 Show the version number.
129 -w, --warning=OPT_WARNING
130 Interval in seconds for warning state. Must be lower than -c
131
132Performance data:
133 - last_ago
134 Time interval in seconds for last snapshot.
135 - warning
136 Interval in seconds.
137 - critical
138 Interval in seconds.
139 - snapshot_count
140 How many snapshot exists in the given dataset and all child
141 datasets exists.
142"
143
144# Exit codes
145STATE_OK=0
146STATE_WARNING=1
147STATE_CRITICAL=2
148STATE_UNKNOWN=3
149
150_get_last_snapshot() {
151 zfs get creation -Hpr -t snapshot "$1" | \
152 awk 'BEGIN {max = 0} {if ($3>max) max=$3} END {print max}'
153}
154
155_getopts() {
156 while getopts ':c:d:hsvw:-:' OPT ; do
157 case $OPT in
158
159 c)
160 OPT_CRITICAL=$OPTARG
161 ;;
162
163 d)
164 OPT_DATASET="$OPTARG"
165 ;;
166
167 h)
168 echo "$USAGE"
169 exit 0
170 ;;
171
172 s)
173 echo "$SHORT_DESCRIPTION"
174 exit 0
175 ;;
176
177 v)
178 echo "$VERSION"
179 exit 0
180 ;;
181
182 w)
183 OPT_WARNING=$OPTARG
184 ;;
185
186 \?)
187 echo "Invalid option “-$OPTARG”!" >&2
188 exit 2
189 ;;
190
191 :)
192 echo "Option “-$OPTARG” requires an argument!" >&2
193 exit 3
194 ;;
195
196 -)
197 LONG_OPTARG="${OPTARG#*=}"
198
199 case $OPTARG in
200
201 critical=?*)
202 OPT_CRITICAL=$LONG_OPTARG
203 ;;
204
205 dataset=?*)
206 OPT_DATASET="$LONG_OPTARG"
207 ;;
208
209 help)
210 echo "$USAGE"
211 exit 0
212 ;;
213
214 short-description)
215 echo "$SHORT_DESCRIPTION"
216 exit 0
217 ;;
218
219 version)
220 echo "$VERSION"
221 exit 0
222 ;;
223
224 warning=?*)
225 OPT_WARNING=$LONG_OPTARG
226 ;;
227
228 critical*|dataset*|warning*)
229 echo "Option “--$OPTARG” requires an argument!" >&2
230 exit 3
231 ;;
232
233 help*|short-description*|version*)
234 echo "No argument allowed for the option “--$OPTARG”!" >&2
235 exit 4
236 ;;
237
238 '') # "--" terminates argument processing
239 break
240 ;;
241
242 *)
243 echo "Invalid option “--$OPTARG”!" >&2
244 exit 2
245 ;;
246
247 esac
248 ;;
249
250 esac
251 done
252}
253
254_snapshot_count() {
255 # FreeBSD wc adds some whitespaces before the number!
256 # cat $HOME/debug | wc -l
257 # 7
258 local COUNT
259 COUNT="$(zfs list -t snapshot | grep "$1" | wc -l)"
260 echo $COUNT
261}
262
263_performance_data() {
264 echo "| \
265last_ago=${DIFF}s;$OPT_WARNING;$OPT_CRITICAL;0 \
266count=$(_snapshot_count "$OPT_DATASET");;;0\
267"
268}
269
270## This SEPARATOR is required for test purposes. Please don’t remove! ##
271
272_getopts $@
273
274if [ -z "$OPT_WARNING" ]; then
275 # 1 day
276 OPT_WARNING=86400
277fi
278
279if [ -z "$OPT_CRITICAL" ]; then
280 # 3 day
281 OPT_CRITICAL=259200
282fi
283
284if [ -z "$OPT_DATASET" ]; then
285 echo "Dataset has to be set! Use option -d <dataset>" >&2
286 echo "$USAGE" >&2
287 exit $STATE_UNKNOWN
288fi
289
290if ! zfs list "$OPT_DATASET" > /dev/null 2>&1; then
291 echo "'$OPT_DATASET' is no ZFS dataset!" >&2
292 echo "$USAGE" >&2
293 exit $STATE_UNKNOWN
294fi
295
296NOW=$(_now_to_timestamp)
297
298CREATION_DATE=$(_get_last_snapshot "$OPT_DATASET")
299
300DIFF=$((NOW - CREATION_DATE))
301
302if [ "$OPT_WARNING" -gt "$OPT_CRITICAL" ]; then
303 echo '-w OPT_WARNING must be smaller than -c OPT_CRITICAL'
304 _usage >&2
305 exit $STATE_UNKNOWN
306fi
307
308RETURN=STATE_UNKNOWN
309
310if [ "$DIFF" -gt "$OPT_CRITICAL" ]; then
311 RETURN=$STATE_CRITICAL
312 MESSAGE="CRITICAL:"
313elif [ "$DIFF" -gt "$OPT_WARNING" ]; then
314 RETURN=$STATE_WARNING
315 MESSAGE="WARNING:"
316else
317 RETURN=$STATE_OK
318 MESSAGE="OK:"
319fi
320
321DATE="$(_timestamp_to_datetime "$CREATION_DATE")"
322
323echo "$MESSAGE Last snapshot for dataset '$OPT_DATASET' was created on $DATE $(_performance_data)"
324
325exit $RETURN
diff --git a/flakes/private/monitoring/plugins/notify_by_apprise b/flakes/private/monitoring/plugins/notify_by_apprise
new file mode 100755
index 0000000..82bc5a3
--- /dev/null
+++ b/flakes/private/monitoring/plugins/notify_by_apprise
@@ -0,0 +1,31 @@
1#!/usr/bin/env bash
2
3APPRISE_USERNAME="Naemon"
4APPRISE_USERICON="https://assets.immae.eu/monitoring.png"
5APPRISE_URLS=$(echo "$2" | sed -e "s/{username}/$APPRISE_USERNAME/g" -e "s@{image_url}@$APPRISE_USERICON@g")
6
7if [ "$SERVICESTATE" = "CRITICAL" ]; then
8 ICON="❗"
9elif [ "$SERVICESTATE" = "WARNING" ]; then
10 ICON="⚠️:"
11elif [ "$SERVICESTATE" = "OK" ]; then
12 ICON="✅"
13elif [ "$SERVICESTATE" = "UNKNOWN" ]; then
14 ICON="❓"
15elif [ "$HOSTSTATE" = "UP" ]; then
16 ICON="✅"
17elif [ "$HOSTSTATE" = "DOWN" ]; then
18 ICON="❗"
19elif [ "$HOSTSTATE" = "UNKNOWN" ]; then
20 ICON="❓"
21elif [ "$HOSTSTATE" = "UNREACHABLE" ]; then
22 ICON="❓"
23else
24 ICON="◻"
25fi
26
27if [ "$1" = "host" ]; then
28 apprise --title "${ICON} ${NOTIFICATIONTYPE} ${HOST} is ${HOSTSTATE}" --body "$HOSTOUTPUT" $APPRISE_URLS
29else
30 apprise --title "${ICON} ${NOTIFICATIONTYPE} ${SERVICEDESC} on ${HOST} is ${SERVICESTATE}" --body "$SERVICEOUTPUT" $APPRISE_URLS
31fi
diff --git a/flakes/private/monitoring/plugins/notify_by_email b/flakes/private/monitoring/plugins/notify_by_email
new file mode 100755
index 0000000..959db26
--- /dev/null
+++ b/flakes/private/monitoring/plugins/notify_by_email
@@ -0,0 +1,29 @@
1#!/usr/bin/env bash
2
3# $1 = service/host
4
5# $2 = type (PROBLEM RECOVERY ACKNOWLEDGEMENT FLAPPINGSTART FLAPPINGSTOP FLAPPINGDISABLED DOWNTIMESTART DOWNTIMESTOP DOWNTIMECANCELLED)
6# http://www.naemon.org/documentation/usersguide/macrolist.html#notificationtype
7
8# $3 = host alias
9
10# $4 = date (YYYY-MM-DDTHH:MM:SS)
11
12# $5 = E-mail
13
14NOTIFICATION_TYPE="$2"
15HOST_ALIAS="$3"
16DATE="$4"
17CONTACT="$5"
18
19message=""
20
21if [ "$1" = "host" ]; then
22 message=$(printf "%b" "***** Naemon *****\n\nNotification Type: $NOTIFICATION_TYPE\n\nHost: $HOST_ALIAS\nState: $HOSTSTATE\nInfo: $HOSTOUTPUT\n\nDate/Time: $DATE\n")
23 subject="** $NOTIFICATION_TYPE Host Alert: $HOST_ALIAS is $HOSTSTATE **"
24else
25 message=$(printf "%b" "***** Naemon *****\n\nNotification Type: $NOTIFICATION_TYPE\n\nService: $SERVICEDESC\nHost: $HOST_ALIAS\nState: $SERVICESTATE\n\nDate/Time: $DATE\n\nAdditional Info:\n\n$SERVICEOUTPUT\n")
26 subject="** $NOTIFICATION_TYPE Service Alert: $HOST_ALIAS/$SERVICEDESC is $SERVICESTATE **"
27fi
28
29echo "$message" | MAILRC=/dev/null mail -r "$ADMINEMAIL" -n -s "$subject" "$CONTACT"
diff --git a/flakes/private/monitoring/plugins/notify_by_slack b/flakes/private/monitoring/plugins/notify_by_slack
new file mode 100755
index 0000000..1b16a0d
--- /dev/null
+++ b/flakes/private/monitoring/plugins/notify_by_slack
@@ -0,0 +1,46 @@
1#!/usr/bin/env bash
2
3SLACK_CHANNEL="$1"
4SLACK_USERNAME="Naemon"
5SLACK_URL="$2"
6SLACK_USERICON="https://assets.immae.eu/monitoring.png"
7
8if [ "$SERVICESTATE" = "CRITICAL" ]; then
9 ICON=":exclamation:"
10 COLOR="#DA0505"
11elif [ "$SERVICESTATE" = "WARNING" ]; then
12 ICON=":warning:"
13 COLOR="#F1E903"
14elif [ "$SERVICESTATE" = "OK" ]; then
15 ICON=":white_check_mark:"
16 COLOR="#36a64f"
17elif [ "$SERVICESTATE" = "UNKNOWN" ]; then
18 ICON=":question:"
19 COLOR="#000000"
20else
21 ICON=":white_medium_square:"
22 COLOR="#ffffff"
23fi
24
25payload=$(echo "{}" | jq -r \
26 --arg "icon_url" "$SLACK_USERICON" \
27 --arg "channel" "$SLACK_CHANNEL" \
28 --arg "username" "$SLACK_USERNAME" \
29 --arg "text" "${ICON} ${SERVICEDESC} on ${HOST} is ${SERVICESTATE}" \
30 --arg "color" "$COLOR" \
31 --arg "host" "$HOST" \
32 --arg "desc" "$SERVICEDESC" \
33 --arg "state" "$SERVICESTATE" \
34 --arg "output" "$SERVICEOUTPUT" \
35 '.icon_url = $icon_url |
36 .channel = $channel |
37 .username = $username |
38 .text = $text |
39 .attachments = [{fallback:"", color:$color,fields: [{},{},{},{}]}] |
40 .attachments[0].fields[0] = {title:"Host",value:$host,short:true} |
41 .attachments[0].fields[1] = {title:"Service description",value:$desc,short:true} |
42 .attachments[0].fields[2] = {title:"Status",value:$state,short:true} |
43 .attachments[0].fields[3] = {title:"Message",value:$output,short:false}
44 ')
45
46curl -X POST --data "payload=$payload" $SLACK_URL
diff --git a/flakes/private/monitoring/plugins/send_nrdp.sh b/flakes/private/monitoring/plugins/send_nrdp.sh
new file mode 100755
index 0000000..c83c8cb
--- /dev/null
+++ b/flakes/private/monitoring/plugins/send_nrdp.sh
@@ -0,0 +1,57 @@
1#!/bin/bash
2
3TEMPLATE='{
4 "cmd": "submitcheck",
5 "token": $token,
6 "checkresult": [{
7 "hostname": $hostname,
8 "state": $state,
9 "output": $output,
10 "type": $type,
11 "servicename": $servicename
12 }]
13}'
14
15while getopts "u:t:H:s:S:o:" option
16do
17 case $option in
18 u) url=$OPTARG ;;
19 t) token=$OPTARG ;;
20 H) hostname=$OPTARG ;;
21 s) servicename=$OPTARG ;;
22 S) state=$OPTARG ;;
23 o) output=$OPTARG ;;
24 esac
25done
26
27if [ -n "$servicename" ]; then
28 checktype="service"
29else
30 checktype="host"
31fi
32
33payload=$(jq -n \
34 --arg type "$checktype" \
35 --arg hostname "$hostname" \
36 --arg servicename "$servicename" \
37 --arg output "$output" \
38 --arg token "$token" \
39 --arg state "$state" \
40 "$TEMPLATE")
41
42rslt=$(curl -f --silent --insecure -d "$payload" -H "Content-Type: application/json" "$url")
43ret=$?
44
45if [ $ret != 0 ];then
46 echo "ERROR: could not connect to NRDP server at $url"
47 exit 1
48fi
49
50status=$(echo "$rslt" | jq -r .status)
51message=$(echo "$rslt" | jq -r .message)
52
53if [ "$status" != "ok" ];then
54 echo "ERROR: The NRDP Server said $message"
55 exit 2
56fi
57echo "Sent 1 checks to $url"
diff --git a/flakes/private/monitoring/send_mails b/flakes/private/monitoring/send_mails
new file mode 100755
index 0000000..105c505
--- /dev/null
+++ b/flakes/private/monitoring/send_mails
@@ -0,0 +1,15 @@
1#!/usr/bin/env bash
2
3CHECK_DIR=$1
4shift
5RETURN_PATH=$1
6shift
7
8for mail in "$@"; do
9 echo "Test Mail" | MAILRC=/dev/null mail -n -r "$RETURN_PATH" -s "TestMailImmae " "$mail"
10done
11
12if [ -d "$CHECK_DIR" ]; then
13 cd $CHECK_DIR
14 stat -c '%n;%y' *
15fi
diff --git a/flakes/private/monitoring/to_objects.nix b/flakes/private/monitoring/to_objects.nix
new file mode 100644
index 0000000..57a71ad
--- /dev/null
+++ b/flakes/private/monitoring/to_objects.nix
@@ -0,0 +1,77 @@
1{ lib }:
2 with lib.attrsets;
3 with lib.strings;
4 with lib.lists;
5 with lib.trivial;
6let
7 pad = width: str: let
8 padWidth = width - stringLength str;
9 padding = concatStrings (genList (const " ") padWidth);
10 in str + optionalString (padWidth > 0) padding;
11 toStr = k: v:
12 if k == "check_command" && builtins.isList v
13 then builtins.concatStringsSep "!" v
14 else builtins.toString v;
15
16 toService = service: ''
17 define service {
18 ${builtins.concatStringsSep "\n" (mapAttrsToList (k: v:
19 " ${pad 30 k} ${toStr k v}"
20 ) (filterAttrs (k: v: ! hasPrefix "__passive_" k) service))}
21 }
22 '';
23 toServices = services: builtins.concatStringsSep "\n" (map toService services);
24
25 toCommand = k: v: ''
26 define command {
27 ${pad 30 "command_name"} ${k}
28 ${pad 30 "command_line"} ${v}
29 }
30 '';
31 toCommands = a: builtins.concatStringsSep "\n" (mapAttrsToList toCommand a);
32
33 toOther = keyname: k: v: ''
34 define ${keyname} {
35 ${pad 30 "${keyname}_name"} ${k}
36 ${builtins.concatStringsSep "\n" (mapAttrsToList (kk: vv:
37 " ${pad 30 kk} ${vv}"
38 ) v)}
39 }
40 '';
41 toOtherNoName = keyname: v: ''
42 define ${keyname} {
43 ${builtins.concatStringsSep "\n" (mapAttrsToList (kk: vv:
44 " ${pad 30 kk} ${vv}"
45 ) v)}
46 }
47 '';
48 toOthers = keyname: a: builtins.concatStringsSep "\n" (mapAttrsToList (toOther keyname) a);
49 toOthersArray = keyname: a: builtins.concatStringsSep "\n" (map (toOtherNoName keyname) a);
50
51 toTemplate = keyname: k: v: ''
52 define ${keyname} {
53 ${pad 30 "name"} ${k}
54 ${pad 30 "register"} 0
55 ${builtins.concatStringsSep "\n" (mapAttrsToList (kk: vv:
56 " ${pad 30 kk} ${builtins.toString vv}"
57 ) v)}
58 }
59 '';
60 toTemplates' = keyname: a: builtins.concatStringsSep "\n" (mapAttrsToList (toTemplate keyname) a);
61 toTemplates = v: builtins.concatStringsSep "\n" (mapAttrsToList toTemplates' v);
62
63 toObjects' = keyname: v:
64 if keyname == "service"
65 then toServices v
66 else if keyname == "command"
67 then toCommands v
68 else if keyname == "templates"
69 then toTemplates v
70 else if builtins.elem keyname ["hostgroup" "host" "contactgroup" "contact" "timeperiod" "servicegroup"]
71 then toOthers keyname v
72 else if builtins.elem keyname ["servicedependency" "hostdependency"]
73 then toOthersArray keyname v
74 else builtins.trace ("Warning: unknown object type " + keyname) "";
75 toObjects = v: builtins.concatStringsSep "\n" (mapAttrsToList toObjects' v);
76in
77 toObjects
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-10 07:26:29 +0000