]> git.immae.eu Git - perso/Immae/Config/Nix.git/blame - modules/private/monitoring/default.nix
Add beta for Denise OMS
[perso/Immae/Config/Nix.git] / modules / private / monitoring / default.nix
CommitLineData
e820134d 1{ config, pkgs, lib, name, nodes, ... }:
3bc32d9e 2let
e820134d 3 cfg = config.myServices.monitoring;
71a2425e
IB
4 send_mails = pkgs.runCommand "send_mails" {
5 buildInputs = [ pkgs.makeWrapper ];
6 } ''
7 mkdir -p $out/bin
8 cp ${./send_mails} $out/bin/send_mails
9 patchShebangs $out
10 wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
11 pkgs.mailutils
12 ]}
13 '';
9f16e659
IB
14 postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable
15 then config.myServices.databasesReplication.postgresql.mainPackage
16 else if config.myServices.databases.enable
17 then config.myServices.databases.postgresql.package
18 else pkgs.postgresql;
6ee77836
IB
19 zfsPlugin = pkgs.fetchurl {
20 url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh";
21 sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf";
22 };
f7a3019f
IB
23 megacli = pkgs.megacli.overrideAttrs(old: { meta = old.meta // { license = null; }; });
24 megaCliPlugin = pkgs.runCommand "megaCliPlugin" {
25 plugin = pkgs.fetchurl {
26 name = "check_megaraid_sas";
27 url = "https://exchange.nagios.org/components/com_mtree/attachment.php?link_id=6381&cf_id=24";
28 sha256 = "0yf60p4c0hb4q3fng9fc14qc89bqm0f1sijayzygadaqcl44jx4p";
29 };
30 } ''
31 mkdir $out
32 cp $plugin $out/check_megaraid_sas
33 chmod +x $out/check_megaraid_sas
34 patchShebangs $out
35 substituteInPlace $out/check_megaraid_sas --replace /usr/sbin/MegaCli ${megacli}/bin/MegaCli64
36 substituteInPlace $out/check_megaraid_sas --replace 'sudo $megacli' '/run/wrappers/bin/sudo $megacli'
37 sed -i -e "s/use utils qw(%ERRORS);/my %ERRORS = ('OK' => 0, 'WARNING' => 1, 'CRITICAL' => 2, 'UNKNOWN' => 3);/" $out/check_megaraid_sas
38 '';
3bc32d9e
IB
39 myplugins = pkgs.runCommand "buildplugins" {
40 buildInputs = [ pkgs.makeWrapper pkgs.perl ];
41 } ''
42 mkdir $out
6ee77836 43 cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh
f7a3019f 44 cp ${megaCliPlugin}/check_megaraid_sas $out/
3bc32d9e
IB
45 cp ${./plugins}/* $out/
46 patchShebangs $out
47 wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
6ee77836
IB
48 wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [
49 pkgs.which pkgs.zfs pkgs.gawk
50 ]}
3bc32d9e 51 wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
b11f0e17 52 pkgs.curl pkgs.jq
3bc32d9e
IB
53 ]}
54 wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
55 pkgs.gnugrep pkgs.gawk pkgs.procps-ng
56 ]}
9f202523 57 wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
9f16e659 58 postgresqlBinary
9f202523 59 ]}
6015a3b5
IB
60 wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [
61 pkgs.gnugrep pkgs.coreutils pkgs.redis
62 ]}
63 wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [
64 pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb
65 ]}
16b80abd
IB
66 wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
67 pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
68 ]}
71a2425e
IB
69 wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
70 pkgs.openssh send_mails
71 ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
72 pkgs.perlPackages.TimeDate
73 ]}
e820134d
IB
74 wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
75 pkgs.lftp
76 ]}
77 wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [
78 pkgs.git pkgs.openssh
79 ]}
80 wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [
81 pkgs.openssl
82 ]}
25844101
IB
83 wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
84 pkgs.s3cmd pkgs.python3
85 ]}
5a61f6ad 86 makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
171d8e1a
IB
87 pkgs.duplicity
88 ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
5a61f6ad
IB
89 makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
90 pkgs.duplicity
91 ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
e820134d
IB
92 wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
93 pkgs.mailutils
94 ]}
95 wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
96 pkgs.curl pkgs.jq
97 ]}
2edbb2d8
IB
98 wrapProgram $out/notify_eban_url --prefix PATH : ${lib.makeBinPath [
99 pkgs.curl
100 ]}
6191bdeb
IB
101 wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
102 (pkgs.python3.withPackages (ps: [ps.ovh]))
103 ]}
2d7caffb
IB
104 wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
105 pkgs.iproute pkgs.bc
106 ]}
3bc32d9e 107 '';
eb071dd4
IB
108 toObjects = pkgs.callPackage ./to_objects.nix {};
109 commonConfig = {
6ee77836
IB
110 dilion = {
111 processWarn = "250"; processAlert = "400";
112 loadWarn = "8.0"; loadAlert = "10.0";
113 interface = "eth0";
114 };
eb071dd4
IB
115 eldiron = {
116 processWarn = "250"; processAlert = "400";
117 loadWarn = "8.0"; loadAlert = "10.0";
2d7caffb 118 interface = "eth0";
eb071dd4
IB
119 };
120 backup-2 = {
25844101 121 processWarn = "60"; processAlert = "70";
eb071dd4 122 loadWarn = "1.0"; loadAlert = "2.0";
2d7caffb 123 interface = "ens3";
eb071dd4 124 };
e820134d
IB
125 monitoring-1 = {
126 processWarn = "50"; processAlert = "60";
7ad4966f
IB
127 loadWarn = "4.0"; loadAlert = "6.0";
128 load15Warn = "1.0"; load15Alert = "2.0";
2d7caffb 129 interface = "ens3";
e820134d 130 };
6ee77836
IB
131 quatresaisons = {
132 processWarn = "250"; processAlert = "400";
133 loadWarn = "8.0"; loadAlert = "10.0";
134 interface = "eth0";
135 };
eb071dd4 136 };
46a61a1b
IB
137 externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
138 (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
e820134d 139 masterPassiveObjects = let
6ee77836 140 passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
e820134d
IB
141 toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
142 passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
143 (_: n: toPassiveServices n.config.myServices.monitoring.services)
144 passiveNodes
46a61a1b
IB
145 ) ++ lib.flatten (lib.attrsets.mapAttrsToList
146 (_: n: toPassiveServices n.service)
147 externalObjects);
e820134d 148 in {
71a2425e 149 service = passiveServices;
e820134d
IB
150 host = lib.lists.foldr
151 (a: b: a//b)
152 {}
46a61a1b
IB
153 (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
154 ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
e820134d 155 };
71a2425e
IB
156 emailCheck = host: hostFQDN: let
157 allCfg = config.myEnv.monitoring.email_check;
158 cfg = allCfg."${host}";
159 reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
160 to_email = cfg': host':
161 let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
162 in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
163 mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
164 mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
165 command = if cfg.local
166 then
167 [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
168 else
169 [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
170 in
171 {
172 service_description = "${hostFQDN} email service is active";
173 use = "mail-service";
174 host_name = hostFQDN;
175 servicegroups = "webstatus-email";
176 check_command = command;
177 };
178 otherObjects = map
179 (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
2edbb2d8 180 [ "ulminfo-fr" "phare" "eban" ];
e820134d 181 masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
eb071dd4 182 commonObjects = pkgs.callPackage ./objects_common.nix ({
e820134d 183 master = cfg.master;
619e4f46 184 hostFQDN = config.hostEnv.fqdn;
e820134d 185 hostName = name;
eb071dd4
IB
186 sudo = "/run/wrappers/bin/sudo";
187 } // builtins.getAttr name commonConfig);
188 hostObjects =
189 let
190 specific_file = ./. + "/objects_" + name + ".nix";
9f202523 191 in
e820134d
IB
192 lib.attrsets.optionalAttrs
193 (builtins.pathExists specific_file)
194 (pkgs.callPackage specific_file {
171d8e1a 195 inherit config nodes emailCheck;
619e4f46 196 hostFQDN = config.hostEnv.fqdn;
e820134d
IB
197 hostName = name;
198 });
46a61a1b
IB
199 objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
200 "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
201 ) externalObjects;
3bc32d9e
IB
202in
203{
204 options = {
9f202523
IB
205 myServices.monitoring = {
206 enable = lib.mkOption {
207 type = lib.types.bool;
208 default = false;
209 description = ''
210 Whether to enable monitoring.
211 '';
212 };
e820134d
IB
213 master = lib.mkOption {
214 type = lib.types.bool;
215 default = false;
216 description = ''
217 This instance is the master instance
218 '';
219 };
220 hosts = lib.mkOption {
221 readOnly = true;
222 description = "Hosts list for this host";
223 default = (commonObjects.host or {}) // (hostObjects.host or {});
224 };
225 services = lib.mkOption {
226 readOnly = true;
227 description = "Services list for this host";
228 default = commonObjects.service ++ hostObjects.service;
229 };
3bc32d9e
IB
230 };
231 };
232
e820134d 233 config = lib.mkIf cfg.enable {
46a61a1b
IB
234 services.nginx = lib.mkIf config.myServices.status.enable {
235 virtualHosts."status.immae.eu".locations = objectsFiles;
236 };
237
d2e703c5 238 services.duplyBackup.profiles.monitoring = {
6a8252b1
IB
239 rootDir = config.services.naemon.varDir;
240 };
3bc32d9e
IB
241 security.sudo.extraRules = [
242 {
243 commands = [
244 { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
245 { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
f7a3019f 246 { command = "${megacli}/bin/MegaCli64"; options = [ "NOPASSWD" ]; }
3bc32d9e
IB
247 ];
248 users = [ "naemon" ];
249 runAs = "root";
250 }
9f202523
IB
251 {
252 commands = [
9f202523
IB
253 { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
254 ];
255 users = [ "naemon" ];
16b80abd
IB
256 runAs = "ALL";
257 }
258 {
259 commands = [
260 { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
261 ];
262 users = [ "naemon" ];
9f202523
IB
263 runAs = "postgres";
264 }
265 {
266 commands = [
6015a3b5 267 { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; }
9f6a7862
IB
268 ];
269 users = [ "naemon" ];
270 runAs = "mysql";
271 }
6015a3b5
IB
272 {
273 commands = [
16b80abd 274 { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; }
6015a3b5
IB
275 ];
276 users = [ "naemon" ];
16b80abd 277 runAs = "openldap";
6015a3b5 278 }
9f6a7862
IB
279 {
280 commands = [
16b80abd 281 { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; }
9f202523
IB
282 ];
283 users = [ "naemon" ];
16b80abd 284 runAs = "redis";
9f202523 285 }
3bc32d9e
IB
286 ];
287 environment.etc."mdadm.conf" = {
288 enable = true;
289 mode = "0644";
290 user = "root";
ab8f306d 291 text = "MAILADDR ${config.myEnv.monitoring.email}";
3bc32d9e
IB
292 };
293
e820134d
IB
294 secrets.keys = [
295 {
296 dest = "naemon/id_rsa";
297 user = "naemon";
298 group = "naemon";
171d8e1a 299 permissions = "0400";
e820134d
IB
300 text = config.myEnv.monitoring.ssh_secret_key;
301 }
5a61f6ad
IB
302 ] ++ lib.optionals cfg.master (
303 lib.mapAttrsToList (k: v:
171d8e1a 304 {
5a61f6ad 305 dest = "${k}_access_key";
171d8e1a
IB
306 user = "naemon";
307 group = "naemon";
308 permissions = "0400";
309 text = ''
5a61f6ad
IB
310 export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
311 export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
312 export BASE_URL="${v.remote "immae-eldiron"}"
171d8e1a 313 '';
5a61f6ad 314 }) config.myEnv.backup.remotes
171d8e1a 315 );
3bc32d9e
IB
316 # needed since extraResource is not in the closure
317 systemd.services.naemon.path = [ myplugins ];
318 services.naemon = {
319 enable = true;
320 extraConfig = ''
3bc32d9e
IB
321 use_syslog=1
322 log_initial_states=1
323 date_format=iso8601
ab8f306d 324 admin_email=${config.myEnv.monitoring.email}
e820134d 325 '' + lib.optionalString (!cfg.master) ''
3bc32d9e
IB
326 obsess_over_services=1
327 ocsp_command=notify-master
a97118c4
IB
328 '' + lib.optionalString (cfg.master) ''
329 broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
330 broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
3bc32d9e
IB
331 '';
332 extraResource = ''
333 $USER2$=${myplugins}
ab8f306d
IB
334 $USER200$=${config.myEnv.monitoring.status_url}
335 $USER201$=${config.myEnv.monitoring.status_token}
e820134d
IB
336 $USER202$=${config.myEnv.monitoring.http_user_password}
337 $USER203$=${config.secrets.fullPaths."naemon/id_rsa"}
338 $USER204$=${config.myEnv.monitoring.imap_login}
339 $USER205$=${config.myEnv.monitoring.imap_password}
340 $USER206$=${config.myEnv.monitoring.slack_channel}
341 $USER207$=${config.myEnv.monitoring.slack_url}
25844101 342 $USER208$=${builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys)}
6191bdeb
IB
343 $USER209$=${builtins.concatStringsSep "," [
344 config.myEnv.monitoring.ovh_sms.endpoint
345 config.myEnv.monitoring.ovh_sms.application_key
346 config.myEnv.monitoring.ovh_sms.application_secret
347 config.myEnv.monitoring.ovh_sms.consumer_key
348 config.myEnv.monitoring.ovh_sms.account
349 ]}
2edbb2d8 350 $USER210$=${config.myEnv.monitoring.eban.password}
3bc32d9e 351 '';
e820134d
IB
352 objectDefs = toObjects commonObjects
353 + toObjects hostObjects
354 + lib.optionalString cfg.master (toObjects masterObjects)
71a2425e
IB
355 + lib.optionalString cfg.master (toObjects masterPassiveObjects)
356 + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
3bc32d9e
IB
357 };
358 };
359}