1 { config, pkgs, lib, name, nodes, ... }:
3 cfg = config.myServices.monitoring;
4 send_mails = pkgs.runCommand "send_mails" {
5 buildInputs = [ pkgs.makeWrapper ];
8 cp ${./send_mails} $out/bin/send_mails
10 wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
14 postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable
15 then config.myServices.databasesReplication.postgresql.mainPackage
16 else if config.myServices.databases.enable
17 then config.myServices.databases.postgresql.package
19 zfsPlugin = pkgs.fetchurl {
20 url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh";
21 sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf";
23 myplugins = pkgs.runCommand "buildplugins" {
24 buildInputs = [ pkgs.makeWrapper pkgs.perl ];
27 cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh
28 cp ${./plugins}/* $out/
30 wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
31 wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [
32 pkgs.which pkgs.zfs pkgs.gawk
34 wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
37 wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
38 pkgs.gnugrep pkgs.gawk pkgs.procps-ng
40 wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
43 wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [
44 pkgs.gnugrep pkgs.coreutils pkgs.redis
46 wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [
47 pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb
49 wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
50 pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
52 wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
53 pkgs.openssh send_mails
54 ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
55 pkgs.perlPackages.TimeDate
57 wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
60 wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [
63 wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [
66 wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
67 pkgs.s3cmd pkgs.python3
69 makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
71 ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
72 makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
74 ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
75 wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
78 wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
81 wrapProgram $out/notify_eban_url --prefix PATH : ${lib.makeBinPath [
84 wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
85 (pkgs.python3.withPackages (ps: [ps.ovh]))
87 wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
91 toObjects = pkgs.callPackage ./to_objects.nix {};
94 processWarn = "250"; processAlert = "400";
95 loadWarn = "8.0"; loadAlert = "10.0";
99 processWarn = "250"; processAlert = "400";
100 loadWarn = "8.0"; loadAlert = "10.0";
104 processWarn = "60"; processAlert = "70";
105 loadWarn = "1.0"; loadAlert = "2.0";
109 processWarn = "50"; processAlert = "60";
110 loadWarn = "4.0"; loadAlert = "6.0";
111 load15Warn = "1.0"; load15Alert = "2.0";
115 processWarn = "250"; processAlert = "400";
116 loadWarn = "8.0"; loadAlert = "10.0";
120 externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
121 (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
122 masterPassiveObjects = let
123 passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
124 toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
125 passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
126 (_: n: toPassiveServices n.config.myServices.monitoring.services)
128 ) ++ lib.flatten (lib.attrsets.mapAttrsToList
129 (_: n: toPassiveServices n.service)
132 service = passiveServices;
133 host = lib.lists.foldr
136 (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
137 ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
139 emailCheck = host: hostFQDN: let
140 allCfg = config.myEnv.monitoring.email_check;
141 cfg = allCfg."${host}";
142 reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
143 to_email = cfg': host':
144 let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
145 in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
146 mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
147 mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
148 command = if cfg.local
150 [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
152 [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
155 service_description = "${hostFQDN} email service is active";
156 use = "mail-service";
157 host_name = hostFQDN;
158 servicegroups = "webstatus-email";
159 check_command = command;
162 (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
163 [ "ulminfo-fr" "phare" "eban" ];
164 masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
165 commonObjects = pkgs.callPackage ./objects_common.nix ({
167 hostFQDN = config.hostEnv.fqdn;
169 sudo = "/run/wrappers/bin/sudo";
170 } // builtins.getAttr name commonConfig);
173 specific_file = ./. + "/objects_" + name + ".nix";
175 lib.attrsets.optionalAttrs
176 (builtins.pathExists specific_file)
177 (pkgs.callPackage specific_file {
178 inherit config nodes emailCheck;
179 hostFQDN = config.hostEnv.fqdn;
182 objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
183 "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
188 myServices.monitoring = {
189 enable = lib.mkOption {
190 type = lib.types.bool;
193 Whether to enable monitoring.
196 master = lib.mkOption {
197 type = lib.types.bool;
200 This instance is the master instance
203 hosts = lib.mkOption {
205 description = "Hosts list for this host";
206 default = (commonObjects.host or {}) // (hostObjects.host or {});
208 services = lib.mkOption {
210 description = "Services list for this host";
211 default = commonObjects.service ++ hostObjects.service;
216 config = lib.mkIf cfg.enable {
217 services.nginx = lib.mkIf config.myServices.status.enable {
218 virtualHosts."status.immae.eu".locations = objectsFiles;
221 services.duplyBackup.profiles.monitoring = {
222 rootDir = config.services.naemon.varDir;
224 security.sudo.extraRules = [
227 { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
228 { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
230 users = [ "naemon" ];
235 { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
237 users = [ "naemon" ];
242 { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
244 users = [ "naemon" ];
249 { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; }
251 users = [ "naemon" ];
256 { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; }
258 users = [ "naemon" ];
263 { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; }
265 users = [ "naemon" ];
269 environment.etc."mdadm.conf" = {
273 text = "MAILADDR ${config.myEnv.monitoring.email}";
278 dest = "naemon/id_rsa";
281 permissions = "0400";
282 text = config.myEnv.monitoring.ssh_secret_key;
284 ] ++ lib.optionals cfg.master (
285 lib.mapAttrsToList (k: v:
287 dest = "${k}_access_key";
290 permissions = "0400";
292 export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
293 export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
294 export BASE_URL="${v.remote "immae-eldiron"}"
296 }) config.myEnv.backup.remotes
298 # needed since extraResource is not in the closure
299 systemd.services.naemon.path = [ myplugins ];
306 admin_email=${config.myEnv.monitoring.email}
307 '' + lib.optionalString (!cfg.master) ''
308 obsess_over_services=1
309 ocsp_command=notify-master
310 '' + lib.optionalString (cfg.master) ''
311 broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
312 broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
316 $USER200$=${config.myEnv.monitoring.status_url}
317 $USER201$=${config.myEnv.monitoring.status_token}
318 $USER202$=${config.myEnv.monitoring.http_user_password}
319 $USER203$=${config.secrets.fullPaths."naemon/id_rsa"}
320 $USER204$=${config.myEnv.monitoring.imap_login}
321 $USER205$=${config.myEnv.monitoring.imap_password}
322 $USER206$=${config.myEnv.monitoring.slack_channel}
323 $USER207$=${config.myEnv.monitoring.slack_url}
324 $USER208$=${builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys)}
325 $USER209$=${builtins.concatStringsSep "," [
326 config.myEnv.monitoring.ovh_sms.endpoint
327 config.myEnv.monitoring.ovh_sms.application_key
328 config.myEnv.monitoring.ovh_sms.application_secret
329 config.myEnv.monitoring.ovh_sms.consumer_key
330 config.myEnv.monitoring.ovh_sms.account
332 $USER210$=${config.myEnv.monitoring.eban.password}
334 objectDefs = toObjects commonObjects
335 + toObjects hostObjects
336 + lib.optionalString cfg.master (toObjects masterObjects)
337 + lib.optionalString cfg.master (toObjects masterPassiveObjects)
338 + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));