From: Ismaël Bouya Date: Sun, 2 May 2021 02:10:13 +0000 (+0200) Subject: Refactor monitoring to avoid useless resources in each machine X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=acab8301f6a0a76d97b36d009a60d4172bf62981;p=perso%2FImmae%2FConfig%2FNix.git Refactor monitoring to avoid useless resources in each machine --- diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix index 5f8a8c9..f00fb7c 100644 --- a/modules/private/monitoring/default.nix +++ b/modules/private/monitoring/default.nix @@ -1,109 +1,23 @@ { config, pkgs, lib, name, nodes, ... }: let cfg = config.myServices.monitoring; - send_mails = pkgs.runCommand "send_mails" { - buildInputs = [ pkgs.makeWrapper ]; - } '' - mkdir -p $out/bin - cp ${./send_mails} $out/bin/send_mails - patchShebangs $out - wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [ - pkgs.mailutils - ]} - ''; - postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable - then config.myServices.databasesReplication.postgresql.mainPackage - else if config.myServices.databases.enable - then config.myServices.databases.postgresql.package - else pkgs.postgresql; - zfsPlugin = pkgs.fetchurl { - url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh"; - sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf"; + activatedPlugins = [ "memory" "command" "bandwidth" ] + ++ (if cfg.master then (masterObjects.activatedPlugins or []) else []) + ++ (if cfg.master then (lib.flatten (map (v: v.activatedPlugins or []) otherObjects)) else []) + ++ (hostObjects.activatedPlugins or []) + ++ (if cfg.master then ["notify-primary"] else ["notify-secondary"]); + allPluginsConfig = import ./myplugins.nix { + inherit pkgs lib config; + sudo = "/run/wrappers/bin/sudo"; }; - megacli = pkgs.megacli.overrideAttrs(old: { meta = old.meta // { license = null; }; }); - megaCliPlugin = pkgs.runCommand "megaCliPlugin" { - plugin = pkgs.fetchurl { - name = "check_megaraid_sas"; - url = "https://exchange.nagios.org/components/com_mtree/attachment.php?link_id=6381&cf_id=24"; - sha256 = "0yf60p4c0hb4q3fng9fc14qc89bqm0f1sijayzygadaqcl44jx4p"; - }; - } '' - mkdir $out - cp $plugin $out/check_megaraid_sas - chmod +x $out/check_megaraid_sas - patchShebangs $out - substituteInPlace $out/check_megaraid_sas --replace /usr/sbin/MegaCli ${megacli}/bin/MegaCli64 - substituteInPlace $out/check_megaraid_sas --replace 'sudo $megacli' '/run/wrappers/bin/sudo $megacli' - sed -i -e "s/use utils qw(%ERRORS);/my %ERRORS = ('OK' => 0, 'WARNING' => 1, 'CRITICAL' => 2, 'UNKNOWN' => 3);/" $out/check_megaraid_sas - ''; - myplugins = pkgs.runCommand "buildplugins" { + mypluginsConfig = lib.getAttrs activatedPlugins allPluginsConfig; + myplugins = let + mypluginsChunk = builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList (k: v: v.chunk or "") mypluginsConfig); + in pkgs.runCommand "buildplugins" { buildInputs = [ pkgs.makeWrapper pkgs.perl ]; } '' mkdir $out - cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh - cp ${megaCliPlugin}/check_megaraid_sas $out/ - cp ${./plugins}/* $out/ - patchShebangs $out - wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir} - wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [ - pkgs.which pkgs.zfs pkgs.gawk - ]} - wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [ - pkgs.curl pkgs.jq - ]} - wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [ - pkgs.gnugrep pkgs.gawk pkgs.procps-ng - ]} - wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [ - postgresqlBinary - ]} - wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [ - pkgs.gnugrep pkgs.coreutils pkgs.redis - ]} - wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [ - pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb - ]} - wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [ - pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap - ]} - wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [ - pkgs.openssh send_mails - ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [ - pkgs.perlPackages.TimeDate - ]} - wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [ - pkgs.lftp - ]} - wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [ - pkgs.git pkgs.openssh - ]} - wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [ - pkgs.openssl - ]} - wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [ - pkgs.s3cmd pkgs.python3 - ]} - makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [ - pkgs.duplicity - ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"} - makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [ - pkgs.duplicity - ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"} - wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [ - pkgs.mailutils - ]} - wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [ - pkgs.curl pkgs.jq - ]} - wrapProgram $out/notify_eban_url --prefix PATH : ${lib.makeBinPath [ - pkgs.curl - ]} - wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [ - (pkgs.python3.withPackages (ps: [ps.ovh])) - ]} - wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [ - pkgs.iproute pkgs.bc - ]} + ${mypluginsChunk} ''; toObjects = pkgs.callPackage ./to_objects.nix {}; commonConfig = { @@ -183,7 +97,7 @@ let master = cfg.master; hostFQDN = config.hostEnv.fqdn; hostName = name; - sudo = "/run/wrappers/bin/sudo"; + inherit mypluginsConfig; } // builtins.getAttr name commonConfig); hostObjects = let @@ -263,52 +177,21 @@ in services.duplyBackup.profiles.monitoring = { rootDir = config.services.naemon.varDir; }; - security.sudo.extraRules = [ + security.sudo.extraRules = let + pluginsSudo = lib.lists.remove null (lib.attrsets.mapAttrsToList (k: v: + if (v ? sudo) + then ({ users = [ "naemon" ]; } // (v.sudo myplugins)) + else null) mypluginsConfig); + in [ { commands = [ { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; } { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; } - { command = "${megacli}/bin/MegaCli64"; options = [ "NOPASSWD" ]; } ]; users = [ "naemon" ]; runAs = "root"; } - { - commands = [ - { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "ALL"; - } - { - commands = [ - { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "postgres"; - } - { - commands = [ - { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "mysql"; - } - { - commands = [ - { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "openldap"; - } - { - commands = [ - { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; } - ]; - users = [ "naemon" ]; - runAs = "redis"; - } - ]; + ] ++ pluginsSudo; environment.etc."mdadm.conf" = { enable = true; mode = "0644"; @@ -354,26 +237,13 @@ in broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0 ''; - extraResource = '' + extraResource = let + resources = lib.mapAttrsToList (k: v: v.resources or {}) mypluginsConfig; + joined = lib.zipAttrsWith (n: v: if builtins.length (lib.unique v) == 1 then builtins.head v else abort "Non-unique resources names") resources; + joinedStr = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "$" + "${k}$=${v}") joined); + in '' $USER2$=${myplugins} - $USER200$=${config.myEnv.monitoring.status_url} - $USER201$=${config.myEnv.monitoring.status_token} - $USER202$=${config.myEnv.monitoring.http_user_password} - $USER203$=${config.secrets.fullPaths."naemon/id_rsa"} - $USER204$=${config.myEnv.monitoring.imap_login} - $USER205$=${config.myEnv.monitoring.imap_password} - $USER206$=${config.myEnv.monitoring.slack_channel} - $USER207$=${config.myEnv.monitoring.slack_url} - $USER208$=${builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys)} - $USER209$=${builtins.concatStringsSep "," [ - config.myEnv.monitoring.ovh_sms.endpoint - config.myEnv.monitoring.ovh_sms.application_key - config.myEnv.monitoring.ovh_sms.application_secret - config.myEnv.monitoring.ovh_sms.consumer_key - config.myEnv.monitoring.ovh_sms.account - ]} - $USER210$=${config.myEnv.monitoring.eban.user} - $USER211$=${config.myEnv.monitoring.eban.password} + ${joinedStr} ''; objectDefs = toObjects commonObjects + toObjects hostObjects diff --git a/modules/private/monitoring/myplugins.nix b/modules/private/monitoring/myplugins.nix new file mode 100644 index 0000000..8c77ee7 --- /dev/null +++ b/modules/private/monitoring/myplugins.nix @@ -0,0 +1,377 @@ +{ sudo, pkgs, lib, config }: +let + cfg = config.myServices.monitoring; +in +{ + notify-secondary = { + resources = { + USER200 = config.myEnv.monitoring.status_url; + USER201 = config.myEnv.monitoring.status_token; + }; + commands = { + notify-master = "$USER2$/send_nrdp.sh -u \"$USER200$\" -t \"$USER201$\" -H \"$HOSTADDRESS$\" -s \"$SERVICEDESC$\" -S \"$SERVICESTATEID$\" -o \"$SERVICEOUTPUT$ | $SERVICEPERFDATA$\""; + }; + chunk = '' + cp ${./plugins}/send_nrdp.sh $out + patchShebangs $out/send_nrdp.sh + wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [ + pkgs.curl pkgs.jq + ]} + ''; + }; + notify-primary = { + resources = { + USER206 = config.myEnv.monitoring.slack_channel; + USER207 = config.myEnv.monitoring.slack_url; + USER210 = config.myEnv.monitoring.eban.user; + USER211 = config.myEnv.monitoring.eban.password; + }; + commands = { + # $OVE is to force naemon to run via shell instead of execve which fails here + notify-host-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" HOSTSTATE=\"$HOSTSTATE$\" HOSTOUTPUT=\"$HOSTOUTPUT$\" $USER2$/notify_by_email host \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; + # $OVE is to force naemon to run via shell instead of execve which fails here + notify-service-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; + notify-by-slack = "HOST=\"$HOSTALIAS$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_slack \"$ARG1$\" \"$ARG2$\""; + notify-host-eban-url = "STATUS_NAME=\"Server\" USER=\"$USER210$\" PASSWORD=\"$USER211$\" HOSTSTATE=\"$HOSTSTATE$\" $USER2$/notify_eban_url"; + notify-service-eban-url = "STATUS_NAME=\"$_SERVICEWEBSTATUS_NAME$\" USER=\"$USER210$\" PASSWORD=\"$USER211$\" SERVICESTATE=\"$SERVICESTATE$\" $USER2$/notify_eban_url"; + }; + chunk = '' + cp ${./plugins}/{notify_by_email,notify_by_slack,notify_eban_url} $out + patchShebangs $out/{notify_by_email,notify_by_slack,notify_eban_url} + wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [ + pkgs.mailutils + ]} + wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [ + pkgs.curl pkgs.jq + ]} + wrapProgram $out/notify_eban_url --prefix PATH : ${lib.makeBinPath [ + pkgs.curl + ]} + ''; + }; + bandwidth = { + commands = { + check_local_bandwidth = "$USER2$/check_bandwidth -i=$ARG1$ -w $ARG2$ -c $ARG3$"; + }; + chunk = '' + cp ${./plugins}/check_bandwidth $out/ + patchShebangs $out/check_bandwidth + wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [ + pkgs.iproute pkgs.bc + ]} + ''; + }; + command = { + commands = { + check_command_match = "$USER2$/check_command -c \"$ARG1$\" -C \"$ARG2$\" $ARG3$"; + check_command_output = "$USER2$/check_command -c \"$ARG1$\" -s 0 -o \"$ARG2$\" $ARG3$"; + check_command_status = "$USER2$/check_command -c \"$ARG1$\" -s \"$ARG2$\" $ARG3$"; + }; + chunk = '' + cp ${./plugins}/check_command $out/ + patchShebangs $out/check_command + wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir} + ''; + }; + dns = { + commands = { + check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$"; + check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$"; + }; + }; + emails = { + resources = { + USER203 = config.secrets.fullPaths."naemon/id_rsa"; + }; + commands = { + check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$"; + check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$"; + }; + chunk = let + send_mails = pkgs.runCommand "send_mails" { + buildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out/bin + cp ${./send_mails} $out/bin/send_mails + patchShebangs $out + wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [ + pkgs.mailutils + ]} + ''; + in '' + cp ${./plugins}/check_emails $out/ + patchShebangs $out/check_emails + wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [ + pkgs.openssh send_mails + ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [ + pkgs.perlPackages.TimeDate + ]} + ''; + }; + eriomem = { + resources = { + USER208 = builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys); + }; + commands = { + check_backup_eriomem = "$USER2$/check_eriomem $USER208$"; + check_backup_eriomem_age = "$USER2$/check_backup_eriomem_age $ARG1$"; + }; + chunk = '' + cp ${./plugins}/check_eriomem $out/ + patchShebangs $out/check_eriomem + wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [ + pkgs.s3cmd pkgs.python3 + ]} + cp ${./plugins}/check_backup_age $out/check_backup_eriomem_age + patchShebangs $out/check_backup_eriomem_age + wrapProgram $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [ + pkgs.duplicity + ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"} + ''; + }; + file_date = { + commands = { + check_last_file_date = "${sudo} -u \"$ARG3$\" $USER2$/check_last_file_date \"$ARG1$\" \"$ARG2$\""; + }; + chunk = '' + cp ${./plugins}/check_last_file_date $out/ + patchShebangs $out/check_last_file_date + ''; + sudo = myplugins: { + commands = [ + { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; } + ]; + runAs = "ALL"; + }; + }; + ftp = { + commands = { + check_ftp_database = "$USER2$/check_ftp_database"; + }; + chunk = '' + cp ${./plugins}/check_ftp_database $out/ + patchShebangs $out/check_ftp_database + wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [ + pkgs.lftp + ]} + ''; + }; + git = { + resources = { + USER203 = config.secrets.fullPaths."naemon/id_rsa"; + }; + commands = { + check_git = "$USER2$/check_git $USER203$"; + }; + chunk = '' + cp ${./plugins}/check_git $out/ + patchShebangs $out/check_git + wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [ + pkgs.git pkgs.openssh + ]} + ''; + }; + http = { + resources = { + USER202 = config.myEnv.monitoring.http_user_password; + }; + commands = { + check_http = "$USER1$/check_http --sni -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; + check_https = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; + check_https_auth = "$USER1$/check_http --sni --ssl -a \"$USER202$\" -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; + check_https_certificate = "$USER1$/check_http --sni --ssl -H \"$ARG1$\" -C 21,15"; + check_https_code = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -e \"$ARG3$\" -r \"$ARG4$\""; + }; + }; + imap = { + resources = { + USER204 = config.myEnv.monitoring.imap_login; + USER205 = config.myEnv.monitoring.imap_password; + }; + commands = { + check_imap_connection = "$USER2$/check_imap_connection -u \"$USER204$\" -p \"$USER205$\" -H \"imap.immae.eu:143\""; + }; + chunk = '' + cp ${./plugins}/check_imap_connection $out/ + patchShebangs $out/check_imap_connection + wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [ + pkgs.openssl + ]} + ''; + }; + megaraid = let + megacli = pkgs.megacli.overrideAttrs(old: { meta = old.meta // { license = null; }; }); + in { + commands = { + check_megaraid = "$USER2$/check_megaraid_sas --sudo"; + }; + chunk = let + megaCliPlugin = pkgs.runCommand "megaCliPlugin" { + plugin = pkgs.fetchurl { + name = "check_megaraid_sas"; + url = "https://exchange.nagios.org/components/com_mtree/attachment.php?link_id=6381&cf_id=24"; + sha256 = "0yf60p4c0hb4q3fng9fc14qc89bqm0f1sijayzygadaqcl44jx4p"; + }; + } '' + mkdir $out + cp $plugin $out/check_megaraid_sas + chmod +x $out/check_megaraid_sas + patchShebangs $out + substituteInPlace $out/check_megaraid_sas --replace /usr/sbin/MegaCli ${megacli}/bin/MegaCli64 + substituteInPlace $out/check_megaraid_sas --replace 'sudo $megacli' '${sudo} $megacli' + sed -i -e "s/use utils qw(%ERRORS);/my %ERRORS = ('OK' => 0, 'WARNING' => 1, 'CRITICAL' => 2, 'UNKNOWN' => 3);/" $out/check_megaraid_sas + ''; + in '' + cp ${megaCliPlugin}/check_megaraid_sas $out/ + patchShebangs $out/check_megaraid_sas + ''; + sudo = _: { + commands = [ + { command = "${megacli}/bin/MegaCli64"; options = [ "NOPASSWD" ]; } + ]; + runAs = "root"; + }; + }; + memory = { + commands = { + check_memory = "$USER2$/check_mem.sh -w $ARG1$ -c $ARG2$"; + }; + chunk = '' + cp ${./plugins}/check_mem.sh $out/ + patchShebangs $out/check_mem.sh + wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [ + pkgs.gnugrep pkgs.gawk pkgs.procps-ng + ]} + ''; + }; + mysql = { + commands = { + check_mysql_replication = "${sudo} -u mysql $USER2$/check_mysql_replication \"$ARG1$\" \"$ARG2$\""; + }; + chunk = '' + cp ${./plugins}/check_mysql_replication $out/ + patchShebangs $out/check_mysql_replication + wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [ + pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb + ]} + ''; + sudo = myplugins: { + commands = [ + { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; } + ]; + runAs = "mysql"; + }; + }; + openldap = { + commands = { + check_openldap_replication = "${sudo} -u openldap $USER2$/check_openldap_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\" \"$ARG4$\" \"$ARG5$\""; + }; + chunk = '' + cp ${./plugins}/check_openldap_replication $out/ + patchShebangs $out/check_openldap_replication + wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [ + pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap + ]} + ''; + sudo = myplugins: { + commands = [ + { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; } + ]; + runAs = "openldap"; + }; + }; + ovh = { + resources = { + USER209 = builtins.concatStringsSep "," [ + config.myEnv.monitoring.ovh_sms.endpoint + config.myEnv.monitoring.ovh_sms.application_key + config.myEnv.monitoring.ovh_sms.application_secret + config.myEnv.monitoring.ovh_sms.consumer_key + config.myEnv.monitoring.ovh_sms.account + ]; + }; + commands = { + check_backup_ovh_age = "$USER2$/check_backup_ovh_age $ARG1$"; + check_ovh_sms = "$USER2$/check_ovh_sms \"$USER209$\""; + }; + chunk = '' + cp ${./plugins}/check_backup_age $out/check_backup_ovh_age + patchShebangs $out/check_backup_ovh_age + wrapProgram $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [ + pkgs.duplicity + ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"} + cp ${./plugins}/check_ovh_sms $out/ + patchShebangs $out/check_ovh_sms + wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [ + (pkgs.python3.withPackages (ps: [ps.ovh])) + ]} + ''; + }; + postgresql = { + commands = { + check_postgresql_replication = "${sudo} -u postgres $USER2$/check_postgres_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\""; + }; + chunk = let + postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable + then config.myServices.databasesReplication.postgresql.mainPackage + else if config.myServices.databases.enable + then config.myServices.databases.postgresql.package + else pkgs.postgresql; + in '' + cp ${./plugins}/check_postgres_replication $out/ + patchShebangs $out/check_postgres_replication + wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [ + postgresqlBinary + ]} + ''; + + sudo = myplugins: { + commands = [ + { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; } + ]; + runAs = "postgres"; + }; + }; + redis = { + commands = { + check_redis_replication = "${sudo} -u redis $USER2$/check_redis_replication \"$ARG1$\""; + }; + chunk = '' + cp ${./plugins}/check_redis_replication $out/ + patchShebangs $out/check_redis_replication + wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [ + pkgs.gnugrep pkgs.coreutils pkgs.redis + ]} + ''; + sudo = myplugins: { + commands = [ + { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; } + ]; + runAs = "redis"; + }; + }; + tcp = { + commands = { + check_tcp = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -e \"$ARG2$\" -Mcrit"; + check_tcp_ssl = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -S -D 21,15"; + }; + }; + zfs = { + commands = { + check_zfs = "$USER2$/check_zpool.sh -p ALL -w 80 -c 90"; + }; + chunk = let + zfsPlugin = pkgs.fetchurl { + url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh"; + sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf"; + }; + in '' + cp ${zfsPlugin} $out/check_zpool.sh + chmod +x $out/check_zpool.sh + patchShebangs $out/check_zpool.sh + wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [ + pkgs.which pkgs.zfs pkgs.gawk + ]} + ''; + }; +} diff --git a/modules/private/monitoring/objects_backup-2.nix b/modules/private/monitoring/objects_backup-2.nix index 4cdf59a..a930a7d 100644 --- a/modules/private/monitoring/objects_backup-2.nix +++ b/modules/private/monitoring/objects_backup-2.nix @@ -11,6 +11,7 @@ let }; in { + activatedPlugins = [ "file_date" "mysql" "openldap" "redis" "emails" ]; service = [ (emailCheck "backup-2" hostFQDN // { passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-email"; freshness_threshold = "1350"; }; diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix index 10cc149..7c9f642 100644 --- a/modules/private/monitoring/objects_common.nix +++ b/modules/private/monitoring/objects_common.nix @@ -10,9 +10,9 @@ , load5Alert ? loadAlert , load15Alert ? load5Alert , mdadm -, sudo , master , lib +, mypluginsConfig , ... }: let @@ -109,58 +109,18 @@ in ]; } ]; - command = { - check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$"; - check_emails = "$USER2$/check_emails -H $HOSTADDRESS$ -i $USER203$ -l $ARG1$ -p $ARG2$ -s $ARG3$ -f $ARG4$"; - check_emails_local = "$USER2$/check_emails -H $HOSTADDRESS$ -n $ARG1$ -r $ADMINEMAIL$ -s $ARG2$ -f $ARG3$"; - check_backup_eriomem = "$USER2$/check_eriomem $USER208$"; - check_backup_eriomem_age = "$USER2$/check_backup_eriomem_age $ARG1$"; - check_backup_ovh_age = "$USER2$/check_backup_ovh_age $ARG1$"; - check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$"; - check_ftp_database = "$USER2$/check_ftp_database"; - check_git = "$USER2$/check_git $USER203$"; - check_http = "$USER1$/check_http --sni -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; - check_https = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; - check_https_auth = "$USER1$/check_http --sni --ssl -a \"$USER202$\" -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; - check_https_certificate = "$USER1$/check_http --sni --ssl -H \"$ARG1$\" -C 21,15"; - check_https_code = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -e \"$ARG3$\" -r \"$ARG4$\""; - check_imap_connection = "$USER2$/check_imap_connection -u \"$USER204$\" -p \"$USER205$\" -H \"imap.immae.eu:143\""; + command = lib.foldr (v: o: o // (v.commands or {})) {} (builtins.attrValues mypluginsConfig) // { check_local_disk = "$USER1$/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$"; check_local_procs = "$USER1$/check_procs -w $ARG1$ -c $ARG2$ -s $ARG3$"; check_local_load = "$USER1$/check_load -r -w $ARG1$ -c $ARG2$"; check_local_swap = "$USER1$/check_swap -n ok -w $ARG1$ -c $ARG2$"; - check_local_bandwidth = "$USER2$/check_bandwidth -i=$ARG1$ -w $ARG2$ -c $ARG3$"; - check_memory = "$USER2$/check_mem.sh -w $ARG1$ -c $ARG2$"; - check_command_match = "$USER2$/check_command -c \"$ARG1$\" -C \"$ARG2$\" $ARG3$"; - check_command_output = "$USER2$/check_command -c \"$ARG1$\" -s 0 -o \"$ARG2$\" $ARG3$"; - check_command_status = "$USER2$/check_command -c \"$ARG1$\" -s \"$ARG2$\" $ARG3$"; check_ntp = "$USER1$/check_ntp_time -t 30 -q -H 0.arch.pool.ntp.org"; check_mailq = "$USER1$/check_mailq -s -w 1 -c 2"; - check_megaraid = "$USER2$/check_megaraid_sas --sudo"; - check_mysql_replication = "${sudo} -u mysql $USER2$/check_mysql_replication \"$ARG1$\" \"$ARG2$\""; - check_postgresql_replication = "${sudo} -u postgres $USER2$/check_postgres_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\""; - check_openldap_replication = "${sudo} -u openldap $USER2$/check_openldap_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\" \"$ARG4$\" \"$ARG5$\""; - check_ovh_sms = "$USER2$/check_ovh_sms \"$USER209$\""; - check_redis_replication = "${sudo} -u redis $USER2$/check_redis_replication \"$ARG1$\""; check_smtp = "$USER1$/check_smtp -H $HOSTADDRESS$ -p 25 -S -D 21,15"; - check_tcp = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -e \"$ARG2$\" -Mcrit"; - check_tcp_ssl = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -S -D 21,15"; - check_zfs = "$USER2$/check_zpool.sh -p ALL -w 80 -c 90"; check_host_alive = "$USER1$/check_ping -H $HOSTADDRESS$ -w 3000.0,80% -c 5000.0,100% -p 5"; - check_last_file_date = "${sudo} -u \"$ARG3$\" $USER2$/check_last_file_date \"$ARG1$\" \"$ARG2$\""; check_ok = "$USER1$/check_dummy 0 \"Dummy OK\""; check_critical = "$USER1$/check_dummy 2 \"Dummy CRITICAL\""; - - # $OVE is to force naemon to run via shell instead of execve which fails here - notify-host-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" HOSTSTATE=\"$HOSTSTATE$\" HOSTOUTPUT=\"$HOSTOUTPUT$\" $USER2$/notify_by_email host \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; - # $OVE is to force naemon to run via shell instead of execve which fails here - notify-service-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; - notify-by-slack = "HOST=\"$HOSTALIAS$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_slack \"$ARG1$\" \"$ARG2$\""; - notify-host-eban-url = "STATUS_NAME=\"Server\" USER=\"$USER210$\" PASSWORD=\"$USER211$\" HOSTSTATE=\"$HOSTSTATE$\" $USER2$/notify_eban_url"; - notify-service-eban-url = "STATUS_NAME=\"$_SERVICEWEBSTATUS_NAME$\" USER=\"$USER210$\" PASSWORD=\"$USER211$\" SERVICESTATE=\"$SERVICESTATE$\" $USER2$/notify_eban_url"; - - notify-master = "$USER2$/send_nrdp.sh -u \"$USER200$\" -t \"$USER201$\" -H \"$HOSTADDRESS$\" -s \"$SERVICEDESC$\" -S \"$SERVICESTATEID$\" -o \"$SERVICEOUTPUT$ | $SERVICEPERFDATA$\""; }; timeperiod = { "24x7" = { diff --git a/modules/private/monitoring/objects_dilion.nix b/modules/private/monitoring/objects_dilion.nix index ea4ec37..1baaf39 100644 --- a/modules/private/monitoring/objects_dilion.nix +++ b/modules/private/monitoring/objects_dilion.nix @@ -11,6 +11,7 @@ let }; in { + activatedPlugins = [ "zfs" ]; service = [ { passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-resources"; }; diff --git a/modules/private/monitoring/objects_eban.nix b/modules/private/monitoring/objects_eban.nix index 9ad49e1..15b19b9 100644 --- a/modules/private/monitoring/objects_eban.nix +++ b/modules/private/monitoring/objects_eban.nix @@ -12,6 +12,7 @@ let } // rest; in { + activatedPlugins = [ "http" ]; contact = { eban = { use = "generic-contact"; diff --git a/modules/private/monitoring/objects_eldiron.nix b/modules/private/monitoring/objects_eldiron.nix index 2c15dd6..75e7b0e 100644 --- a/modules/private/monitoring/objects_eldiron.nix +++ b/modules/private/monitoring/objects_eldiron.nix @@ -11,6 +11,7 @@ let }; in { + activatedPlugins = [ "emails" "postgresql" "zfs" ]; service = [ { passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; }; diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix index 6432ddb..32dbe4b 100644 --- a/modules/private/monitoring/objects_monitoring-1.nix +++ b/modules/private/monitoring/objects_monitoring-1.nix @@ -22,6 +22,7 @@ let }) profile.remotes; in { + activatedPlugins = [ "dns" "ftp" "git" "http" "imap" "ovh" "tcp" ]; host = { # Dummy host for testing # "dummy-host" = { diff --git a/modules/private/monitoring/objects_phare.nix b/modules/private/monitoring/objects_phare.nix index a61b46e..082e7e3 100644 --- a/modules/private/monitoring/objects_phare.nix +++ b/modules/private/monitoring/objects_phare.nix @@ -1,5 +1,6 @@ { emailCheck, ... }: { + activatedPlugins = [ "emails" ]; host = { "phare.normalesup.org" = { alias = "phare.normalesup.org"; diff --git a/modules/private/monitoring/objects_quatresaisons.nix b/modules/private/monitoring/objects_quatresaisons.nix index de0ce86..55d5631 100644 --- a/modules/private/monitoring/objects_quatresaisons.nix +++ b/modules/private/monitoring/objects_quatresaisons.nix @@ -11,6 +11,7 @@ let }; in { + activatedPlugins = [ "megaraid" ]; service = [ { passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-resources"; }; diff --git a/modules/private/monitoring/objects_ulminfo-fr.nix b/modules/private/monitoring/objects_ulminfo-fr.nix index 574e0e3..bd2804b 100644 --- a/modules/private/monitoring/objects_ulminfo-fr.nix +++ b/modules/private/monitoring/objects_ulminfo-fr.nix @@ -1,5 +1,6 @@ { emailCheck, ... }: { + activatedPlugins = [ "emails" ]; host = { "ulminfo.fr" = { alias = "ulminfo.fr"; diff --git a/nixops/secrets b/nixops/secrets index bbc6606..bf72e9c 160000 --- a/nixops/secrets +++ b/nixops/secrets @@ -1 +1 @@ -Subproject commit bbc6606211e970d0df974f0f74693f48186aea9c +Subproject commit bf72e9cc77b6c2217ae9e9a272805b1d917336c3