From c41d0de83b1841097c6be8d8baeed557b065657b Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Sun, 28 Mar 2021 18:38:04 +0200 Subject: [PATCH] Adjustments for monitoring --- modules/private/environment.nix | 1 + modules/private/monitoring/default.nix | 9 +++++---- modules/private/monitoring/objects_common.nix | 14 ++++++++------ modules/private/monitoring/objects_eban.nix | 10 ++++++---- .../private/monitoring/objects_monitoring-1.nix | 2 ++ modules/private/monitoring/objects_tiboqorl-fr.nix | 6 ++++-- modules/private/monitoring/plugins/notify_eban_url | 11 ++--------- nixops/secrets | 2 +- 8 files changed, 29 insertions(+), 26 deletions(-) diff --git a/modules/private/environment.nix b/modules/private/environment.nix index 91e018d..9cd591e 100644 --- a/modules/private/environment.nix +++ b/modules/private/environment.nix @@ -563,6 +563,7 @@ in description = "Eban credentials for webhook"; type = submodule { options = { + user = mkOption { type = str; description = "User"; }; password = mkOption { type = str; description = "Password"; }; }; }; diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix index 342c7f7..d593e55 100644 --- a/modules/private/monitoring/default.nix +++ b/modules/private/monitoring/default.nix @@ -109,12 +109,12 @@ let commonConfig = { dilion = { processWarn = "250"; processAlert = "400"; - loadWarn = "8.0"; loadAlert = "10.0"; + loadWarn = "1.0"; loadAlert = "1.2"; interface = "eth0"; }; eldiron = { processWarn = "250"; processAlert = "400"; - loadWarn = "8.0"; loadAlert = "10.0"; + loadWarn = "1.0"; loadAlert = "1.2"; interface = "eth0"; }; backup-2 = { @@ -130,7 +130,7 @@ let }; quatresaisons = { processWarn = "250"; processAlert = "400"; - loadWarn = "8.0"; loadAlert = "10.0"; + loadWarn = "1.0"; loadAlert = "1.2"; interface = "eth0"; }; }; @@ -347,7 +347,8 @@ in config.myEnv.monitoring.ovh_sms.consumer_key config.myEnv.monitoring.ovh_sms.account ]} - $USER210$=${config.myEnv.monitoring.eban.password} + $USER210$=${config.myEnv.monitoring.eban.user} + $USER211$=${config.myEnv.monitoring.eban.password} ''; objectDefs = toObjects commonObjects + toObjects hostObjects diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix index ee158dc..10cc149 100644 --- a/modules/private/monitoring/objects_common.nix +++ b/modules/private/monitoring/objects_common.nix @@ -3,10 +3,10 @@ , interface ? "eth0" , processWarn ? "250" , processAlert ? "400" -, loadWarn ? "8.0" +, loadWarn ? "0.9" , load5Warn ? loadWarn , load15Warn ? load5Warn -, loadAlert ? "10.0" +, loadAlert ? "1.0" , load5Alert ? loadAlert , load15Alert ? load5Alert , mdadm @@ -127,7 +127,7 @@ in check_imap_connection = "$USER2$/check_imap_connection -u \"$USER204$\" -p \"$USER205$\" -H \"imap.immae.eu:143\""; check_local_disk = "$USER1$/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$"; check_local_procs = "$USER1$/check_procs -w $ARG1$ -c $ARG2$ -s $ARG3$"; - check_local_load = "$USER1$/check_load -w $ARG1$ -c $ARG2$"; + check_local_load = "$USER1$/check_load -r -w $ARG1$ -c $ARG2$"; check_local_swap = "$USER1$/check_swap -n ok -w $ARG1$ -c $ARG2$"; check_local_bandwidth = "$USER2$/check_bandwidth -i=$ARG1$ -w $ARG2$ -c $ARG3$"; check_memory = "$USER2$/check_mem.sh -w $ARG1$ -c $ARG2$"; @@ -157,8 +157,8 @@ in # $OVE is to force naemon to run via shell instead of execve which fails here notify-service-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; notify-by-slack = "HOST=\"$HOSTALIAS$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_slack \"$ARG1$\" \"$ARG2$\""; - notify-host-eban-url = "STATUS_NAME=\"Server\" PASSWORD=\"$USER210$\" HOSTSTATE=\"$HOSTSTATE$\" $USER2$/notify_eban_url"; - notify-service-eban-url = "STATUS_NAME=\"$_SERVICEWEBSTATUS_NAME$\" PASSWORD=\"$USER210$\" SERVICESTATE=\"$SERVICESTATE$\" $USER2$/notify_eban_url"; + notify-host-eban-url = "STATUS_NAME=\"Server\" USER=\"$USER210$\" PASSWORD=\"$USER211$\" HOSTSTATE=\"$HOSTSTATE$\" $USER2$/notify_eban_url"; + notify-service-eban-url = "STATUS_NAME=\"$_SERVICEWEBSTATUS_NAME$\" USER=\"$USER210$\" PASSWORD=\"$USER211$\" SERVICESTATE=\"$SERVICESTATE$\" $USER2$/notify_eban_url"; notify-master = "$USER2$/send_nrdp.sh -u \"$USER200$\" -t \"$USER201$\" -H \"$HOSTADDRESS$\" -s \"$SERVICEDESC$\" -S \"$SERVICESTATEID$\" -o \"$SERVICEOUTPUT$ | $SERVICEPERFDATA$\""; }; @@ -187,7 +187,7 @@ in webstatus-backup = { alias = "Backup resources"; }; }; hostgroup = { - webstatus-hosts = { alias = "Hosts"; }; + webstatus-hosts = { alias = "Hosts"; }; }; contactgroup = { admins = { alias = "Naemon Administrators"; }; @@ -214,6 +214,7 @@ in retain_nonstatus_information = "1"; retain_status_information = "1"; retry_interval = "2"; + _webstatus_namespace = "immae"; }; local-service = { use = "generic-service"; @@ -285,6 +286,7 @@ in notification_interval = "120"; notification_options = "d,u,r,f"; retry_interval = "1"; + _webstatus_namespace = "immae"; }; }; }; diff --git a/modules/private/monitoring/objects_eban.nix b/modules/private/monitoring/objects_eban.nix index 659b0ec..9ad49e1 100644 --- a/modules/private/monitoring/objects_eban.nix +++ b/modules/private/monitoring/objects_eban.nix @@ -8,6 +8,7 @@ let check_interval = "15"; servicegroups = "webstatus-resources"; + _webstatus_namespace = "eban"; } // rest; in { @@ -28,6 +29,7 @@ in contact_groups = "null"; _webstatus_name = "Eban"; _webstatus_vhost = "status.eban.bzh"; + _webstatus_namespace = "eban"; }; }; service = [ @@ -50,11 +52,11 @@ in _webstatus_url = "https://git.eban.bzh/"; }) (serviceTemplate { - service_description = "Eban Cloud is up and running"; - check_command = ["check_https" "cloud.eban.bzh" "/" ""]; + service_description = "Eban Bitwarden is up and running"; + check_command = ["check_https" "bitwarden.eban.bzh" "/" "<title"]; - _webstatus_name = "Cloud"; - _webstatus_url = "https://cloud.eban.bzh/"; + _webstatus_name = "Bitwarden"; + _webstatus_url = "https://bitwarden.eban.bzh/"; }) ]; } diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix index 9503da2..6432ddb 100644 --- a/modules/private/monitoring/objects_monitoring-1.nix +++ b/modules/private/monitoring/objects_monitoring-1.nix @@ -695,6 +695,7 @@ in use = "external-web-service"; contact_groups = "tiboqorl"; check_command = [ "check_https" "sc.groupe-cyllene.com" "/" "<title>Support Client Cyllene" ]; + _webstatus_namespace = "tiboqorl"; } { @@ -703,6 +704,7 @@ in use = "external-web-service"; contact_groups = "tiboqorl"; check_command = [ "check_https" "origny.tiboqorl.fr" "/" "<title>Home Assistant" ]; + _webstatus_namespace = "tiboqorl"; } ] ++ to_resource to_backup_age nodes.eldiron.config.services.duplyBackup; contact = { diff --git a/modules/private/monitoring/objects_tiboqorl-fr.nix b/modules/private/monitoring/objects_tiboqorl-fr.nix index 77403ba..3657c19 100644 --- a/modules/private/monitoring/objects_tiboqorl-fr.nix +++ b/modules/private/monitoring/objects_tiboqorl-fr.nix @@ -10,6 +10,7 @@ let notification_interval = "240"; retry_interval = "1"; servicegroups = "webstatus-resources"; + _webstatus_namespace = "tiboqorl"; host_name = hostFQDN; }; in @@ -23,6 +24,7 @@ in hostgroups = "webstatus-hosts"; _webstatus_name = "tiboqorl"; _webstatus_vhost = "status.immae.eu"; + _webstatus_namespace = "tiboqorl"; }; }; service = [ @@ -58,8 +60,8 @@ in use = "local-service"; check_command = [ "check_local_load" - "3.0,3.0,3.0" # warn - "4.0,4.0,4.0" # alert + "0.75,0.75,0.75" # warn + "1.0,1.0,1.0" # alert ]; } { diff --git a/modules/private/monitoring/plugins/notify_eban_url b/modules/private/monitoring/plugins/notify_eban_url index 431e320..107b5de 100755 --- a/modules/private/monitoring/plugins/notify_eban_url +++ b/modules/private/monitoring/plugins/notify_eban_url @@ -1,13 +1,6 @@ #!/usr/bin/env bash -PASS=$(echo "$PASSWORD" | base64 -d) - if [ "$SERVICESTATE" = "CRITICAL" -o "$SERVICESTATE" = "UNKNOWN" -o "$HOSTSTATE" = "DOWN" -o "$HOSTSTATE" = "UNREACHABLE" ]; then - action=downAlert -elif [ "$SERVICESTATE" = "OK" -o "$HOSTSTATE" = "UP" ]; then - action=upAlert -fi - -if [ -n "$action" ]; then - curl -X GET -G --data-urlencode "service=$STATUS_NAME" --data-urlencode "mdp=$PASS" https://infra.eban.bzh/$action + MESSAGE="$STATUS_NAME seems down!" + curl -X GET -G --data-urlencode "user=$USER" --data-urlencode "pass=$PASSWORD" --data-urlencode "msg=$MESSAGE" https://smsapi.free-mobile.fr/sendmsg fi diff --git a/nixops/secrets b/nixops/secrets index 65b942b..e816753 160000 --- a/nixops/secrets +++ b/nixops/secrets @@ -1 +1 @@ -Subproject commit 65b942ba13daa40fc45fa34d58676c213b5bad87 +Subproject commit e816753f9ffc851a6f1e35cfa267a03e2a0ad2aa -- 2.41.0