From e820134d38c3b7470ea5112f40a6dc967f039878 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Sun, 5 Jan 2020 17:08:32 +0100 Subject: [PATCH] Add monitoring host --- Makefile | 3 +- modules/private/environment.nix | 14 + modules/private/monitoring/default.nix | 99 ++- modules/private/monitoring/nagios-cli.cfg | 68 ++ .../private/monitoring/objects_backup-2.nix | 24 +- .../private/monitoring/objects_caldance-1.nix | 57 ++ modules/private/monitoring/objects_common.nix | 132 +++- .../private/monitoring/objects_eldiron.nix | 15 +- .../private/monitoring/objects_immae-eu.nix | 123 ++++ modules/private/monitoring/objects_master.nix | 38 + .../monitoring/objects_monitoring-1.nix | 665 ++++++++++++++++++ modules/private/monitoring/objects_phare.nix | 22 + .../monitoring/objects_tiboqorl-fr.nix | 12 + .../private/monitoring/objects_ulminfo-fr.nix | 22 + .../monitoring/plugins/check_ftp_database | 11 + modules/private/monitoring/plugins/check_git | 68 ++ .../monitoring/plugins/check_imap_connection | 52 ++ .../monitoring/plugins/check_maison_bbc | 41 ++ .../monitoring/plugins/notify_by_email | 4 +- .../monitoring/plugins/notify_by_slack | 46 ++ .../plugins/notify_maison_bbc_by_email | 27 + modules/private/monitoring/to_objects.nix | 4 +- modules/private/system/monitoring-1.nix | 41 ++ nixops/Makefile | 7 + nixops/default.nix | 1 + 25 files changed, 1557 insertions(+), 39 deletions(-) create mode 100644 modules/private/monitoring/nagios-cli.cfg create mode 100644 modules/private/monitoring/objects_caldance-1.nix create mode 100644 modules/private/monitoring/objects_immae-eu.nix create mode 100644 modules/private/monitoring/objects_master.nix create mode 100644 modules/private/monitoring/objects_monitoring-1.nix create mode 100644 modules/private/monitoring/objects_phare.nix create mode 100644 modules/private/monitoring/objects_tiboqorl-fr.nix create mode 100644 modules/private/monitoring/objects_ulminfo-fr.nix create mode 100755 modules/private/monitoring/plugins/check_ftp_database create mode 100755 modules/private/monitoring/plugins/check_git create mode 100755 modules/private/monitoring/plugins/check_imap_connection create mode 100755 modules/private/monitoring/plugins/check_maison_bbc create mode 100755 modules/private/monitoring/plugins/notify_by_slack create mode 100755 modules/private/monitoring/plugins/notify_maison_bbc_by_email create mode 100644 modules/private/system/monitoring-1.nix diff --git a/Makefile b/Makefile index 6a039bd..fafbe00 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ subrecipes = setup -subrecipes += nixops ssh-eldiron ssh-backup-2 info debug dry-run build upload deploy deploy-reboot reboot +subrecipes += nixops ssh-eldiron ssh-backup-2 ssh-monitoring-1 +subrecipes += info debug dry-run build upload deploy deploy-reboot reboot subrecipes += list-generations delete-generations cleanup subrecipes += pull pull_environment pull_deployment deployment_is_set push push_deployment push_environment ${subrecipes}: diff --git a/modules/private/environment.nix b/modules/private/environment.nix index 9bfb80a..550b060 100644 --- a/modules/private/environment.nix +++ b/modules/private/environment.nix @@ -117,6 +117,11 @@ in default = {}; type = attrsOf (submodule { options = { + emails = mkOption { + default = []; + description = "List of e-mails that the server can be a sender of"; + type = listOf str; + }; ldap = mkOption { description = '' LDAP credentials for the host @@ -453,7 +458,16 @@ in options = { status_url = mkOption { type = str; description = "URL to push status to"; }; status_token = mkOption { type = str; description = "Token for the status url"; }; + http_user_password = mkOption { type = str; description = "HTTP credentials to check services behind wall"; }; email = mkOption { type = str; description = "Admin E-mail"; }; + ssh_public_key = mkOption { type = str; description = "SSH public key"; }; + ssh_secret_key = mkOption { type = str; description = "SSH secret key"; }; + imap_login = mkOption { type = str; description = "IMAP login"; }; + imap_password = mkOption { type = str; description = "IMAP password"; }; + nrdp_tokens = mkOption { type = listOf str; description = "Tokens allowed to push status update"; }; + slack_url = mkOption { type = str; description = "Slack webhook url to push status update"; }; + slack_channel = mkOption { type = str; description = "Slack channel to push status update"; }; + contacts = mkOption { type = attrsOf unspecified; description = "Contact dicts to fill naemon objects"; }; }; }; }; diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix index d6c91ac..a1f4b3f 100644 --- a/modules/private/monitoring/default.nix +++ b/modules/private/monitoring/default.nix @@ -1,5 +1,6 @@ -{ config, pkgs, lib, name, ... }: +{ config, pkgs, lib, name, nodes, ... }: let + cfg = config.myServices.monitoring; myplugins = pkgs.runCommand "buildplugins" { buildInputs = [ pkgs.makeWrapper pkgs.perl ]; } '' @@ -10,6 +11,9 @@ let wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [ pkgs.curl pkgs.jq ]} + wrapProgram $out/check_maison_bbc --prefix PATH : ${lib.makeBinPath [ + pkgs.curl pkgs.jq + ]} wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [ pkgs.gnugrep pkgs.gawk pkgs.procps-ng ]} @@ -25,6 +29,24 @@ let wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [ pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap ]} + wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [ + pkgs.lftp + ]} + wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [ + pkgs.git pkgs.openssh + ]} + wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [ + pkgs.openssl + ]} + wrapProgram $out/notify_maison_bbc_by_email --prefix PATH : ${lib.makeBinPath [ + pkgs.mailutils pkgs.gawk + ]} + wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [ + pkgs.mailutils + ]} + wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [ + pkgs.curl pkgs.jq + ]} ''; toObjects = pkgs.callPackage ./to_objects.nix {}; commonConfig = { @@ -36,16 +58,48 @@ let processWarn = "50"; processAlert = "60"; loadWarn = "1.0"; loadAlert = "2.0"; }; + monitoring-1 = { + processWarn = "50"; processAlert = "60"; + loadWarn = "1.0"; loadAlert = "2.0"; + }; }; + masterPassiveObjects = let + otherPassiveObjects = map + (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") {})) + [ "caldance-1" "ulminfo-fr" "immae-eu" "phare" "tiboqorl-fr" ]; + otherPassiveServices = lib.flatten (map (h: h.service or []) otherPassiveObjects); + otherPassiveHosts = (map (h: h.host)) otherPassiveObjects; + passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron"]) nodes; + toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo); + passiveServices = lib.flatten (lib.attrsets.mapAttrsToList + (_: n: toPassiveServices n.config.myServices.monitoring.services) + passiveNodes + ); + in { + service = passiveServices ++ otherPassiveServices; + host = lib.lists.foldr + (a: b: a//b) + {} + (otherPassiveHosts ++ lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes); + }; + masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; }; commonObjects = pkgs.callPackage ./objects_common.nix ({ + master = cfg.master; hostFQDN = config.hostEnv.FQDN; + hostName = name; sudo = "/run/wrappers/bin/sudo"; } // builtins.getAttr name commonConfig); hostObjects = let specific_file = ./. + "/objects_" + name + ".nix"; in - lib.attrsets.optionalAttrs (builtins.pathExists specific_file) (pkgs.callPackage specific_file { inherit config; }); + lib.attrsets.optionalAttrs + (builtins.pathExists specific_file) + (pkgs.callPackage specific_file { + inherit config; + hostFQDN = config.hostEnv.FQDN; + hostName = name; + }); in { options = { @@ -57,10 +111,27 @@ in Whether to enable monitoring. ''; }; + master = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + This instance is the master instance + ''; + }; + hosts = lib.mkOption { + readOnly = true; + description = "Hosts list for this host"; + default = (commonObjects.host or {}) // (hostObjects.host or {}); + }; + services = lib.mkOption { + readOnly = true; + description = "Services list for this host"; + default = commonObjects.service ++ hostObjects.service; + }; }; }; - config = lib.mkIf config.myServices.monitoring.enable { + config = lib.mkIf cfg.enable { services.duplyBackup.profiles.monitoring = { rootDir = config.services.naemon.varDir; }; @@ -116,6 +187,15 @@ in text = "MAILADDR ${config.myEnv.monitoring.email}"; }; + secrets.keys = [ + { + dest = "naemon/id_rsa"; + user = "naemon"; + group = "naemon"; + premissions = "0400"; + text = config.myEnv.monitoring.ssh_secret_key; + } + ]; # needed since extraResource is not in the closure systemd.services.naemon.path = [ myplugins ]; services.naemon = { @@ -126,7 +206,7 @@ in log_initial_states=1 date_format=iso8601 admin_email=${config.myEnv.monitoring.email} - + '' + lib.optionalString (!cfg.master) '' obsess_over_services=1 ocsp_command=notify-master ''; @@ -134,8 +214,17 @@ in $USER2$=${myplugins} $USER200$=${config.myEnv.monitoring.status_url} $USER201$=${config.myEnv.monitoring.status_token} + $USER202$=${config.myEnv.monitoring.http_user_password} + $USER203$=${config.secrets.fullPaths."naemon/id_rsa"} + $USER204$=${config.myEnv.monitoring.imap_login} + $USER205$=${config.myEnv.monitoring.imap_password} + $USER206$=${config.myEnv.monitoring.slack_channel} + $USER207$=${config.myEnv.monitoring.slack_url} ''; - objectDefs = toObjects commonObjects + toObjects hostObjects; + objectDefs = toObjects commonObjects + + toObjects hostObjects + + lib.optionalString cfg.master (toObjects masterObjects) + + lib.optionalString cfg.master (toObjects masterPassiveObjects); }; }; } diff --git a/modules/private/monitoring/nagios-cli.cfg b/modules/private/monitoring/nagios-cli.cfg new file mode 100644 index 0000000..7bd30cb --- /dev/null +++ b/modules/private/monitoring/nagios-cli.cfg @@ -0,0 +1,68 @@ +# -*- coding: utf-8; -*- + +[cli] +history = /var/lib/naemon/nagios_cli_history + +[ui] +color = 1 +prompt = naemon %s> +prompt_separator = " → " + +[nagios] +log = /var/log/naemon +command_file = /run/naemon/naemon.cmd +log_file = %(log)s/naemon.log +object_cache_file = /var/lib/naemon/objects.cache +status_file = /var/lib/naemon/status.dat + +[object] +host.status = + host_name + current_state + plugin_output + is_flapping + last_check + last_time_down + last_state_change + check_period + notification_period + current_attempt + max_attempts +service.status = + host_name + service_description + current_state + is_flapping + plugin_output + last_time_down + last_state_change + last_check + next_check + check_interval + check_latency + check_period + notification_period + current_attempt + max_attempts + +[string] +level.ok = ↑ OK +level.warning = ! WARNING +level.critical = ↓ CRITICAL +level.unknown = ↕ UNKNOWN + +[color] +error = bold_red + +prompt = normal +prompt.object = bold + +host.host_name = bold +host.plugin_output = bold +service.plugin_output = bold + +level.ok = bold_green +level.warning = bold_yellow +level.critical = bold_red +level.unknown = bold_magenta + diff --git a/modules/private/monitoring/objects_backup-2.nix b/modules/private/monitoring/objects_backup-2.nix index 38b2ff6..52289dd 100644 --- a/modules/private/monitoring/objects_backup-2.nix +++ b/modules/private/monitoring/objects_backup-2.nix @@ -1,52 +1,73 @@ -{ config, pkgs, ... }: +{ config, pkgs, lib, hostFQDN, ... }: +let + defaultPassiveInfo = { + filter = lib.attrsets.filterAttrs + (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_"); + use = "external-passive-service"; + freshness_threshold = "450"; + retry_interval = "1"; + servicegroups = "webstatus-resources"; + host_name = hostFQDN; + }; +in { service = [ { + passiveInfo = defaultPassiveInfo; service_description = "Size on /backup2 partition"; use = "local-service"; check_command = ["check_local_disk" "10%" "5%" "/backup2"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-backup"; }; service_description = "Last backup in /backup2/phare is not too old"; use = "local-service"; check_command = ["check_last_file_date" "/backup2/phare" "14" "backup"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-backup"; }; service_description = "Last backup in /backup2/immae_eu is not too old"; use = "local-service"; check_command = ["check_last_file_date" "/backup2/immae_eu" "14" "backup"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-backup"; }; service_description = "Last backup in /backup2/ulminfo is not too old"; use = "local-service"; check_command = ["check_last_file_date" "/backup2/ulminfo" "14" "backup"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases,webstatus-backup"; }; service_description = "Last postgresql dump in /backup2/eldiron/postgresql_backup is not too old"; use = "local-service"; check_command = ["check_last_file_date" "/backup2/eldiron/postgresql_backup" "7" "postgres"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; }; service_description = "Redis replication for eldiron is up to date"; use = "local-service"; check_command = ["check_redis_replication" "/run/redis_eldiron/redis.sock"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases,webstatus-backup"; }; service_description = "Last redis dump in /backup2/eldiron/redis_backup is not too old"; use = "local-service"; check_command = ["check_last_file_date" "/backup2/eldiron/redis_backup" "7" "redis"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; }; service_description = "Mysql replication for eldiron is up to date"; use = "local-service"; check_command = ["check_mysql_replication" "/run/mysqld_eldiron/mysqld.sock" "/var/secrets/mysql_replication/eldiron/client"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases,webstatus-backup"; }; service_description = "Last mysql dump in /backup2/eldiron/mysql_backup is not too old"; use = "local-service"; check_command = ["check_last_file_date" "/backup2/eldiron/mysql_backup" "7" "mysql"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; }; service_description = "Openldap replication for eldiron is up to date"; use = "local-service"; check_command = let @@ -73,6 +94,7 @@ ]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases,webstatus-backup"; }; service_description = "Last openldap dump in /backup2/eldiron/openldap_backup is not too old"; use = "local-service"; check_command = ["check_last_file_date" "/backup2/eldiron/openldap_backup" "7" "openldap"]; diff --git a/modules/private/monitoring/objects_caldance-1.nix b/modules/private/monitoring/objects_caldance-1.nix new file mode 100644 index 0000000..2879124 --- /dev/null +++ b/modules/private/monitoring/objects_caldance-1.nix @@ -0,0 +1,57 @@ +{ ... }: +let + defaultPassiveInfo = { + use = "external-passive-service"; + freshness_threshold = "450"; + retry_interval = "1"; + host_name = "caldance-1.v.immae.eu"; + }; +in +{ + host = { + "caldance-1.v.immae.eu" = { + alias = "caldance-1.v.immae.eu"; + address = "caldance-1.v.immae.eu"; + use = "linux-server"; + hostgroups = "webstatus-hosts"; + _webstatus_name = "caldance-1"; + }; + }; + service = map (s: defaultPassiveInfo // s) [ + { + service_description = "Size on root partition"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "Total number of process"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "Average load"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "Swap usage"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "fail2ban is active"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "NTP is activated and working"; + servicegroups = "webstatus-resources"; + freshness_threshold = "5400"; + } + + { + service_description = "Databases are present in postgresql"; + servicegroups = "webstatus-databases"; + } + ]; +} diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix index 66fb812..7467306 100644 --- a/modules/private/monitoring/objects_common.nix +++ b/modules/private/monitoring/objects_common.nix @@ -1,27 +1,45 @@ { hostFQDN +, hostName , processWarn ? "250" , processAlert ? "400" , loadWarn ? "8.0" , loadAlert ? "10.0" , mdadm , sudo +, master +, lib , ... }: +let + defaultPassiveInfo = { + filter = lib.attrsets.filterAttrs + (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_"); + use = "external-passive-service"; + freshness_threshold = "450"; + retry_interval = "1"; + servicegroups = "webstatus-resources"; + host_name = hostFQDN; + }; +in { host = { "${hostFQDN}" = { alias = hostFQDN; address = hostFQDN; use = "linux-server"; + hostgroups = "webstatus-hosts"; + _webstatus_name = hostName; }; }; service = [ { + passiveInfo = defaultPassiveInfo; service_description = "Size on root partition"; use = "local-service"; check_command = ["check_local_disk" "20%" "10%" "/"]; } { + passiveInfo = defaultPassiveInfo; service_description = "Total number of process"; use = "local-service"; check_command = [ @@ -32,6 +50,7 @@ ]; } { + passiveInfo = defaultPassiveInfo; service_description = "Average load"; use = "local-service"; check_command = [ @@ -41,21 +60,25 @@ ]; } { + passiveInfo = defaultPassiveInfo; service_description = "Swap usage"; use = "local-service"; check_command = ["check_local_swap" "20" "10"]; } { + passiveInfo = defaultPassiveInfo; service_description = "Memory usage"; use = "local-service"; check_command = ["check_memory" "80" "90"]; } { + passiveInfo = defaultPassiveInfo; service_description = "NTP is activated and working"; use = "local-service"; check_command = ["check_ntp"]; } { + passiveInfo = defaultPassiveInfo; service_description = "No mdadm array is degraded"; use = "local-service"; check_command = [ @@ -67,28 +90,46 @@ } ]; command = { + check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$"; + check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$"; + check_ftp_database = "$USER2$/check_ftp_database"; + check_git = "$USER2$/check_git $USER203$"; + check_http = "$USER1$/check_http --sni -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; + check_https = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; + check_https_auth = "$USER1$/check_http --sni --ssl -a \"$USER202$\" -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\""; + check_https_certificate = "$USER1$/check_http --sni --ssl -H \"$ARG1$\" -C 21,15"; + check_https_code = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -e \"$ARG3$\" -r \"$ARG4$\""; + check_imap_connection = "$USER2$/check_imap_connection -u \"$USER204$\" -p \"$USER205$\" -H \"imap.immae.eu:143\""; check_local_disk = "$USER1$/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$"; check_local_procs = "$USER1$/check_procs -w $ARG1$ -c $ARG2$ -s $ARG3$"; check_local_load = "$USER1$/check_load -w $ARG1$ -c $ARG2$"; check_local_swap = "$USER1$/check_swap -n ok -w $ARG1$ -c $ARG2$"; check_memory = "$USER2$/check_mem.sh -w $ARG1$ -c $ARG2$"; + check_command_match = "$USER2$/check_command -c \"$ARG1$\" -C \"$ARG2$\" $ARG3$"; check_command_output = "$USER2$/check_command -c \"$ARG1$\" -s 0 -o \"$ARG2$\" $ARG3$"; + check_command_status = "$USER2$/check_command -c \"$ARG1$\" -s \"$ARG2$\" $ARG3$"; check_ntp = "$USER1$/check_ntp_time -t 30 -q -H 0.arch.pool.ntp.org"; + check_mailq = "$USER1$/check_mailq -s -w 1 -c 2"; + check_maison_bbc = "$USER2$/check_maison_bbc"; check_mysql_replication = "${sudo} -u mysql $USER2$/check_mysql_replication \"$ARG1$\" \"$ARG2$\""; check_postgresql_replication = "${sudo} -u postgres $USER2$/check_postgres_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\""; check_openldap_replication = "${sudo} -u openldap $USER2$/check_openldap_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\" \"$ARG4$\" \"$ARG5$\""; check_redis_replication = "${sudo} -u redis $USER2$/check_redis_replication \"$ARG1$\""; - check_mailq = "$USER1$/check_mailq -s -w 1 -c 2"; + check_smtp = "$USER1$/check_smtp -H $HOSTADDRESS$ -p 25 -S -D 21,15"; + check_tcp = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -e \"$ARG2$\" -Mcrit"; + check_tcp_ssl = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -S -D 21,15"; check_host_alive = "$USER1$/check_ping -H $HOSTADDRESS$ -w 3000.0,80% -c 5000.0,100% -p 5"; check_last_file_date = "${sudo} -u \"$ARG3$\" $USER2$/check_last_file_date \"$ARG1$\" \"$ARG2$\""; + check_ok = "$USER1$/check_dummy 0 \"Dummy OK\""; + check_critical = "$USER1$/check_dummy 2 \"Dummy CRITICAL\""; - # No notify commands, we go through master - # notify_host_by_email = "SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" HOSTSTATE=\"$HOSTSTATE$\" HOSTOUTPUT=\"$HOSTOUTPUT$\" $USER2$/notify_by_email host \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; - # #$OVE is to force naemon to run via shell instead of execve which fails here - # notify_service_by_email = "SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; - # #sudo /usr/bin/strace -o /tmp/foo -vf -s 256 -u naemon $USER2$/notify_by_email - # #$OVE is to force naemon to run via shell instead of execve which fails here + # $OVE is to force naemon to run via shell instead of execve which fails here + notify-host-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" HOSTSTATE=\"$HOSTSTATE$\" HOSTOUTPUT=\"$HOSTOUTPUT$\" $USER2$/notify_by_email host \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; + # $OVE is to force naemon to run via shell instead of execve which fails here + notify-service-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; + notify-maison-bbc-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_maison_bbc_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE"; + notify-by-slack = "HOST=\"$HOSTALIAS$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_slack \"$ARG1$\" \"$ARG2$\""; notify-master = "$USER2$/send_nrdp.sh -u \"$USER200$\" -t \"$USER201$\" -H \"$HOSTADDRESS$\" -s \"$SERVICEDESC$\" -S \"$SERVICESTATEID$\" -o \"$SERVICEOUTPUT$\""; }; @@ -104,17 +145,24 @@ sunday = "00:00-24:00"; }; }; + servicegroup = { + webstatus-webapps = { alias = "Web applications"; }; + webstatus-websites = { alias = "Personal websites"; }; + webstatus-ssl = { alias = "SSL certificates"; }; + webstatus-dns = { alias = "DNS resolution"; }; + webstatus-remote-services = { alias = "Other remote services"; }; + webstatus-local-services = { alias = "Other local services"; }; + webstatus-email = { alias = "E-mail services"; }; + webstatus-resources = { alias = "Local resources"; }; + webstatus-databases = { alias = "Databases resources"; }; + webstatus-backup = { alias = "Backup resources"; }; + }; + hostgroup = { + webstatus-hosts = { alias = "Hosts"; }; + }; contactgroup = { admins = { alias = "Naemon Administrators"; }; }; - # No contact, we go through master - # contact = { - # immae = { - # alias = "Immae"; - # use = "generic-contact"; - # email = "xxxxxxxxxxxxxxxx"; - # }; - # }; templates = { service = { generic-service = { @@ -130,7 +178,7 @@ notification_interval = "60"; notification_options = "w,u,c,r,f,s"; notification_period = "24x7"; - notifications_enabled = "0"; # no notification since we send them to master + notifications_enabled = if master then "1" else "0"; obsess_over_service = "1"; passive_checks_enabled = "1"; process_perf_data = "1"; @@ -144,19 +192,51 @@ check_interval = "5"; max_check_attempts = "4"; retry_interval = "1"; + servicegroups = "webstatus-resources"; + }; + external-service = { + use = "generic-service"; + check_interval = "5"; + max_check_attempts = "4"; + retry_interval = "1"; + }; + web-service = { + use = "generic-service"; + check_interval = "20"; + max_check_attempts = "2"; + retry_interval = "1"; + }; + external-web-service = { + use = "generic-service"; + check_interval = "20"; + max_check_attempts = "2"; + retry_interval = "1"; + }; + mail-service = { + use = "generic-service"; + check_interval = "15"; + max_check_attempts = "1"; + retry_interval = "1"; + }; + dns-service = { + use = "generic-service"; + check_interval = "120"; + notification_interval = "120"; + max_check_attempts = "5"; + retry_interval = "5"; }; }; # No contact, we go through master - # contact = { - # generic-contact = { - # host_notification_commands = "notify_host_by_email"; - # host_notification_options = "d,u,r,f,s"; - # host_notification_period = "24x7"; - # service_notification_commands = "notify_service_by_email"; - # service_notification_options = "w,u,c,r,f,s"; - # service_notification_period = "24x7"; - # }; - # }; + contact = { + generic-contact = { + host_notification_commands = "notify-host-by-email"; + host_notification_options = "d,u,r,f,s"; + host_notification_period = "24x7"; + service_notification_commands = "notify-service-by-email"; + service_notification_options = "w,u,c,r,f,s"; + service_notification_period = "24x7"; + }; + }; host = { generic-host = { event_handler_enabled = "1"; diff --git a/modules/private/monitoring/objects_eldiron.nix b/modules/private/monitoring/objects_eldiron.nix index 897fc15..bee4645 100644 --- a/modules/private/monitoring/objects_eldiron.nix +++ b/modules/private/monitoring/objects_eldiron.nix @@ -1,12 +1,25 @@ -{ ... }: +{ lib, hostFQDN, ... }: +let + defaultPassiveInfo = { + filter = lib.attrsets.filterAttrs + (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_"); + use = "external-passive-service"; + freshness_threshold = "450"; + retry_interval = "1"; + servicegroups = "webstatus-resources"; + host_name = hostFQDN; + }; +in { service = [ { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; }; service_description = "Postgresql replication for backup-2 is up to date"; use = "local-service"; check_command = ["check_postgresql_replication" "backup-2" "/run/postgresql" "5432"]; } { + passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-email"; }; service_description = "mailq is empty"; use = "local-service"; check_command = ["check_mailq"]; diff --git a/modules/private/monitoring/objects_immae-eu.nix b/modules/private/monitoring/objects_immae-eu.nix new file mode 100644 index 0000000..3dda999 --- /dev/null +++ b/modules/private/monitoring/objects_immae-eu.nix @@ -0,0 +1,123 @@ +{ ... }: +let + defaultPassiveInfo = { + use = "external-passive-service"; + freshness_threshold = "450"; + retry_interval = "1"; + host_name = "immae.eu"; + }; +in +{ + host = { + "immae.eu" = { + alias = "immae.eu"; + address = "immae.eu"; + use = "linux-server"; + hostgroups = "webstatus-hosts"; + _webstatus_name = "immaeEu"; + }; + }; + service = map (s: defaultPassiveInfo // s) [ + { + service_description = "size on root partition"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "total number of process"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "average load"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "swap usage"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "no mdadm array is degraded"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "ntp is activated and working"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "ssh connexion responds"; + } + + { + service_description = "fail2ban is active"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "pacman packages are not too old"; + servicegroups = "webstatus-resources"; + freshness_threshold = "10800"; + notification_interval = "0"; + } + + { + service_description = "rsync backup happened not too long ago"; + servicegroups = "webstatus-backup"; + } + + { + service_description = "eriomem backup is up and not full"; + freshness_threshold = "10800"; + notification_interval = "120"; + servicegroups = "webstatus-backup"; + } + + { + service_description = "postfix service is active"; + } + + { + service_description = "tinc@Immae service is active"; + } + + { + service_description = "sshd service is active"; + } + + { + service_description = "httpd service is active"; + servicegroups = "webstatus-resources"; + } + + { + service_description = "postfix SSL is up to date"; + } + + { + service_description = "mailq is empty"; + servicegroups = "webstatus-email"; + } + + ## Sending e-mail + { + service_description = "immae.eu email service is active"; + servicegroups = "webstatus-email"; + freshness_threshold = "1350"; + } + + #### Web scenarios + { + service_description = "discourse website is running on discourse.capitaines.fr"; + freshness_threshold = "1800"; + } + + { + service_description = "Default website site is running on ns208507.ip-188-165-209.eu"; + freshness_threshold = "1800"; + } + ]; +} diff --git a/modules/private/monitoring/objects_master.nix b/modules/private/monitoring/objects_master.nix new file mode 100644 index 0000000..5b5b15f --- /dev/null +++ b/modules/private/monitoring/objects_master.nix @@ -0,0 +1,38 @@ +{ config, ... }: +{ + contact = { + immae = config.myEnv.monitoring.contacts.immae // { + use = "generic-contact"; + contactgroups = "admins"; + host_notification_commands = "notify-host-by-email,notify-by-slack!$USER206$!$USER207$"; + service_notification_commands = "notify-service-by-email,notify-by-slack!$USER206$!$USER207$"; + }; + }; + command = { + check_passive = "$USER1$/check_dummy 3 \"Service result are stale\""; + }; + templates = { + service = { + external-passive-service = { + active_checks_enabled = "0"; + check_freshness = "1"; + check_period = "24x7"; + contact_groups = "admins"; + event_handler_enabled = "1"; + flap_detection_enabled = "1"; + is_volatile = "0"; + max_check_attempts = "3"; + notification_interval = "60"; + notification_options = "w,u,c,r,f,s"; + notification_period = "24x7"; + notifications_enabled = "1"; + passive_checks_enabled = "1"; + process_perf_data = "1"; + retain_nonstatus_information = "1"; + retain_status_information = "1"; + retry_interval = "2"; + check_command = "check_passive"; + }; + }; + }; +} diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix new file mode 100644 index 0000000..a46b684 --- /dev/null +++ b/modules/private/monitoring/objects_monitoring-1.nix @@ -0,0 +1,665 @@ +{ config, pkgs, ... }: +{ + host = { + # Dummy host for testing + # "dummy-host" = { + # alias = "dummy.host"; + # address = "dummy.host"; + # use = "linux-server"; + # check_command = "check_ok"; + # }; + }; + service = [ + # Dummy service for testing + # { + # service_description = "Dummy failing test"; + # host_name = "dummy-host"; + # use = "local-service"; + # check_interval = "0.3"; + # max_check_attempts = "1"; + # flap_detection_enabled = "0"; + # notification_interval = "0.1"; + # check_command = "check_critical"; + # } + + { + service_description = "ftp has access to database for authentication"; + host_name = "eldiron.immae.eu"; + use = "external-service"; + check_command = "check_ftp_database"; + + servicegroups = "webstatus-remote-services"; + _webstatus_name = "FTP"; + _webstatus_url = "ftp.immae.eu"; + } + + { + service_description = "gitolite is working"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = "check_git"; + + servicegroups = "webstatus-remote-services"; + _webstatus_name = "Git"; + _webstatus_url = "git.immae.eu"; + } + + { + service_description = "postfix SSL is up to date"; + host_name = "eldiron.immae.eu"; + use = "external-service"; + check_command = "check_smtp"; + + servicegroups = "webstatus-ssl"; + _webstatus_name = "SMTP"; + _webstatus_url = "smtp.immae.eu"; + } + + { + service_description = "imap SSL is up to date"; + host_name = "eldiron.immae.eu"; + use = "external-service"; + check_command = ["check_tcp_ssl" "993"]; + + servicegroups = "webstatus-ssl"; + _webstatus_name = "IMAP"; + _webstatus_url = "imap.immae.eu"; + } + + { + service_description = "imap connection works"; + host_name = "eldiron.immae.eu"; + use = "external-service"; + check_command = "check_imap_connection"; + + servicegroups = "webstatus-remote-services,webstatus-email"; + _webstatus_name = "IMAP"; + _webstatus_url = "imap.immae.eu"; + } + + # DNS services + { + service_description = "eldiron dns is active and authoritative for aten.pro"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "aten.pro" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "aten.pro"; + } + + { + service_description = "eldiron dns is active and authoritative for bouya.org"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "bouya.org" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "bouya.org"; + } + + { + service_description = "eldiron dns is active and authoritative for filedesreves.fr"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "filedesreves.fr" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "filedesreves.fr"; + } + + { + service_description = "eldiron dns is active and authoritative for ludivinecassal.com"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "ludivinecassal.com" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "ludivinecassal.com"; + } + + { + service_description = "eldiron dns is active and authoritative for icommandeur.org"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "icommandeur.org" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "icommandeur.org"; + } + + { + service_description = "eldiron dns is active and authoritative for immae.eu"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "immae.eu" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "immae.eu"; + } + + { + service_description = "eldiron dns is active and authoritative for immae.fr"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "immae.fr" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "immae.fr"; + } + + { + service_description = "eldiron dns is active and authoritative for piedsjaloux.fr"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "piedsjaloux.fr" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "piedsjaloux.fr"; + } + + { + service_description = "eldiron dns is active and authoritative for saison-photo.org"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "saison-photo.org" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "saison-photo.org"; + } + + { + service_description = "eldiron dns is active and authoritative for tellesflorian.com"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "tellesflorian.com" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "tellesflorian.com"; + } + + { + service_description = "eldiron dns is active and authoritative for capitaines.fr"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_dns" "capitaines.fr" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "capitaines.fr"; + } + + { + service_description = "cloudns dns is active and authoritative for aten.pro"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "aten.pro" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "aten.pro (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for bouya.org"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "bouya.org" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "bouya.org (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for filedesreves.fr"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "filedesreves.fr" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "filedesreves.fr (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for ludivinecassal.com"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "ludivinecassal.com" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "ludivinecassal.com (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for icommandeur.org"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "icommandeur.org" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "icommandeur.org (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for immae.eu"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "immae.eu" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "immae.eu (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for immae.fr"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "immae.fr" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "immae.fr (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for piedsjaloux.fr"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "piedsjaloux.fr" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "piedsjaloux.fr (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for tellesflorian.com"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "tellesflorian.com" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "tellesflorian.com (Secondary DNS ClouDNS)"; + } + + { + service_description = "cloudns dns is active and authoritative for saison-photo.org"; + host_name = "eldiron.immae.eu"; + use = "dns-service"; + check_command = ["check_external_dns" "pns1.cloudns.net" "saison-photo.org" "-A"]; + + servicegroups = "webstatus-dns"; + _webstatus_name = "saison-photo.org (Secondary DNS ClouDNS)"; + } + + #### Web scenarios + { + service_description = "blog website is running on immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "www.immae.eu" "/blog/" "egrep -ri TODO /etc"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "Immae’s Blog"; + _webstatus_url = "https://www.immae.eu/blog"; + } + + { + service_description = "owncloud website is running on cloud.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "cloud.immae.eu" "/" "a safe home for all your data"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Nextcloud"; + _webstatus_url = "https://cloud.immae.eu"; + } + + { + service_description = "davical website is running on dav.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "dav.immae.eu" "/davical/" "Log On Please"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Davical"; + _webstatus_url = "https://dav.immae.eu/davical"; + } + + { + service_description = "adminer website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https_auth" "tools.immae.eu" "/adminer/" "www.adminer.org"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Adminer"; + _webstatus_url = "https://tools.immae.eu/adminer/"; + } + + { + service_description = "ttrss website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "tools.immae.eu" "/ttrss/" "Tiny Tiny RSS"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "TT-RSS"; + _webstatus_url = "https://tools.immae.eu/ttrss/"; + } + + { + service_description = "mpd website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https_auth" "tools.immae.eu" "/mpd/" "<title>ympd"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "MPD (YMPD)"; + _webstatus_url = "https://tools.immae.eu/mpd/"; + } + + { + service_description = "rompr mpd website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https_auth" "tools.immae.eu" "/rompr/" "<title>RompЯ"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "MPD (ROMPR)"; + _webstatus_url = "https://tools.immae.eu/rompr/"; + } + + { + service_description = "wallabag website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "tools.immae.eu" "/wallabag/" "<title>Bienvenue sur wallabag"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Wallabag"; + _webstatus_url = "https://tools.immae.eu/wallabag/"; + } + + { + service_description = "yourl website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "tools.immae.eu" "/url/admin/" "<title>YOURLS"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "YOURLS"; + _webstatus_url = "https://tools.immae.eu/url/admin/"; + } + + { + service_description = "roundcube website is running on mail.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https_code" "mail.immae.eu" "/roundcube/" "401" "<title>Roundcube"]; + + servicegroups = "webstatus-webapps,webstatus-email"; + _webstatus_name = "Roundcube"; + _webstatus_url = "https://mail.immae.eu/roundcube/"; + } + + { + service_description = "dokuwiki website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "tools.immae.eu" "/dokuwiki/" "<title>start"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Dokuwiki"; + _webstatus_url = "https://tools.immae.eu/dokuwiki/"; + } + + { + service_description = "shaarli website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "tools.immae.eu" "/Shaarli/immae" "<title>Immae"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Shaarli"; + _webstatus_url = "https://tools.immae.eu/Shaarli/"; + } + + { + service_description = "ldap website is running on tools.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "tools.immae.eu" "/ldap/" "<title>phpLDAPadmin"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "LDAP"; + _webstatus_url = "https://tools.immae.eu/ldap/"; + } + + { + service_description = "gitweb website is running on git.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "git.immae.eu" "/" "git web interface"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Git"; + _webstatus_url = "https://git.immae.eu/"; + } + + { + service_description = "mantisbt website is running on git.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "git.immae.eu" "/mantisbt/" "<title>My View - MantisBT"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Mantisbt"; + _webstatus_url = "https://git.immae.eu/mantisbt"; + } + + { + service_description = "diaspora website is running on diaspora.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "diaspora.immae.eu" "/" "is the online social world where you are in control"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Diaspora"; + _webstatus_url = "https://diaspora.immae.eu/"; + } + + { + service_description = "peertube website is running on peertube.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "peertube.immae.eu" "/" "<title>Immae’s PeerTube"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Peertube"; + _webstatus_url = "https://peertube.immae.eu/"; + } + + { + service_description = "etherpad website is running on ether.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "ether.immae.eu" "/" "<title>Etherpad"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Etherpad"; + _webstatus_url = "https://ether.immae.eu/"; + } + + { + service_description = "mediagoblin website is running on mgoblin.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "mgoblin.immae.eu" "/" "<title>GNU MediaGoblin"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Mediagoblin"; + _webstatus_url = "https://mgoblin.immae.eu/"; + } + + { + service_description = "mastodon website is running on mastodon.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "mastodon.immae.eu" "/" "Mastodon"]; + + servicegroups = "webstatus-webapps"; + _webstatus_name = "Mastodon"; + _webstatus_url = "https://mastodon.immae.eu/"; + } + + # Other web pages + { + service_description = "Jerome website is running on naturaloutil.immae.eu"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "naturaloutil.immae.eu" "/ping.php" "YES"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "naturaloutil.immae.eu"; + _webstatus_url = "https://naturaloutil.immae.eu/"; + } + + { + service_description = "Telio website is running on realistesmedia.fr"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "realistesmedia.fr" "/" "Réal'istes"]; + contact_groups = "telio-tortay"; + } + + { + service_description = "Chloe website site is running on osteopathe-cc.fr"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "www.osteopathe-cc.fr" "/" "<title>Chloé Condamin ostéopathe D.O."]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "osteopathe-cc.fr"; + _webstatus_url = "https://www.osteopathe-cc.fr/"; + } + + { + service_description = "Richie website is running on europe-richie.org"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "www.europe-richie.org" "/" "<title>.Europe Richie]"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "europe-richie.org"; + _webstatus_url = "https://www.europe-richie.org/"; + } + + { + service_description = "Ludivine website site is running on ludivinecassal.com"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "www.ludivinecassal.com" "/" "<title>Ludivine Cassal"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "ludivinecassal.com"; + _webstatus_url = "https://www.ludivinecassal.com/"; + } + + { + service_description = "Aten website site is running on aten.pro"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "aten.pro" "/" "<title>ATEN"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "aten.pro"; + _webstatus_url = "https://aten.pro/"; + } + + { + service_description = "PiedsJaloux website site is running on piedsjaloux.fr"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "www.piedsjaloux.fr" "/" "<title>Les Pieds Jaloux"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "piedsjaloux.fr"; + _webstatus_url = "https://www.piedsjaloux.fr/"; + } + + { + service_description = "Denise and Jerome website site is running on denisejerome.piedsjaloux.fr"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "denisejerome.piedsjaloux.fr" "/" "<title>Accueil - Mariage"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "denisejerome.piedsjaloux.fr"; + _webstatus_url = "https://denisejerome.piedsjaloux.fr/"; + } + + { + service_description = "Connexionswing website site is running on connexionswing.com"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "www.connexionswing.com" "/" "<title>Cours, stages"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "connexionswing.com"; + _webstatus_url = "https://www.connexionswing.com/"; + } + + { + service_description = "Sandetludo website site is running on sandetludo.com"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = ["check_https" "www.sandetludo.com" "/" "<title>Cours de West Coast"]; + + servicegroups = "webstatus-websites"; + _webstatus_name = "sandetludo.com"; + _webstatus_url = "https://www.sandetludo.com/"; + } + + { + service_description = "Maison bbc accepts add requests on http"; + host_name = "eldiron.immae.eu"; + use = "external-web-service"; + check_command = [ "check_http" "maison.bbc.bouya.org" "/add.php" "^$"]; + } + + { + service_description = "Maison bbc has up to date data"; + host_name = "eldiron.immae.eu"; + use = "generic-service"; + contact_groups = "maison-bbc"; + notification_interval = "1440"; + check_command = "check_maison_bbc"; + flap_detection_enabled = "0"; + max_check_attempts = "1"; + } + + # SSL + { + service_description = "ldap SSL is up to date"; + host_name = "eldiron.immae.eu"; + use = "external-service"; + check_command = ["check_tcp_ssl" "636"]; + + servicegroups = "webstatus-ssl"; + _webstatus_name = "LDAP"; + _webstatus_url = "ldap.immae.eu"; + } + ]; + contact = { + telio-tortay = config.myEnv.monitoring.contacts.telio-tortay // { + use = "generic-contact"; + contactgroups = "telio-tortay"; + }; + papa = config.myEnv.monitoring.contacts.papa // { + use = "generic-contact"; + contactgroups = "maison-bbc"; + service_notification_commands = "notify-maison-bbc-by-email"; + }; + }; + contactgroup = { + maison-bbc = { alias = "Maison BBC"; }; + telio-tortay = { alias = "Telio Tortay"; members = "immae"; }; + }; +} diff --git a/modules/private/monitoring/objects_phare.nix b/modules/private/monitoring/objects_phare.nix new file mode 100644 index 0000000..ab46436 --- /dev/null +++ b/modules/private/monitoring/objects_phare.nix @@ -0,0 +1,22 @@ +{ ... }: +{ + host = { + "phare.normalesup.org" = { + alias = "phare.normalesup.org"; + address = "phare.normalesup.org"; + use = "linux-server"; + hostgroups = "webstatus-hosts"; + _webstatus_name = "phare"; + }; + }; + service = [ + { + service_description = "phare.normalesup.org email service is active"; + use = "external-passive-service"; + host_name = "phare.normalesup.org"; + freshness_threshold = "1350"; + retry_interval = "1"; + servicegroups = "webstatus-email"; + } + ]; +} diff --git a/modules/private/monitoring/objects_tiboqorl-fr.nix b/modules/private/monitoring/objects_tiboqorl-fr.nix new file mode 100644 index 0000000..847143e --- /dev/null +++ b/modules/private/monitoring/objects_tiboqorl-fr.nix @@ -0,0 +1,12 @@ +{ ... }: +{ + host = { + "tiboqorl.fr" = { + alias = "tiboqorl.fr"; + address = "tiboqorl.fr"; + use = "linux-server"; + hostgroups = "webstatus-hosts"; + _webstatus_name = "tiboqorl"; + }; + }; +} diff --git a/modules/private/monitoring/objects_ulminfo-fr.nix b/modules/private/monitoring/objects_ulminfo-fr.nix new file mode 100644 index 0000000..87a3e05 --- /dev/null +++ b/modules/private/monitoring/objects_ulminfo-fr.nix @@ -0,0 +1,22 @@ +{ ... }: +{ + host = { + "ulminfo.fr" = { + alias = "ulminfo.fr"; + address = "ulminfo.fr"; + use = "linux-server"; + hostgroups = "webstatus-hosts"; + _webstatus_name = "ulminfo"; + }; + }; + service = [ + { + service_description = "ulminfo.fr email service is active"; + use = "external-passive-service"; + host_name = "ulminfo.fr"; + freshness_threshold = "1350"; + retry_interval = "1"; + servicegroups = "webstatus-email"; + } + ]; +} diff --git a/modules/private/monitoring/plugins/check_ftp_database b/modules/private/monitoring/plugins/check_ftp_database new file mode 100755 index 0000000..9a41424 --- /dev/null +++ b/modules/private/monitoring/plugins/check_ftp_database @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +OUT=$(echo "ls" | lftp -u test_ftp,test_ftp eldiron.immae.eu | grep it_works | wc -l) + +if [ "$OUT" -eq 1 ]; then + echo "ftp connection OK - access to ftp is working" + exit 0 +else + echo "ftp connection CRITICAL - no access to ftp" + exit 2 +fi diff --git a/modules/private/monitoring/plugins/check_git b/modules/private/monitoring/plugins/check_git new file mode 100755 index 0000000..8c09925 --- /dev/null +++ b/modules/private/monitoring/plugins/check_git @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +SSH_KEY="$1" + +TMPDIR=$(mktemp -d) + +if [ ! -d "$TMPDIR" ]; then + echo "gitolite UNKNOWN - impossible to create temp dir" + exit 3 +fi + +trap "rm -rf $TMPDIR" EXIT + +ERRORS="" +OUTPUT="" + +cd "$TMPDIR" +OUT=$(git clone -q git://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1) +ERR=$? +if [ -n "$OUT" ]; then +OUTPUT="$OUTPUT +$OUT" +fi +if [ "$ERR" != 0 ]; then + ERRORS="$ERRORS git://" +fi +rm -rf Monitor + +OUT=$(git clone -q http://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1) +ERR=$? +if [ -n "$OUT" ]; then +OUTPUT="$OUTPUT +$OUT" +fi +if [ "$ERR" != 0 ]; then + ERRORS="$ERRORS http://" +fi +rm -rf Monitor + +OUT=$(git clone -q https://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1) +ERR=$? +if [ -n "$OUT" ]; then +OUTPUT="$OUTPUT +$OUT" +fi +if [ "$ERR" != 0 ]; then + ERRORS="$ERRORS https://" +fi +rm -rf Monitor + +OUT=$(GIT_SSH_COMMAND="ssh -i $SSH_KEY -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no" git clone -q gitolite@git.immae.eu:perso/Immae/Projets/Ruby/Monitor 2>&1) +ERR=$? +if [ -n "$OUT" ]; then +OUTPUT="$OUTPUT +$OUT" +fi +if [ "$ERR" != 0 ]; then + ERRORS="$ERRORS ssh" +fi +rm -rf Monitor + +if [ -n "$ERRORS" ]; then + echo "gitolite CRITICAL - impossible to clone via$ERRORS|$OUTPUT" + exit 2 +else + echo "gitolite OK - ssh, git, http and https work|$OUTPUT" + exit 0 +fi diff --git a/modules/private/monitoring/plugins/check_imap_connection b/modules/private/monitoring/plugins/check_imap_connection new file mode 100755 index 0000000..304eae6 --- /dev/null +++ b/modules/private/monitoring/plugins/check_imap_connection @@ -0,0 +1,52 @@ +#!/usr/bin/env perl + +use strict; +use Getopt::Std; +$| = 1; + +my %opts; +getopts('h:u:p:H:', \%opts); + +my $STATE_OK = 0; +my $STATE_WARNING = 1; +my $STATE_CRITICAL = 2; +my $STATE_UNKNOWN = 3; + +if ($opts{'h'} || !$opts{'u'} || !$opts{'p'} || !$opts{'H'}) { + &print_help(); + exit($STATE_UNKNOWN); +} + +my $user = $opts{'u'}; +my $password = $opts{'p'}; +my $host = $opts{'H'}; + +my $cmd_result = `(echo "a login $user $password"; echo "b logout") | openssl s_client -quiet -ign_eof -connect $host -starttls imap 2>&1`; +my $expected_result = "a OK Logged in"; + +chomp($cmd_result); +if ($cmd_result !~ /$expected_result/) { + print "IMAP CRITICAL - Unable to connect via imaps\n"; + exit($STATE_CRITICAL); +} else { + print "IMAP OK - imaps connected successfully\n"; + exit($STATE_OK); +} + +sub print_help() { + print << "EOF"; +Check whether imap works via ssl and is able to connect its database. + +Options: +-h + Print detailed help screen +-u + User to log in as +-p + Password to log in +-H + Host to log in to + +EOF +} + diff --git a/modules/private/monitoring/plugins/check_maison_bbc b/modules/private/monitoring/plugins/check_maison_bbc new file mode 100755 index 0000000..43a5817 --- /dev/null +++ b/modules/private/monitoring/plugins/check_maison_bbc @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# kw, kwh, th_hr Bureau, th_hr Chambre_RDC, th_hr Exterieure, th_hr +# Garage, th_hr Mezzanine, th_hr Sejour, th_hr Soufflage + +alert=$(date +"%F %H:%M:%S" -d "3 hours ago") +res=$(curl https://maison.bbc.bouya.org/stats.php) + +ignored_text="th_hr Exterieure th_hr Soufflage" + +ignored_bat="^$" +ignored="^(th_hr Exterieure|th_hr Soufflage)$" + +missing() { + echo "to_entries|[.[]|select(.value < \"$alert\" and .key != \"bat\" $1)|.key + \": \" + .value]|join(\" ; \")" +} + +missing_check=$(echo "$res" | jq -r "$(missing "and (.key | test(\"$ignored\") | not)")") +missing_show=$(echo "$res" | jq -r "$(missing "")") + +battery() { + echo ".bat|to_entries|[.[]|select((.value[0] < \"$alert\" or (.value[1] | test(\"^0|1|2$\") | not)) $1)|.key + \": \" + .value[0] + \" bat \" + .value[1]]|join(\" ; \")" +} + +bat_check=$(echo "$res" | jq -r "$(battery "and (.key | test(\"$ignored_bat\") | not)")") +bat_show=$(echo "$res" | jq -r "$(battery "")") + +if [ -n "$missing_check" -o -n "$bat_check" ]; then + text="" + if [ -n "$missing_show" ]; then + text="$text - Dernières dates: $missing_show" + fi + if [ -n "$bat_show" ]; then + text="$text - Batteries: $bat_show" + fi + echo "Zibase CRITICAL$text" + exit 2 +else + echo "Zibase OK - Derniers événements il y a moins de 3h (ignorés : $ignored_text)" + exit 0 +fi diff --git a/modules/private/monitoring/plugins/notify_by_email b/modules/private/monitoring/plugins/notify_by_email index ad0dcc7..959db26 100755 --- a/modules/private/monitoring/plugins/notify_by_email +++ b/modules/private/monitoring/plugins/notify_by_email @@ -26,6 +26,4 @@ else subject="** $NOTIFICATION_TYPE Service Alert: $HOST_ALIAS/$SERVICEDESC is $SERVICESTATE **" fi -# sendwait waits for sendmail to finish its job, otherwise it continues in the -# background and gets killed too early -echo "$message" | MAILRC=/dev/null mail -r "naemon@immae.eu" -n -Ssendwait -s "$subject" "$CONTACT" +echo "$message" | MAILRC=/dev/null mail -r "$ADMINEMAIL" -n -s "$subject" "$CONTACT" diff --git a/modules/private/monitoring/plugins/notify_by_slack b/modules/private/monitoring/plugins/notify_by_slack new file mode 100755 index 0000000..b59a403 --- /dev/null +++ b/modules/private/monitoring/plugins/notify_by_slack @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +SLACK_CHANNEL="$1" +SLACK_USERNAME="Naemon" +SLACK_URL="$2" +SLACK_USERICON="https://release.immae.eu/monitoring.png" + +if [ "$SERVICESTATE" = "CRITICAL" ]; then + ICON=":exclamation:" + COLOR="#DA0505" +elif [ "$SERVICESTATE" = "WARNING" ]; then + ICON=":warning:" + COLOR="#F1E903" +elif [ "$SERVICESTATE" = "OK" ]; then + ICON=":white_check_mark:" + COLOR="#36a64f" +elif [ "$SERVICESTATE" = "UNKNOWN" ]; then + ICON=":question:" + COLOR="#000000" +else + ICON=":white_medium_square:" + COLOR="#ffffff" +fi + +payload=$(echo "{}" | jq -r \ + --arg "icon_url" "$SLACK_USERICON" \ + --arg "channel" "$SLACK_CHANNEL" \ + --arg "username" "$SLACK_USERNAME" \ + --arg "text" "${ICON} ${SERVICEDESC} on ${HOST} is ${SERVICESTATE}" \ + --arg "color" "$COLOR" \ + --arg "host" "$HOST" \ + --arg "desc" "$SERVICEDESC" \ + --arg "state" "$SERVICESTATE" \ + --arg "output" "$SERVICEOUTPUT" \ + '.icon_url = $icon_url | + .channel = $channel | + .username = $username | + .text = $text | + .attachments = [{fallback:"", color:$color,fields: [{},{},{},{}]}] | + .attachments[0].fields[0] = {title:"Host",value:$host,short:true} | + .attachments[0].fields[1] = {title:"Service description",value:$desc,short:true} | + .attachments[0].fields[2] = {title:"Status",value:$state,short:true} | + .attachments[0].fields[3] = {title:"Message",value:$output,short:false} + ') + +curl -X POST --data "payload=$payload" $SLACK_URL diff --git a/modules/private/monitoring/plugins/notify_maison_bbc_by_email b/modules/private/monitoring/plugins/notify_maison_bbc_by_email new file mode 100755 index 0000000..84958b8 --- /dev/null +++ b/modules/private/monitoring/plugins/notify_maison_bbc_by_email @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# $1 = service/host + +# $2 = type (PROBLEM RECOVERY ACKNOWLEDGEMENT FLAPPINGSTART FLAPPINGSTOP FLAPPINGDISABLED DOWNTIMESTART DOWNTIMESTOP DOWNTIMECANCELLED) +# http://www.naemon.org/documentation/usersguide/macrolist.html#notificationtype + +# $3 = host alias + +# $4 = date (YYYY-MM-DDTHH:MM:SS) + +# $5 = E-mail + +NOTIFICATION_TYPE="$2" +DATE="$4" +CONTACT="$5" + +output=$(echo "$SERVICEOUTPUT" | awk '{gsub(" - ", "\n\n", $0); gsub (" (;|:) ", "\n ", $0); gsub(" dates: ", " dates:\n ", $0); gsub("Batteries: ", "Batteries:\n ", $0); print $0}') +if [ "$NOTIFICATION_TYPE" = "RECOVERY" ]; then + subject="Fin d'alerte Maison BBC Zibase" + message=$(printf "%b" "Alerte terminée le $DATE.\nInformations:\n$output") +else + subject="Alerte Maison BBC Zibase" + message=$(printf "%b" "Alerte lancée le $DATE.\nInformations:\n$output") +fi + +echo "$message" | MAILRC=/dev/null mail -r "$ADMINEMAIL" -n -s "$subject" "$CONTACT" diff --git a/modules/private/monitoring/to_objects.nix b/modules/private/monitoring/to_objects.nix index 5ad76e0..7b4b523 100644 --- a/modules/private/monitoring/to_objects.nix +++ b/modules/private/monitoring/to_objects.nix @@ -17,7 +17,7 @@ let define service { ${builtins.concatStringsSep "\n" (mapAttrsToList (k: v: " ${pad 30 k} ${toStr k v}" - ) service)} + ) (filterAttrs (k: v: ! builtins.elem k ["passiveInfo" "filter"]) service))} } ''; toServices = services: builtins.concatStringsSep "\n" (map toService services); @@ -59,7 +59,7 @@ let then toCommands v else if keyname == "templates" then toTemplates v - else if builtins.elem keyname ["host" "contactgroup" "contact" "timeperiod"] + else if builtins.elem keyname ["hostgroup" "host" "contactgroup" "contact" "timeperiod" "servicegroup"] then toOthers keyname v else ""; toObjects = v: builtins.concatStringsSep "\n" (mapAttrsToList toObjects' v); diff --git a/modules/private/system/monitoring-1.nix b/modules/private/system/monitoring-1.nix new file mode 100644 index 0000000..1460478 --- /dev/null +++ b/modules/private/system/monitoring-1.nix @@ -0,0 +1,41 @@ +{ privateFiles }: +{ config, pkgs, resources, ... }: +{ + boot.kernelPackages = pkgs.linuxPackages_latest; + myEnv = import "${privateFiles}/environment.nix" // { inherit privateFiles; }; + hostEnv.FQDN = "monitoring-1.v.immae.eu"; + + imports = builtins.attrValues (import ../..); + + deployment = { + targetEnv = "hetznerCloud"; + hetznerCloud = { + authToken = config.myEnv.hetznerCloud.authToken; + datacenter = "hel1-dc2"; + location ="hel1"; + serverType = "cx11"; + }; + }; + + myServices.monitoring.enable = true; + myServices.monitoring.master = true; + myServices.status.enable = true; + networking = { + firewall.enable = true; + interfaces."ens3".ipv4.addresses = pkgs.lib.attrsets.mapAttrsToList + (n: ips: { address = ips.ip4; prefixLength = 32; }) + (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.myEnv.servers.monitoring-1.ips); + interfaces."ens3".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList + (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or [])) + config.myEnv.servers.monitoring-1.ips); + defaultGateway6 = { address = "fe80::1"; interface = "ens3"; }; + }; + myServices.mailRelay.enable = true; + + # This value determines the NixOS release with which your system is + # to be compatible, in order to avoid breaking some software such as + # database servers. You should change this only after NixOS release + # notes say you should. + # https://nixos.org/nixos/manual/release-notes.html + system.stateVersion = "19.03"; # Did you read the comment? +} diff --git a/nixops/Makefile b/nixops/Makefile index 5e654ee..02d34f8 100644 --- a/nixops/Makefile +++ b/nixops/Makefile @@ -36,6 +36,9 @@ ssh-eldiron: ssh-backup-2: $(NIXOPS_PRIV) ssh backup-2 -- $(SSH_ARGS) +ssh-monitoring-1: + $(NIXOPS_PRIV) ssh monitoring-1 -- $(SSH_ARGS) + info: $(NIXOPS_PRIV) list $(NIXOPS_PRIV) info @@ -74,11 +77,15 @@ list-generations: delete-generations: nix-env -p $(profile) --delete-generations $(GEN) $(NIXOPS_PRIV) ssh eldiron -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN) + $(NIXOPS_PRIV) ssh backup-2 -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN) + $(NIXOPS_PRIV) ssh monitoring-1 -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN) .PHONY: delete-generations cleanup: delete-generations nix-store --gc $(NIXOPS_PRIV) ssh eldiron -- nix-store --gc + $(NIXOPS_PRIV) ssh backup-2 -- nix-store --gc + $(NIXOPS_PRIV) ssh monitoring-1 -- nix-store --gc .PHONY: cleanup ###### Pull environment and deployment from remote diff --git a/nixops/default.nix b/nixops/default.nix index 45a235e..5f4f4d2 100644 --- a/nixops/default.nix +++ b/nixops/default.nix @@ -9,4 +9,5 @@ resources.sshKeyPairs.ssh-key = {}; eldiron = import ../modules/private/system/eldiron.nix { inherit privateFiles; }; backup-2 = import ../modules/private/system/backup-2.nix { inherit privateFiles; }; + monitoring-1 = import ../modules/private/system/monitoring-1.nix { inherit privateFiles; }; } -- 2.41.0