subrecipes = setup
-subrecipes += nixops ssh-eldiron ssh-backup-2 info debug dry-run build upload deploy deploy-reboot reboot
+subrecipes += nixops ssh-eldiron ssh-backup-2 ssh-monitoring-1
+subrecipes += info debug dry-run build upload deploy deploy-reboot reboot
subrecipes += list-generations delete-generations cleanup
subrecipes += pull pull_environment pull_deployment deployment_is_set push push_deployment push_environment
${subrecipes}:
default = {};
type = attrsOf (submodule {
options = {
+ emails = mkOption {
+ default = [];
+ description = "List of e-mails that the server can be a sender of";
+ type = listOf str;
+ };
ldap = mkOption {
description = ''
LDAP credentials for the host
options = {
status_url = mkOption { type = str; description = "URL to push status to"; };
status_token = mkOption { type = str; description = "Token for the status url"; };
+ http_user_password = mkOption { type = str; description = "HTTP credentials to check services behind wall"; };
email = mkOption { type = str; description = "Admin E-mail"; };
+ ssh_public_key = mkOption { type = str; description = "SSH public key"; };
+ ssh_secret_key = mkOption { type = str; description = "SSH secret key"; };
+ imap_login = mkOption { type = str; description = "IMAP login"; };
+ imap_password = mkOption { type = str; description = "IMAP password"; };
+ nrdp_tokens = mkOption { type = listOf str; description = "Tokens allowed to push status update"; };
+ slack_url = mkOption { type = str; description = "Slack webhook url to push status update"; };
+ slack_channel = mkOption { type = str; description = "Slack channel to push status update"; };
+ contacts = mkOption { type = attrsOf unspecified; description = "Contact dicts to fill naemon objects"; };
};
};
};
-{ config, pkgs, lib, name, ... }:
+{ config, pkgs, lib, name, nodes, ... }:
let
+ cfg = config.myServices.monitoring;
myplugins = pkgs.runCommand "buildplugins" {
buildInputs = [ pkgs.makeWrapper pkgs.perl ];
} ''
wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
pkgs.curl pkgs.jq
]}
+ wrapProgram $out/check_maison_bbc --prefix PATH : ${lib.makeBinPath [
+ pkgs.curl pkgs.jq
+ ]}
wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
pkgs.gnugrep pkgs.gawk pkgs.procps-ng
]}
wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
]}
+ wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
+ pkgs.lftp
+ ]}
+ wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [
+ pkgs.git pkgs.openssh
+ ]}
+ wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [
+ pkgs.openssl
+ ]}
+ wrapProgram $out/notify_maison_bbc_by_email --prefix PATH : ${lib.makeBinPath [
+ pkgs.mailutils pkgs.gawk
+ ]}
+ wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
+ pkgs.mailutils
+ ]}
+ wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
+ pkgs.curl pkgs.jq
+ ]}
'';
toObjects = pkgs.callPackage ./to_objects.nix {};
commonConfig = {
processWarn = "50"; processAlert = "60";
loadWarn = "1.0"; loadAlert = "2.0";
};
+ monitoring-1 = {
+ processWarn = "50"; processAlert = "60";
+ loadWarn = "1.0"; loadAlert = "2.0";
+ };
};
+ masterPassiveObjects = let
+ otherPassiveObjects = map
+ (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") {}))
+ [ "caldance-1" "ulminfo-fr" "immae-eu" "phare" "tiboqorl-fr" ];
+ otherPassiveServices = lib.flatten (map (h: h.service or []) otherPassiveObjects);
+ otherPassiveHosts = (map (h: h.host)) otherPassiveObjects;
+ passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron"]) nodes;
+ toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
+ passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
+ (_: n: toPassiveServices n.config.myServices.monitoring.services)
+ passiveNodes
+ );
+ in {
+ service = passiveServices ++ otherPassiveServices;
+ host = lib.lists.foldr
+ (a: b: a//b)
+ {}
+ (otherPassiveHosts ++ lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes);
+ };
+ masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
commonObjects = pkgs.callPackage ./objects_common.nix ({
+ master = cfg.master;
hostFQDN = config.hostEnv.FQDN;
+ hostName = name;
sudo = "/run/wrappers/bin/sudo";
} // builtins.getAttr name commonConfig);
hostObjects =
let
specific_file = ./. + "/objects_" + name + ".nix";
in
- lib.attrsets.optionalAttrs (builtins.pathExists specific_file) (pkgs.callPackage specific_file { inherit config; });
+ lib.attrsets.optionalAttrs
+ (builtins.pathExists specific_file)
+ (pkgs.callPackage specific_file {
+ inherit config;
+ hostFQDN = config.hostEnv.FQDN;
+ hostName = name;
+ });
in
{
options = {
Whether to enable monitoring.
'';
};
+ master = lib.mkOption {
+ type = lib.types.bool;
+ default = false;
+ description = ''
+ This instance is the master instance
+ '';
+ };
+ hosts = lib.mkOption {
+ readOnly = true;
+ description = "Hosts list for this host";
+ default = (commonObjects.host or {}) // (hostObjects.host or {});
+ };
+ services = lib.mkOption {
+ readOnly = true;
+ description = "Services list for this host";
+ default = commonObjects.service ++ hostObjects.service;
+ };
};
};
- config = lib.mkIf config.myServices.monitoring.enable {
+ config = lib.mkIf cfg.enable {
services.duplyBackup.profiles.monitoring = {
rootDir = config.services.naemon.varDir;
};
text = "MAILADDR ${config.myEnv.monitoring.email}";
};
+ secrets.keys = [
+ {
+ dest = "naemon/id_rsa";
+ user = "naemon";
+ group = "naemon";
+ premissions = "0400";
+ text = config.myEnv.monitoring.ssh_secret_key;
+ }
+ ];
# needed since extraResource is not in the closure
systemd.services.naemon.path = [ myplugins ];
services.naemon = {
log_initial_states=1
date_format=iso8601
admin_email=${config.myEnv.monitoring.email}
-
+ '' + lib.optionalString (!cfg.master) ''
obsess_over_services=1
ocsp_command=notify-master
'';
$USER2$=${myplugins}
$USER200$=${config.myEnv.monitoring.status_url}
$USER201$=${config.myEnv.monitoring.status_token}
+ $USER202$=${config.myEnv.monitoring.http_user_password}
+ $USER203$=${config.secrets.fullPaths."naemon/id_rsa"}
+ $USER204$=${config.myEnv.monitoring.imap_login}
+ $USER205$=${config.myEnv.monitoring.imap_password}
+ $USER206$=${config.myEnv.monitoring.slack_channel}
+ $USER207$=${config.myEnv.monitoring.slack_url}
'';
- objectDefs = toObjects commonObjects + toObjects hostObjects;
+ objectDefs = toObjects commonObjects
+ + toObjects hostObjects
+ + lib.optionalString cfg.master (toObjects masterObjects)
+ + lib.optionalString cfg.master (toObjects masterPassiveObjects);
};
};
}
--- /dev/null
+# -*- coding: utf-8; -*-
+
+[cli]
+history = /var/lib/naemon/nagios_cli_history
+
+[ui]
+color = 1
+prompt = naemon %s>
+prompt_separator = " → "
+
+[nagios]
+log = /var/log/naemon
+command_file = /run/naemon/naemon.cmd
+log_file = %(log)s/naemon.log
+object_cache_file = /var/lib/naemon/objects.cache
+status_file = /var/lib/naemon/status.dat
+
+[object]
+host.status =
+ host_name
+ current_state
+ plugin_output
+ is_flapping
+ last_check
+ last_time_down
+ last_state_change
+ check_period
+ notification_period
+ current_attempt
+ max_attempts
+service.status =
+ host_name
+ service_description
+ current_state
+ is_flapping
+ plugin_output
+ last_time_down
+ last_state_change
+ last_check
+ next_check
+ check_interval
+ check_latency
+ check_period
+ notification_period
+ current_attempt
+ max_attempts
+
+[string]
+level.ok = ↑ OK
+level.warning = ! WARNING
+level.critical = ↓ CRITICAL
+level.unknown = ↕ UNKNOWN
+
+[color]
+error = bold_red
+
+prompt = normal
+prompt.object = bold
+
+host.host_name = bold
+host.plugin_output = bold
+service.plugin_output = bold
+
+level.ok = bold_green
+level.warning = bold_yellow
+level.critical = bold_red
+level.unknown = bold_magenta
+
-{ config, pkgs, ... }:
+{ config, pkgs, lib, hostFQDN, ... }:
+let
+ defaultPassiveInfo = {
+ filter = lib.attrsets.filterAttrs
+ (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_");
+ use = "external-passive-service";
+ freshness_threshold = "450";
+ retry_interval = "1";
+ servicegroups = "webstatus-resources";
+ host_name = hostFQDN;
+ };
+in
{
service = [
{
+ passiveInfo = defaultPassiveInfo;
service_description = "Size on /backup2 partition";
use = "local-service";
check_command = ["check_local_disk" "10%" "5%" "/backup2"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-backup"; };
service_description = "Last backup in /backup2/phare is not too old";
use = "local-service";
check_command = ["check_last_file_date" "/backup2/phare" "14" "backup"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-backup"; };
service_description = "Last backup in /backup2/immae_eu is not too old";
use = "local-service";
check_command = ["check_last_file_date" "/backup2/immae_eu" "14" "backup"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-backup"; };
service_description = "Last backup in /backup2/ulminfo is not too old";
use = "local-service";
check_command = ["check_last_file_date" "/backup2/ulminfo" "14" "backup"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases,webstatus-backup"; };
service_description = "Last postgresql dump in /backup2/eldiron/postgresql_backup is not too old";
use = "local-service";
check_command = ["check_last_file_date" "/backup2/eldiron/postgresql_backup" "7" "postgres"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; };
service_description = "Redis replication for eldiron is up to date";
use = "local-service";
check_command = ["check_redis_replication" "/run/redis_eldiron/redis.sock"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases,webstatus-backup"; };
service_description = "Last redis dump in /backup2/eldiron/redis_backup is not too old";
use = "local-service";
check_command = ["check_last_file_date" "/backup2/eldiron/redis_backup" "7" "redis"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; };
service_description = "Mysql replication for eldiron is up to date";
use = "local-service";
check_command = ["check_mysql_replication" "/run/mysqld_eldiron/mysqld.sock" "/var/secrets/mysql_replication/eldiron/client"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases,webstatus-backup"; };
service_description = "Last mysql dump in /backup2/eldiron/mysql_backup is not too old";
use = "local-service";
check_command = ["check_last_file_date" "/backup2/eldiron/mysql_backup" "7" "mysql"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; };
service_description = "Openldap replication for eldiron is up to date";
use = "local-service";
check_command = let
];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases,webstatus-backup"; };
service_description = "Last openldap dump in /backup2/eldiron/openldap_backup is not too old";
use = "local-service";
check_command = ["check_last_file_date" "/backup2/eldiron/openldap_backup" "7" "openldap"];
--- /dev/null
+{ ... }:
+let
+ defaultPassiveInfo = {
+ use = "external-passive-service";
+ freshness_threshold = "450";
+ retry_interval = "1";
+ host_name = "caldance-1.v.immae.eu";
+ };
+in
+{
+ host = {
+ "caldance-1.v.immae.eu" = {
+ alias = "caldance-1.v.immae.eu";
+ address = "caldance-1.v.immae.eu";
+ use = "linux-server";
+ hostgroups = "webstatus-hosts";
+ _webstatus_name = "caldance-1";
+ };
+ };
+ service = map (s: defaultPassiveInfo // s) [
+ {
+ service_description = "Size on root partition";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "Total number of process";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "Average load";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "Swap usage";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "fail2ban is active";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "NTP is activated and working";
+ servicegroups = "webstatus-resources";
+ freshness_threshold = "5400";
+ }
+
+ {
+ service_description = "Databases are present in postgresql";
+ servicegroups = "webstatus-databases";
+ }
+ ];
+}
{ hostFQDN
+, hostName
, processWarn ? "250"
, processAlert ? "400"
, loadWarn ? "8.0"
, loadAlert ? "10.0"
, mdadm
, sudo
+, master
+, lib
, ...
}:
+let
+ defaultPassiveInfo = {
+ filter = lib.attrsets.filterAttrs
+ (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_");
+ use = "external-passive-service";
+ freshness_threshold = "450";
+ retry_interval = "1";
+ servicegroups = "webstatus-resources";
+ host_name = hostFQDN;
+ };
+in
{
host = {
"${hostFQDN}" = {
alias = hostFQDN;
address = hostFQDN;
use = "linux-server";
+ hostgroups = "webstatus-hosts";
+ _webstatus_name = hostName;
};
};
service = [
{
+ passiveInfo = defaultPassiveInfo;
service_description = "Size on root partition";
use = "local-service";
check_command = ["check_local_disk" "20%" "10%" "/"];
}
{
+ passiveInfo = defaultPassiveInfo;
service_description = "Total number of process";
use = "local-service";
check_command = [
];
}
{
+ passiveInfo = defaultPassiveInfo;
service_description = "Average load";
use = "local-service";
check_command = [
];
}
{
+ passiveInfo = defaultPassiveInfo;
service_description = "Swap usage";
use = "local-service";
check_command = ["check_local_swap" "20" "10"];
}
{
+ passiveInfo = defaultPassiveInfo;
service_description = "Memory usage";
use = "local-service";
check_command = ["check_memory" "80" "90"];
}
{
+ passiveInfo = defaultPassiveInfo;
service_description = "NTP is activated and working";
use = "local-service";
check_command = ["check_ntp"];
}
{
+ passiveInfo = defaultPassiveInfo;
service_description = "No mdadm array is degraded";
use = "local-service";
check_command = [
}
];
command = {
+ check_dns = "$USER1$/check_dns -H $ARG1$ -s $HOSTADDRESS$ $ARG2$";
+ check_external_dns = "$USER1$/check_dns -H $ARG2$ -s $ARG1$ $ARG3$";
+ check_ftp_database = "$USER2$/check_ftp_database";
+ check_git = "$USER2$/check_git $USER203$";
+ check_http = "$USER1$/check_http --sni -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\"";
+ check_https = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\"";
+ check_https_auth = "$USER1$/check_http --sni --ssl -a \"$USER202$\" -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -r \"$ARG3$\"";
+ check_https_certificate = "$USER1$/check_http --sni --ssl -H \"$ARG1$\" -C 21,15";
+ check_https_code = "$USER1$/check_http --sni --ssl -f stickyport -H \"$ARG1$\" -u \"$ARG2$\" -e \"$ARG3$\" -r \"$ARG4$\"";
+ check_imap_connection = "$USER2$/check_imap_connection -u \"$USER204$\" -p \"$USER205$\" -H \"imap.immae.eu:143\"";
check_local_disk = "$USER1$/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$";
check_local_procs = "$USER1$/check_procs -w $ARG1$ -c $ARG2$ -s $ARG3$";
check_local_load = "$USER1$/check_load -w $ARG1$ -c $ARG2$";
check_local_swap = "$USER1$/check_swap -n ok -w $ARG1$ -c $ARG2$";
check_memory = "$USER2$/check_mem.sh -w $ARG1$ -c $ARG2$";
+ check_command_match = "$USER2$/check_command -c \"$ARG1$\" -C \"$ARG2$\" $ARG3$";
check_command_output = "$USER2$/check_command -c \"$ARG1$\" -s 0 -o \"$ARG2$\" $ARG3$";
+ check_command_status = "$USER2$/check_command -c \"$ARG1$\" -s \"$ARG2$\" $ARG3$";
check_ntp = "$USER1$/check_ntp_time -t 30 -q -H 0.arch.pool.ntp.org";
+ check_mailq = "$USER1$/check_mailq -s -w 1 -c 2";
+ check_maison_bbc = "$USER2$/check_maison_bbc";
check_mysql_replication = "${sudo} -u mysql $USER2$/check_mysql_replication \"$ARG1$\" \"$ARG2$\"";
check_postgresql_replication = "${sudo} -u postgres $USER2$/check_postgres_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\"";
check_openldap_replication = "${sudo} -u openldap $USER2$/check_openldap_replication \"$ARG1$\" \"$ARG2$\" \"$ARG3$\" \"$ARG4$\" \"$ARG5$\"";
check_redis_replication = "${sudo} -u redis $USER2$/check_redis_replication \"$ARG1$\"";
- check_mailq = "$USER1$/check_mailq -s -w 1 -c 2";
+ check_smtp = "$USER1$/check_smtp -H $HOSTADDRESS$ -p 25 -S -D 21,15";
+ check_tcp = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -e \"$ARG2$\" -Mcrit";
+ check_tcp_ssl = "$USER1$/check_tcp -H $HOSTADDRESS$ -p $ARG1$ -S -D 21,15";
check_host_alive = "$USER1$/check_ping -H $HOSTADDRESS$ -w 3000.0,80% -c 5000.0,100% -p 5";
check_last_file_date = "${sudo} -u \"$ARG3$\" $USER2$/check_last_file_date \"$ARG1$\" \"$ARG2$\"";
+ check_ok = "$USER1$/check_dummy 0 \"Dummy OK\"";
+ check_critical = "$USER1$/check_dummy 2 \"Dummy CRITICAL\"";
- # No notify commands, we go through master
- # notify_host_by_email = "SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" HOSTSTATE=\"$HOSTSTATE$\" HOSTOUTPUT=\"$HOSTOUTPUT$\" $USER2$/notify_by_email host \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE";
- # #$OVE is to force naemon to run via shell instead of execve which fails here
- # notify_service_by_email = "SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE";
- # #sudo /usr/bin/strace -o /tmp/foo -vf -s 256 -u naemon $USER2$/notify_by_email
- # #$OVE is to force naemon to run via shell instead of execve which fails here
+ # $OVE is to force naemon to run via shell instead of execve which fails here
+ notify-host-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" HOSTSTATE=\"$HOSTSTATE$\" HOSTOUTPUT=\"$HOSTOUTPUT$\" $USER2$/notify_by_email host \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE";
+ # $OVE is to force naemon to run via shell instead of execve which fails here
+ notify-service-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE";
+ notify-maison-bbc-by-email = "ADMINEMAIL=\"$ADMINEMAIL$\" SERVICENOTIFICATIONID=\"$SERVICENOTIFICATIONID$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_maison_bbc_by_email service \"$NOTIFICATIONTYPE$\" \"$HOSTALIAS$\" \"$LONGDATETIME$\" \"$CONTACTEMAIL$\" $OVE";
+ notify-by-slack = "HOST=\"$HOSTALIAS$\" SERVICESTATE=\"$SERVICESTATE$\" SERVICEDESC=\"$SERVICEDESC$\" SERVICEOUTPUT=\"$SERVICEOUTPUT$\" $USER2$/notify_by_slack \"$ARG1$\" \"$ARG2$\"";
notify-master = "$USER2$/send_nrdp.sh -u \"$USER200$\" -t \"$USER201$\" -H \"$HOSTADDRESS$\" -s \"$SERVICEDESC$\" -S \"$SERVICESTATEID$\" -o \"$SERVICEOUTPUT$\"";
};
sunday = "00:00-24:00";
};
};
+ servicegroup = {
+ webstatus-webapps = { alias = "Web applications"; };
+ webstatus-websites = { alias = "Personal websites"; };
+ webstatus-ssl = { alias = "SSL certificates"; };
+ webstatus-dns = { alias = "DNS resolution"; };
+ webstatus-remote-services = { alias = "Other remote services"; };
+ webstatus-local-services = { alias = "Other local services"; };
+ webstatus-email = { alias = "E-mail services"; };
+ webstatus-resources = { alias = "Local resources"; };
+ webstatus-databases = { alias = "Databases resources"; };
+ webstatus-backup = { alias = "Backup resources"; };
+ };
+ hostgroup = {
+ webstatus-hosts = { alias = "Hosts"; };
+ };
contactgroup = {
admins = { alias = "Naemon Administrators"; };
};
- # No contact, we go through master
- # contact = {
- # immae = {
- # alias = "Immae";
- # use = "generic-contact";
- # email = "xxxxxxxxxxxxxxxx";
- # };
- # };
templates = {
service = {
generic-service = {
notification_interval = "60";
notification_options = "w,u,c,r,f,s";
notification_period = "24x7";
- notifications_enabled = "0"; # no notification since we send them to master
+ notifications_enabled = if master then "1" else "0";
obsess_over_service = "1";
passive_checks_enabled = "1";
process_perf_data = "1";
check_interval = "5";
max_check_attempts = "4";
retry_interval = "1";
+ servicegroups = "webstatus-resources";
+ };
+ external-service = {
+ use = "generic-service";
+ check_interval = "5";
+ max_check_attempts = "4";
+ retry_interval = "1";
+ };
+ web-service = {
+ use = "generic-service";
+ check_interval = "20";
+ max_check_attempts = "2";
+ retry_interval = "1";
+ };
+ external-web-service = {
+ use = "generic-service";
+ check_interval = "20";
+ max_check_attempts = "2";
+ retry_interval = "1";
+ };
+ mail-service = {
+ use = "generic-service";
+ check_interval = "15";
+ max_check_attempts = "1";
+ retry_interval = "1";
+ };
+ dns-service = {
+ use = "generic-service";
+ check_interval = "120";
+ notification_interval = "120";
+ max_check_attempts = "5";
+ retry_interval = "5";
};
};
# No contact, we go through master
- # contact = {
- # generic-contact = {
- # host_notification_commands = "notify_host_by_email";
- # host_notification_options = "d,u,r,f,s";
- # host_notification_period = "24x7";
- # service_notification_commands = "notify_service_by_email";
- # service_notification_options = "w,u,c,r,f,s";
- # service_notification_period = "24x7";
- # };
- # };
+ contact = {
+ generic-contact = {
+ host_notification_commands = "notify-host-by-email";
+ host_notification_options = "d,u,r,f,s";
+ host_notification_period = "24x7";
+ service_notification_commands = "notify-service-by-email";
+ service_notification_options = "w,u,c,r,f,s";
+ service_notification_period = "24x7";
+ };
+ };
host = {
generic-host = {
event_handler_enabled = "1";
-{ ... }:
+{ lib, hostFQDN, ... }:
+let
+ defaultPassiveInfo = {
+ filter = lib.attrsets.filterAttrs
+ (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_");
+ use = "external-passive-service";
+ freshness_threshold = "450";
+ retry_interval = "1";
+ servicegroups = "webstatus-resources";
+ host_name = hostFQDN;
+ };
+in
{
service = [
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-databases"; };
service_description = "Postgresql replication for backup-2 is up to date";
use = "local-service";
check_command = ["check_postgresql_replication" "backup-2" "/run/postgresql" "5432"];
}
{
+ passiveInfo = defaultPassiveInfo // { servicegroups = "webstatus-email"; };
service_description = "mailq is empty";
use = "local-service";
check_command = ["check_mailq"];
--- /dev/null
+{ ... }:
+let
+ defaultPassiveInfo = {
+ use = "external-passive-service";
+ freshness_threshold = "450";
+ retry_interval = "1";
+ host_name = "immae.eu";
+ };
+in
+{
+ host = {
+ "immae.eu" = {
+ alias = "immae.eu";
+ address = "immae.eu";
+ use = "linux-server";
+ hostgroups = "webstatus-hosts";
+ _webstatus_name = "immaeEu";
+ };
+ };
+ service = map (s: defaultPassiveInfo // s) [
+ {
+ service_description = "size on root partition";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "total number of process";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "average load";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "swap usage";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "no mdadm array is degraded";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "ntp is activated and working";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "ssh connexion responds";
+ }
+
+ {
+ service_description = "fail2ban is active";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "pacman packages are not too old";
+ servicegroups = "webstatus-resources";
+ freshness_threshold = "10800";
+ notification_interval = "0";
+ }
+
+ {
+ service_description = "rsync backup happened not too long ago";
+ servicegroups = "webstatus-backup";
+ }
+
+ {
+ service_description = "eriomem backup is up and not full";
+ freshness_threshold = "10800";
+ notification_interval = "120";
+ servicegroups = "webstatus-backup";
+ }
+
+ {
+ service_description = "postfix service is active";
+ }
+
+ {
+ service_description = "tinc@Immae service is active";
+ }
+
+ {
+ service_description = "sshd service is active";
+ }
+
+ {
+ service_description = "httpd service is active";
+ servicegroups = "webstatus-resources";
+ }
+
+ {
+ service_description = "postfix SSL is up to date";
+ }
+
+ {
+ service_description = "mailq is empty";
+ servicegroups = "webstatus-email";
+ }
+
+ ## Sending e-mail
+ {
+ service_description = "immae.eu email service is active";
+ servicegroups = "webstatus-email";
+ freshness_threshold = "1350";
+ }
+
+ #### Web scenarios
+ {
+ service_description = "discourse website is running on discourse.capitaines.fr";
+ freshness_threshold = "1800";
+ }
+
+ {
+ service_description = "Default website site is running on ns208507.ip-188-165-209.eu";
+ freshness_threshold = "1800";
+ }
+ ];
+}
--- /dev/null
+{ config, ... }:
+{
+ contact = {
+ immae = config.myEnv.monitoring.contacts.immae // {
+ use = "generic-contact";
+ contactgroups = "admins";
+ host_notification_commands = "notify-host-by-email,notify-by-slack!$USER206$!$USER207$";
+ service_notification_commands = "notify-service-by-email,notify-by-slack!$USER206$!$USER207$";
+ };
+ };
+ command = {
+ check_passive = "$USER1$/check_dummy 3 \"Service result are stale\"";
+ };
+ templates = {
+ service = {
+ external-passive-service = {
+ active_checks_enabled = "0";
+ check_freshness = "1";
+ check_period = "24x7";
+ contact_groups = "admins";
+ event_handler_enabled = "1";
+ flap_detection_enabled = "1";
+ is_volatile = "0";
+ max_check_attempts = "3";
+ notification_interval = "60";
+ notification_options = "w,u,c,r,f,s";
+ notification_period = "24x7";
+ notifications_enabled = "1";
+ passive_checks_enabled = "1";
+ process_perf_data = "1";
+ retain_nonstatus_information = "1";
+ retain_status_information = "1";
+ retry_interval = "2";
+ check_command = "check_passive";
+ };
+ };
+ };
+}
--- /dev/null
+{ config, pkgs, ... }:
+{
+ host = {
+ # Dummy host for testing
+ # "dummy-host" = {
+ # alias = "dummy.host";
+ # address = "dummy.host";
+ # use = "linux-server";
+ # check_command = "check_ok";
+ # };
+ };
+ service = [
+ # Dummy service for testing
+ # {
+ # service_description = "Dummy failing test";
+ # host_name = "dummy-host";
+ # use = "local-service";
+ # check_interval = "0.3";
+ # max_check_attempts = "1";
+ # flap_detection_enabled = "0";
+ # notification_interval = "0.1";
+ # check_command = "check_critical";
+ # }
+
+ {
+ service_description = "ftp has access to database for authentication";
+ host_name = "eldiron.immae.eu";
+ use = "external-service";
+ check_command = "check_ftp_database";
+
+ servicegroups = "webstatus-remote-services";
+ _webstatus_name = "FTP";
+ _webstatus_url = "ftp.immae.eu";
+ }
+
+ {
+ service_description = "gitolite is working";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = "check_git";
+
+ servicegroups = "webstatus-remote-services";
+ _webstatus_name = "Git";
+ _webstatus_url = "git.immae.eu";
+ }
+
+ {
+ service_description = "postfix SSL is up to date";
+ host_name = "eldiron.immae.eu";
+ use = "external-service";
+ check_command = "check_smtp";
+
+ servicegroups = "webstatus-ssl";
+ _webstatus_name = "SMTP";
+ _webstatus_url = "smtp.immae.eu";
+ }
+
+ {
+ service_description = "imap SSL is up to date";
+ host_name = "eldiron.immae.eu";
+ use = "external-service";
+ check_command = ["check_tcp_ssl" "993"];
+
+ servicegroups = "webstatus-ssl";
+ _webstatus_name = "IMAP";
+ _webstatus_url = "imap.immae.eu";
+ }
+
+ {
+ service_description = "imap connection works";
+ host_name = "eldiron.immae.eu";
+ use = "external-service";
+ check_command = "check_imap_connection";
+
+ servicegroups = "webstatus-remote-services,webstatus-email";
+ _webstatus_name = "IMAP";
+ _webstatus_url = "imap.immae.eu";
+ }
+
+ # DNS services
+ {
+ service_description = "eldiron dns is active and authoritative for aten.pro";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "aten.pro" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "aten.pro";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for bouya.org";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "bouya.org" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "bouya.org";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for filedesreves.fr";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "filedesreves.fr" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "filedesreves.fr";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for ludivinecassal.com";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "ludivinecassal.com" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "ludivinecassal.com";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for icommandeur.org";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "icommandeur.org" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "icommandeur.org";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "immae.eu" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "immae.eu";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for immae.fr";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "immae.fr" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "immae.fr";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for piedsjaloux.fr";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "piedsjaloux.fr" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "piedsjaloux.fr";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for saison-photo.org";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "saison-photo.org" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "saison-photo.org";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for tellesflorian.com";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "tellesflorian.com" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "tellesflorian.com";
+ }
+
+ {
+ service_description = "eldiron dns is active and authoritative for capitaines.fr";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_dns" "capitaines.fr" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "capitaines.fr";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for aten.pro";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "aten.pro" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "aten.pro (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for bouya.org";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "bouya.org" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "bouya.org (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for filedesreves.fr";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "filedesreves.fr" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "filedesreves.fr (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for ludivinecassal.com";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "ludivinecassal.com" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "ludivinecassal.com (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for icommandeur.org";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "icommandeur.org" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "icommandeur.org (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "immae.eu" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "immae.eu (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for immae.fr";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "immae.fr" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "immae.fr (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for piedsjaloux.fr";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "piedsjaloux.fr" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "piedsjaloux.fr (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for tellesflorian.com";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "tellesflorian.com" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "tellesflorian.com (Secondary DNS ClouDNS)";
+ }
+
+ {
+ service_description = "cloudns dns is active and authoritative for saison-photo.org";
+ host_name = "eldiron.immae.eu";
+ use = "dns-service";
+ check_command = ["check_external_dns" "pns1.cloudns.net" "saison-photo.org" "-A"];
+
+ servicegroups = "webstatus-dns";
+ _webstatus_name = "saison-photo.org (Secondary DNS ClouDNS)";
+ }
+
+ #### Web scenarios
+ {
+ service_description = "blog website is running on immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "www.immae.eu" "/blog/" "egrep -ri TODO /etc"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "Immae’s Blog";
+ _webstatus_url = "https://www.immae.eu/blog";
+ }
+
+ {
+ service_description = "owncloud website is running on cloud.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "cloud.immae.eu" "/" "a safe home for all your data"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Nextcloud";
+ _webstatus_url = "https://cloud.immae.eu";
+ }
+
+ {
+ service_description = "davical website is running on dav.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "dav.immae.eu" "/davical/" "Log On Please"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Davical";
+ _webstatus_url = "https://dav.immae.eu/davical";
+ }
+
+ {
+ service_description = "adminer website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https_auth" "tools.immae.eu" "/adminer/" "www.adminer.org"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Adminer";
+ _webstatus_url = "https://tools.immae.eu/adminer/";
+ }
+
+ {
+ service_description = "ttrss website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "tools.immae.eu" "/ttrss/" "<title>Tiny Tiny RSS"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "TT-RSS";
+ _webstatus_url = "https://tools.immae.eu/ttrss/";
+ }
+
+ {
+ service_description = "mpd website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https_auth" "tools.immae.eu" "/mpd/" "<title>ympd"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "MPD (YMPD)";
+ _webstatus_url = "https://tools.immae.eu/mpd/";
+ }
+
+ {
+ service_description = "rompr mpd website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https_auth" "tools.immae.eu" "/rompr/" "<title>RompЯ"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "MPD (ROMPR)";
+ _webstatus_url = "https://tools.immae.eu/rompr/";
+ }
+
+ {
+ service_description = "wallabag website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "tools.immae.eu" "/wallabag/" "<title>Bienvenue sur wallabag"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Wallabag";
+ _webstatus_url = "https://tools.immae.eu/wallabag/";
+ }
+
+ {
+ service_description = "yourl website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "tools.immae.eu" "/url/admin/" "<title>YOURLS"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "YOURLS";
+ _webstatus_url = "https://tools.immae.eu/url/admin/";
+ }
+
+ {
+ service_description = "roundcube website is running on mail.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https_code" "mail.immae.eu" "/roundcube/" "401" "<title>Roundcube"];
+
+ servicegroups = "webstatus-webapps,webstatus-email";
+ _webstatus_name = "Roundcube";
+ _webstatus_url = "https://mail.immae.eu/roundcube/";
+ }
+
+ {
+ service_description = "dokuwiki website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "tools.immae.eu" "/dokuwiki/" "<title>start"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Dokuwiki";
+ _webstatus_url = "https://tools.immae.eu/dokuwiki/";
+ }
+
+ {
+ service_description = "shaarli website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "tools.immae.eu" "/Shaarli/immae" "<title>Immae"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Shaarli";
+ _webstatus_url = "https://tools.immae.eu/Shaarli/";
+ }
+
+ {
+ service_description = "ldap website is running on tools.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "tools.immae.eu" "/ldap/" "<title>phpLDAPadmin"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "LDAP";
+ _webstatus_url = "https://tools.immae.eu/ldap/";
+ }
+
+ {
+ service_description = "gitweb website is running on git.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "git.immae.eu" "/" "git web interface"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Git";
+ _webstatus_url = "https://git.immae.eu/";
+ }
+
+ {
+ service_description = "mantisbt website is running on git.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "git.immae.eu" "/mantisbt/" "<title>My View - MantisBT"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Mantisbt";
+ _webstatus_url = "https://git.immae.eu/mantisbt";
+ }
+
+ {
+ service_description = "diaspora website is running on diaspora.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "diaspora.immae.eu" "/" "is the online social world where you are in control"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Diaspora";
+ _webstatus_url = "https://diaspora.immae.eu/";
+ }
+
+ {
+ service_description = "peertube website is running on peertube.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "peertube.immae.eu" "/" "<title>Immae’s PeerTube"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Peertube";
+ _webstatus_url = "https://peertube.immae.eu/";
+ }
+
+ {
+ service_description = "etherpad website is running on ether.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "ether.immae.eu" "/" "<title>Etherpad"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Etherpad";
+ _webstatus_url = "https://ether.immae.eu/";
+ }
+
+ {
+ service_description = "mediagoblin website is running on mgoblin.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "mgoblin.immae.eu" "/" "<title>GNU MediaGoblin"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Mediagoblin";
+ _webstatus_url = "https://mgoblin.immae.eu/";
+ }
+
+ {
+ service_description = "mastodon website is running on mastodon.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "mastodon.immae.eu" "/" "Mastodon</title>"];
+
+ servicegroups = "webstatus-webapps";
+ _webstatus_name = "Mastodon";
+ _webstatus_url = "https://mastodon.immae.eu/";
+ }
+
+ # Other web pages
+ {
+ service_description = "Jerome website is running on naturaloutil.immae.eu";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "naturaloutil.immae.eu" "/ping.php" "YES"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "naturaloutil.immae.eu";
+ _webstatus_url = "https://naturaloutil.immae.eu/";
+ }
+
+ {
+ service_description = "Telio website is running on realistesmedia.fr";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "realistesmedia.fr" "/" "<title>Réal'istes"];
+ contact_groups = "telio-tortay";
+ }
+
+ {
+ service_description = "Chloe website site is running on osteopathe-cc.fr";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "www.osteopathe-cc.fr" "/" "<title>Chloé Condamin ostéopathe D.O."];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "osteopathe-cc.fr";
+ _webstatus_url = "https://www.osteopathe-cc.fr/";
+ }
+
+ {
+ service_description = "Richie website is running on europe-richie.org";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "www.europe-richie.org" "/" "<title>.Europe Richie]"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "europe-richie.org";
+ _webstatus_url = "https://www.europe-richie.org/";
+ }
+
+ {
+ service_description = "Ludivine website site is running on ludivinecassal.com";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "www.ludivinecassal.com" "/" "<title>Ludivine Cassal"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "ludivinecassal.com";
+ _webstatus_url = "https://www.ludivinecassal.com/";
+ }
+
+ {
+ service_description = "Aten website site is running on aten.pro";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "aten.pro" "/" "<title>ATEN"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "aten.pro";
+ _webstatus_url = "https://aten.pro/";
+ }
+
+ {
+ service_description = "PiedsJaloux website site is running on piedsjaloux.fr";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "www.piedsjaloux.fr" "/" "<title>Les Pieds Jaloux"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "piedsjaloux.fr";
+ _webstatus_url = "https://www.piedsjaloux.fr/";
+ }
+
+ {
+ service_description = "Denise and Jerome website site is running on denisejerome.piedsjaloux.fr";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "denisejerome.piedsjaloux.fr" "/" "<title>Accueil - Mariage"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "denisejerome.piedsjaloux.fr";
+ _webstatus_url = "https://denisejerome.piedsjaloux.fr/";
+ }
+
+ {
+ service_description = "Connexionswing website site is running on connexionswing.com";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "www.connexionswing.com" "/" "<title>Cours, stages"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "connexionswing.com";
+ _webstatus_url = "https://www.connexionswing.com/";
+ }
+
+ {
+ service_description = "Sandetludo website site is running on sandetludo.com";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = ["check_https" "www.sandetludo.com" "/" "<title>Cours de West Coast"];
+
+ servicegroups = "webstatus-websites";
+ _webstatus_name = "sandetludo.com";
+ _webstatus_url = "https://www.sandetludo.com/";
+ }
+
+ {
+ service_description = "Maison bbc accepts add requests on http";
+ host_name = "eldiron.immae.eu";
+ use = "external-web-service";
+ check_command = [ "check_http" "maison.bbc.bouya.org" "/add.php" "^$"];
+ }
+
+ {
+ service_description = "Maison bbc has up to date data";
+ host_name = "eldiron.immae.eu";
+ use = "generic-service";
+ contact_groups = "maison-bbc";
+ notification_interval = "1440";
+ check_command = "check_maison_bbc";
+ flap_detection_enabled = "0";
+ max_check_attempts = "1";
+ }
+
+ # SSL
+ {
+ service_description = "ldap SSL is up to date";
+ host_name = "eldiron.immae.eu";
+ use = "external-service";
+ check_command = ["check_tcp_ssl" "636"];
+
+ servicegroups = "webstatus-ssl";
+ _webstatus_name = "LDAP";
+ _webstatus_url = "ldap.immae.eu";
+ }
+ ];
+ contact = {
+ telio-tortay = config.myEnv.monitoring.contacts.telio-tortay // {
+ use = "generic-contact";
+ contactgroups = "telio-tortay";
+ };
+ papa = config.myEnv.monitoring.contacts.papa // {
+ use = "generic-contact";
+ contactgroups = "maison-bbc";
+ service_notification_commands = "notify-maison-bbc-by-email";
+ };
+ };
+ contactgroup = {
+ maison-bbc = { alias = "Maison BBC"; };
+ telio-tortay = { alias = "Telio Tortay"; members = "immae"; };
+ };
+}
--- /dev/null
+{ ... }:
+{
+ host = {
+ "phare.normalesup.org" = {
+ alias = "phare.normalesup.org";
+ address = "phare.normalesup.org";
+ use = "linux-server";
+ hostgroups = "webstatus-hosts";
+ _webstatus_name = "phare";
+ };
+ };
+ service = [
+ {
+ service_description = "phare.normalesup.org email service is active";
+ use = "external-passive-service";
+ host_name = "phare.normalesup.org";
+ freshness_threshold = "1350";
+ retry_interval = "1";
+ servicegroups = "webstatus-email";
+ }
+ ];
+}
--- /dev/null
+{ ... }:
+{
+ host = {
+ "tiboqorl.fr" = {
+ alias = "tiboqorl.fr";
+ address = "tiboqorl.fr";
+ use = "linux-server";
+ hostgroups = "webstatus-hosts";
+ _webstatus_name = "tiboqorl";
+ };
+ };
+}
--- /dev/null
+{ ... }:
+{
+ host = {
+ "ulminfo.fr" = {
+ alias = "ulminfo.fr";
+ address = "ulminfo.fr";
+ use = "linux-server";
+ hostgroups = "webstatus-hosts";
+ _webstatus_name = "ulminfo";
+ };
+ };
+ service = [
+ {
+ service_description = "ulminfo.fr email service is active";
+ use = "external-passive-service";
+ host_name = "ulminfo.fr";
+ freshness_threshold = "1350";
+ retry_interval = "1";
+ servicegroups = "webstatus-email";
+ }
+ ];
+}
--- /dev/null
+#!/usr/bin/env bash
+
+OUT=$(echo "ls" | lftp -u test_ftp,test_ftp eldiron.immae.eu | grep it_works | wc -l)
+
+if [ "$OUT" -eq 1 ]; then
+ echo "ftp connection OK - access to ftp is working"
+ exit 0
+else
+ echo "ftp connection CRITICAL - no access to ftp"
+ exit 2
+fi
--- /dev/null
+#!/usr/bin/env bash
+
+SSH_KEY="$1"
+
+TMPDIR=$(mktemp -d)
+
+if [ ! -d "$TMPDIR" ]; then
+ echo "gitolite UNKNOWN - impossible to create temp dir"
+ exit 3
+fi
+
+trap "rm -rf $TMPDIR" EXIT
+
+ERRORS=""
+OUTPUT=""
+
+cd "$TMPDIR"
+OUT=$(git clone -q git://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1)
+ERR=$?
+if [ -n "$OUT" ]; then
+OUTPUT="$OUTPUT
+$OUT"
+fi
+if [ "$ERR" != 0 ]; then
+ ERRORS="$ERRORS git://"
+fi
+rm -rf Monitor
+
+OUT=$(git clone -q http://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1)
+ERR=$?
+if [ -n "$OUT" ]; then
+OUTPUT="$OUTPUT
+$OUT"
+fi
+if [ "$ERR" != 0 ]; then
+ ERRORS="$ERRORS http://"
+fi
+rm -rf Monitor
+
+OUT=$(git clone -q https://git.immae.eu/perso/Immae/Projets/Ruby/Monitor.git 2>&1)
+ERR=$?
+if [ -n "$OUT" ]; then
+OUTPUT="$OUTPUT
+$OUT"
+fi
+if [ "$ERR" != 0 ]; then
+ ERRORS="$ERRORS https://"
+fi
+rm -rf Monitor
+
+OUT=$(GIT_SSH_COMMAND="ssh -i $SSH_KEY -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no" git clone -q gitolite@git.immae.eu:perso/Immae/Projets/Ruby/Monitor 2>&1)
+ERR=$?
+if [ -n "$OUT" ]; then
+OUTPUT="$OUTPUT
+$OUT"
+fi
+if [ "$ERR" != 0 ]; then
+ ERRORS="$ERRORS ssh"
+fi
+rm -rf Monitor
+
+if [ -n "$ERRORS" ]; then
+ echo "gitolite CRITICAL - impossible to clone via$ERRORS|$OUTPUT"
+ exit 2
+else
+ echo "gitolite OK - ssh, git, http and https work|$OUTPUT"
+ exit 0
+fi
--- /dev/null
+#!/usr/bin/env perl
+
+use strict;
+use Getopt::Std;
+$| = 1;
+
+my %opts;
+getopts('h:u:p:H:', \%opts);
+
+my $STATE_OK = 0;
+my $STATE_WARNING = 1;
+my $STATE_CRITICAL = 2;
+my $STATE_UNKNOWN = 3;
+
+if ($opts{'h'} || !$opts{'u'} || !$opts{'p'} || !$opts{'H'}) {
+ &print_help();
+ exit($STATE_UNKNOWN);
+}
+
+my $user = $opts{'u'};
+my $password = $opts{'p'};
+my $host = $opts{'H'};
+
+my $cmd_result = `(echo "a login $user $password"; echo "b logout") | openssl s_client -quiet -ign_eof -connect $host -starttls imap 2>&1`;
+my $expected_result = "a OK Logged in";
+
+chomp($cmd_result);
+if ($cmd_result !~ /$expected_result/) {
+ print "IMAP CRITICAL - Unable to connect via imaps\n";
+ exit($STATE_CRITICAL);
+} else {
+ print "IMAP OK - imaps connected successfully\n";
+ exit($STATE_OK);
+}
+
+sub print_help() {
+ print << "EOF";
+Check whether imap works via ssl and is able to connect its database.
+
+Options:
+-h
+ Print detailed help screen
+-u
+ User to log in as
+-p
+ Password to log in
+-H
+ Host to log in to
+
+EOF
+}
+
--- /dev/null
+#!/usr/bin/env bash
+
+# kw, kwh, th_hr Bureau, th_hr Chambre_RDC, th_hr Exterieure, th_hr
+# Garage, th_hr Mezzanine, th_hr Sejour, th_hr Soufflage
+
+alert=$(date +"%F %H:%M:%S" -d "3 hours ago")
+res=$(curl https://maison.bbc.bouya.org/stats.php)
+
+ignored_text="th_hr Exterieure th_hr Soufflage"
+
+ignored_bat="^$"
+ignored="^(th_hr Exterieure|th_hr Soufflage)$"
+
+missing() {
+ echo "to_entries|[.[]|select(.value < \"$alert\" and .key != \"bat\" $1)|.key + \": \" + .value]|join(\" ; \")"
+}
+
+missing_check=$(echo "$res" | jq -r "$(missing "and (.key | test(\"$ignored\") | not)")")
+missing_show=$(echo "$res" | jq -r "$(missing "")")
+
+battery() {
+ echo ".bat|to_entries|[.[]|select((.value[0] < \"$alert\" or (.value[1] | test(\"^0|1|2$\") | not)) $1)|.key + \": \" + .value[0] + \" bat \" + .value[1]]|join(\" ; \")"
+}
+
+bat_check=$(echo "$res" | jq -r "$(battery "and (.key | test(\"$ignored_bat\") | not)")")
+bat_show=$(echo "$res" | jq -r "$(battery "")")
+
+if [ -n "$missing_check" -o -n "$bat_check" ]; then
+ text=""
+ if [ -n "$missing_show" ]; then
+ text="$text - Dernières dates: $missing_show"
+ fi
+ if [ -n "$bat_show" ]; then
+ text="$text - Batteries: $bat_show"
+ fi
+ echo "Zibase CRITICAL$text"
+ exit 2
+else
+ echo "Zibase OK - Derniers événements il y a moins de 3h (ignorés : $ignored_text)"
+ exit 0
+fi
subject="** $NOTIFICATION_TYPE Service Alert: $HOST_ALIAS/$SERVICEDESC is $SERVICESTATE **"
fi
-# sendwait waits for sendmail to finish its job, otherwise it continues in the
-# background and gets killed too early
-echo "$message" | MAILRC=/dev/null mail -r "naemon@immae.eu" -n -Ssendwait -s "$subject" "$CONTACT"
+echo "$message" | MAILRC=/dev/null mail -r "$ADMINEMAIL" -n -s "$subject" "$CONTACT"
--- /dev/null
+#!/usr/bin/env bash
+
+SLACK_CHANNEL="$1"
+SLACK_USERNAME="Naemon"
+SLACK_URL="$2"
+SLACK_USERICON="https://release.immae.eu/monitoring.png"
+
+if [ "$SERVICESTATE" = "CRITICAL" ]; then
+ ICON=":exclamation:"
+ COLOR="#DA0505"
+elif [ "$SERVICESTATE" = "WARNING" ]; then
+ ICON=":warning:"
+ COLOR="#F1E903"
+elif [ "$SERVICESTATE" = "OK" ]; then
+ ICON=":white_check_mark:"
+ COLOR="#36a64f"
+elif [ "$SERVICESTATE" = "UNKNOWN" ]; then
+ ICON=":question:"
+ COLOR="#000000"
+else
+ ICON=":white_medium_square:"
+ COLOR="#ffffff"
+fi
+
+payload=$(echo "{}" | jq -r \
+ --arg "icon_url" "$SLACK_USERICON" \
+ --arg "channel" "$SLACK_CHANNEL" \
+ --arg "username" "$SLACK_USERNAME" \
+ --arg "text" "${ICON} ${SERVICEDESC} on ${HOST} is ${SERVICESTATE}" \
+ --arg "color" "$COLOR" \
+ --arg "host" "$HOST" \
+ --arg "desc" "$SERVICEDESC" \
+ --arg "state" "$SERVICESTATE" \
+ --arg "output" "$SERVICEOUTPUT" \
+ '.icon_url = $icon_url |
+ .channel = $channel |
+ .username = $username |
+ .text = $text |
+ .attachments = [{fallback:"", color:$color,fields: [{},{},{},{}]}] |
+ .attachments[0].fields[0] = {title:"Host",value:$host,short:true} |
+ .attachments[0].fields[1] = {title:"Service description",value:$desc,short:true} |
+ .attachments[0].fields[2] = {title:"Status",value:$state,short:true} |
+ .attachments[0].fields[3] = {title:"Message",value:$output,short:false}
+ ')
+
+curl -X POST --data "payload=$payload" $SLACK_URL
--- /dev/null
+#!/usr/bin/env bash
+
+# $1 = service/host
+
+# $2 = type (PROBLEM RECOVERY ACKNOWLEDGEMENT FLAPPINGSTART FLAPPINGSTOP FLAPPINGDISABLED DOWNTIMESTART DOWNTIMESTOP DOWNTIMECANCELLED)
+# http://www.naemon.org/documentation/usersguide/macrolist.html#notificationtype
+
+# $3 = host alias
+
+# $4 = date (YYYY-MM-DDTHH:MM:SS)
+
+# $5 = E-mail
+
+NOTIFICATION_TYPE="$2"
+DATE="$4"
+CONTACT="$5"
+
+output=$(echo "$SERVICEOUTPUT" | awk '{gsub(" - ", "\n\n", $0); gsub (" (;|:) ", "\n ", $0); gsub(" dates: ", " dates:\n ", $0); gsub("Batteries: ", "Batteries:\n ", $0); print $0}')
+if [ "$NOTIFICATION_TYPE" = "RECOVERY" ]; then
+ subject="Fin d'alerte Maison BBC Zibase"
+ message=$(printf "%b" "Alerte terminée le $DATE.\nInformations:\n$output")
+else
+ subject="Alerte Maison BBC Zibase"
+ message=$(printf "%b" "Alerte lancée le $DATE.\nInformations:\n$output")
+fi
+
+echo "$message" | MAILRC=/dev/null mail -r "$ADMINEMAIL" -n -s "$subject" "$CONTACT"
define service {
${builtins.concatStringsSep "\n" (mapAttrsToList (k: v:
" ${pad 30 k} ${toStr k v}"
- ) service)}
+ ) (filterAttrs (k: v: ! builtins.elem k ["passiveInfo" "filter"]) service))}
}
'';
toServices = services: builtins.concatStringsSep "\n" (map toService services);
then toCommands v
else if keyname == "templates"
then toTemplates v
- else if builtins.elem keyname ["host" "contactgroup" "contact" "timeperiod"]
+ else if builtins.elem keyname ["hostgroup" "host" "contactgroup" "contact" "timeperiod" "servicegroup"]
then toOthers keyname v
else "";
toObjects = v: builtins.concatStringsSep "\n" (mapAttrsToList toObjects' v);
--- /dev/null
+{ privateFiles }:
+{ config, pkgs, resources, ... }:
+{
+ boot.kernelPackages = pkgs.linuxPackages_latest;
+ myEnv = import "${privateFiles}/environment.nix" // { inherit privateFiles; };
+ hostEnv.FQDN = "monitoring-1.v.immae.eu";
+
+ imports = builtins.attrValues (import ../..);
+
+ deployment = {
+ targetEnv = "hetznerCloud";
+ hetznerCloud = {
+ authToken = config.myEnv.hetznerCloud.authToken;
+ datacenter = "hel1-dc2";
+ location ="hel1";
+ serverType = "cx11";
+ };
+ };
+
+ myServices.monitoring.enable = true;
+ myServices.monitoring.master = true;
+ myServices.status.enable = true;
+ networking = {
+ firewall.enable = true;
+ interfaces."ens3".ipv4.addresses = pkgs.lib.attrsets.mapAttrsToList
+ (n: ips: { address = ips.ip4; prefixLength = 32; })
+ (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.myEnv.servers.monitoring-1.ips);
+ interfaces."ens3".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
+ (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
+ config.myEnv.servers.monitoring-1.ips);
+ defaultGateway6 = { address = "fe80::1"; interface = "ens3"; };
+ };
+ myServices.mailRelay.enable = true;
+
+ # This value determines the NixOS release with which your system is
+ # to be compatible, in order to avoid breaking some software such as
+ # database servers. You should change this only after NixOS release
+ # notes say you should.
+ # https://nixos.org/nixos/manual/release-notes.html
+ system.stateVersion = "19.03"; # Did you read the comment?
+}
ssh-backup-2:
$(NIXOPS_PRIV) ssh backup-2 -- $(SSH_ARGS)
+ssh-monitoring-1:
+ $(NIXOPS_PRIV) ssh monitoring-1 -- $(SSH_ARGS)
+
info:
$(NIXOPS_PRIV) list
$(NIXOPS_PRIV) info
delete-generations:
nix-env -p $(profile) --delete-generations $(GEN)
$(NIXOPS_PRIV) ssh eldiron -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)
+ $(NIXOPS_PRIV) ssh backup-2 -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)
+ $(NIXOPS_PRIV) ssh monitoring-1 -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)
.PHONY: delete-generations
cleanup: delete-generations
nix-store --gc
$(NIXOPS_PRIV) ssh eldiron -- nix-store --gc
+ $(NIXOPS_PRIV) ssh backup-2 -- nix-store --gc
+ $(NIXOPS_PRIV) ssh monitoring-1 -- nix-store --gc
.PHONY: cleanup
###### Pull environment and deployment from remote
resources.sshKeyPairs.ssh-key = {};
eldiron = import ../modules/private/system/eldiron.nix { inherit privateFiles; };
backup-2 = import ../modules/private/system/backup-2.nix { inherit privateFiles; };
+ monitoring-1 = import ../modules/private/system/monitoring-1.nix { inherit privateFiles; };
}
projects / perso / Immae / Config / Nix.git / commitdiff
