{ config, pkgs, lib, name, nodes, ... }:
let
cfg = config.myServices.monitoring;
+ send_mails = pkgs.runCommand "send_mails" {
+ buildInputs = [ pkgs.makeWrapper ];
+ } ''
+ mkdir -p $out/bin
+ cp ${./send_mails} $out/bin/send_mails
+ patchShebangs $out
+ wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
+ pkgs.mailutils
+ ]}
+ '';
+ postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable
+ then config.myServices.databasesReplication.postgresql.mainPackage
+ else if config.myServices.databases.enable
+ then config.myServices.databases.postgresql.package
+ else pkgs.postgresql;
+ zfsPlugin = pkgs.fetchurl {
+ url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh";
+ sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf";
+ };
+ megacli = pkgs.megacli.overrideAttrs(old: { meta = old.meta // { license = null; }; });
+ megaCliPlugin = pkgs.runCommand "megaCliPlugin" {
+ plugin = pkgs.fetchurl {
+ name = "check_megaraid_sas";
+ url = "https://exchange.nagios.org/components/com_mtree/attachment.php?link_id=6381&cf_id=24";
+ sha256 = "0yf60p4c0hb4q3fng9fc14qc89bqm0f1sijayzygadaqcl44jx4p";
+ };
+ } ''
+ mkdir $out
+ cp $plugin $out/check_megaraid_sas
+ chmod +x $out/check_megaraid_sas
+ patchShebangs $out
+ substituteInPlace $out/check_megaraid_sas --replace /usr/sbin/MegaCli ${megacli}/bin/MegaCli64
+ substituteInPlace $out/check_megaraid_sas --replace 'sudo $megacli' '/run/wrappers/bin/sudo $megacli'
+ sed -i -e "s/use utils qw(%ERRORS);/my %ERRORS = ('OK' => 0, 'WARNING' => 1, 'CRITICAL' => 2, 'UNKNOWN' => 3);/" $out/check_megaraid_sas
+ '';
myplugins = pkgs.runCommand "buildplugins" {
buildInputs = [ pkgs.makeWrapper pkgs.perl ];
} ''
mkdir $out
+ cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh
+ cp ${megaCliPlugin}/check_megaraid_sas $out/
cp ${./plugins}/* $out/
patchShebangs $out
wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
- wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
- pkgs.curl pkgs.jq
+ wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [
+ pkgs.which pkgs.zfs pkgs.gawk
]}
- wrapProgram $out/check_maison_bbc --prefix PATH : ${lib.makeBinPath [
+ wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
pkgs.curl pkgs.jq
]}
wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
pkgs.gnugrep pkgs.gawk pkgs.procps-ng
]}
wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
- pkgs.postgresql
+ postgresqlBinary
]}
wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [
pkgs.gnugrep pkgs.coreutils pkgs.redis
wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
]}
+ wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
+ pkgs.openssh send_mails
+ ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
+ pkgs.perlPackages.TimeDate
+ ]}
wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
pkgs.lftp
]}
wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
pkgs.s3cmd pkgs.python3
]}
- wrapProgram $out/notify_maison_bbc_by_email --prefix PATH : ${lib.makeBinPath [
- pkgs.mailutils pkgs.gawk
- ]}
+ makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
+ pkgs.duplicity
+ ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
+ makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
+ pkgs.duplicity
+ ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
pkgs.mailutils
]}
wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
pkgs.curl pkgs.jq
]}
+ wrapProgram $out/notify_eban_url --prefix PATH : ${lib.makeBinPath [
+ pkgs.curl
+ ]}
+ wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
+ (pkgs.python3.withPackages (ps: [ps.ovh]))
+ ]}
+ wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
+ pkgs.iproute pkgs.bc
+ ]}
'';
toObjects = pkgs.callPackage ./to_objects.nix {};
commonConfig = {
+ dilion = {
+ processWarn = "250"; processAlert = "400";
+ loadWarn = "8.0"; loadAlert = "10.0";
+ interface = "eth0";
+ };
eldiron = {
processWarn = "250"; processAlert = "400";
loadWarn = "8.0"; loadAlert = "10.0";
+ interface = "eth0";
};
backup-2 = {
processWarn = "60"; processAlert = "70";
loadWarn = "1.0"; loadAlert = "2.0";
+ interface = "ens3";
};
monitoring-1 = {
processWarn = "50"; processAlert = "60";
- loadWarn = "1.0"; loadAlert = "2.0";
+ loadWarn = "4.0"; loadAlert = "6.0";
+ load15Warn = "1.0"; load15Alert = "2.0";
+ interface = "ens3";
+ };
+ quatresaisons = {
+ processWarn = "250"; processAlert = "400";
+ loadWarn = "8.0"; loadAlert = "10.0";
+ interface = "eth0";
};
};
+ externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
+ (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
masterPassiveObjects = let
- otherPassiveObjects = map
- (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") {}))
- [ "caldance-1" "ulminfo-fr" "immae-eu" "phare" "tiboqorl-fr" ];
- otherPassiveServices = lib.flatten (map (h: h.service or []) otherPassiveObjects);
- otherPassiveHosts = (map (h: h.host)) otherPassiveObjects;
- passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron"]) nodes;
+ passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
(_: n: toPassiveServices n.config.myServices.monitoring.services)
passiveNodes
- );
+ ) ++ lib.flatten (lib.attrsets.mapAttrsToList
+ (_: n: toPassiveServices n.service)
+ externalObjects);
in {
- service = passiveServices ++ otherPassiveServices;
+ service = passiveServices;
host = lib.lists.foldr
(a: b: a//b)
{}
- (otherPassiveHosts ++ lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes);
+ (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
+ ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
};
+ emailCheck = host: hostFQDN: let
+ allCfg = config.myEnv.monitoring.email_check;
+ cfg = allCfg."${host}";
+ reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
+ to_email = cfg': host':
+ let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
+ in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
+ mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
+ mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
+ command = if cfg.local
+ then
+ [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
+ else
+ [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
+ in
+ {
+ service_description = "${hostFQDN} email service is active";
+ use = "mail-service";
+ host_name = hostFQDN;
+ servicegroups = "webstatus-email";
+ check_command = command;
+ };
+ otherObjects = map
+ (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
+ [ "ulminfo-fr" "phare" "eban" ];
masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
commonObjects = pkgs.callPackage ./objects_common.nix ({
master = cfg.master;
lib.attrsets.optionalAttrs
(builtins.pathExists specific_file)
(pkgs.callPackage specific_file {
- inherit config;
+ inherit config nodes emailCheck;
hostFQDN = config.hostEnv.fqdn;
hostName = name;
});
+ objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
+ "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
+ ) externalObjects;
in
{
options = {
};
config = lib.mkIf cfg.enable {
+ services.nginx = lib.mkIf config.myServices.status.enable {
+ virtualHosts."status.immae.eu".locations = objectsFiles;
+ };
+
services.duplyBackup.profiles.monitoring = {
rootDir = config.services.naemon.varDir;
};
commands = [
{ command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
{ command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
+ { command = "${megacli}/bin/MegaCli64"; options = [ "NOPASSWD" ]; }
];
users = [ "naemon" ];
runAs = "root";
dest = "naemon/id_rsa";
user = "naemon";
group = "naemon";
- premissions = "0400";
+ permissions = "0400";
text = config.myEnv.monitoring.ssh_secret_key;
}
- ];
+ ] ++ lib.optionals cfg.master (
+ lib.mapAttrsToList (k: v:
+ {
+ dest = "${k}_access_key";
+ user = "naemon";
+ group = "naemon";
+ permissions = "0400";
+ text = ''
+ export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
+ export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
+ export BASE_URL="${v.remote "immae-eldiron"}"
+ '';
+ }) config.myEnv.backup.remotes
+ );
# needed since extraResource is not in the closure
systemd.services.naemon.path = [ myplugins ];
services.naemon = {
enable = true;
extraConfig = ''
- broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
use_syslog=1
log_initial_states=1
date_format=iso8601
'' + lib.optionalString (!cfg.master) ''
obsess_over_services=1
ocsp_command=notify-master
+ '' + lib.optionalString (cfg.master) ''
+ broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
+ broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
'';
extraResource = ''
$USER2$=${myplugins}
$USER206$=${config.myEnv.monitoring.slack_channel}
$USER207$=${config.myEnv.monitoring.slack_url}
$USER208$=${builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys)}
+ $USER209$=${builtins.concatStringsSep "," [
+ config.myEnv.monitoring.ovh_sms.endpoint
+ config.myEnv.monitoring.ovh_sms.application_key
+ config.myEnv.monitoring.ovh_sms.application_secret
+ config.myEnv.monitoring.ovh_sms.consumer_key
+ config.myEnv.monitoring.ovh_sms.account
+ ]}
+ $USER210$=${config.myEnv.monitoring.eban.password}
'';
objectDefs = toObjects commonObjects
+ toObjects hostObjects
+ lib.optionalString cfg.master (toObjects masterObjects)
- + lib.optionalString cfg.master (toObjects masterPassiveObjects);
+ + lib.optionalString cfg.master (toObjects masterPassiveObjects)
+ + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
};
};
}
projects / perso / Immae / Config / Nix.git / blobdiff