[perso/Immae/Config/Nix.git] / modules / private / monitoring / default.nix

{ config, pkgs, lib, name, nodes, ... }:
let
  cfg = config.myServices.monitoring;
  activatedPlugins = [ "memory" "command" "bandwidth" ]
    ++ (if cfg.master then (masterObjects.activatedPlugins or []) else [])
    ++ (if cfg.master then (lib.flatten (map (v: v.activatedPlugins or []) otherObjects)) else [])
    ++ (hostObjects.activatedPlugins or [])
    ++ (if cfg.master then ["notify-primary"] else ["notify-secondary"]);
  allPluginsConfig = import ./myplugins.nix {
    inherit pkgs lib config;
    sudo = "/run/wrappers/bin/sudo";
  };
  mypluginsConfig = lib.getAttrs activatedPlugins allPluginsConfig;
  myplugins = let
    mypluginsChunk = builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList (k: v: v.chunk or "") mypluginsConfig);
  in pkgs.runCommand "buildplugins" {
    buildInputs = [ pkgs.makeWrapper pkgs.perl ];
  } ''
    mkdir $out
    ${mypluginsChunk}
    '';
  toObjects = pkgs.callPackage ./to_objects.nix {};
  commonConfig = {
    dilion = {
      processWarn = "250"; processAlert = "400";
      loadWarn = "1.0"; loadAlert = "1.2";
      interface = "eth0";
    };
    eldiron = {
      processWarn = "250"; processAlert = "400";
      loadWarn = "1.0"; loadAlert = "1.2";
      interface = "eth0";
    };
    backup-2 = {
      processWarn = "60"; processAlert = "70";
      loadWarn = "1.0"; loadAlert = "2.0";
      interface = "ens3";
    };
    monitoring-1 = {
      processWarn = "50"; processAlert = "60";
      loadWarn = "4.0"; loadAlert = "6.0";
      load15Warn = "1.0"; load15Alert = "2.0";
      interface = "ens3";
    };
    quatresaisons = {
      processWarn = "250"; processAlert = "400";
      loadWarn = "1.0"; loadAlert = "1.2";
      interface = "eth0";
    };
  };
  externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
    (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
  masterPassiveObjects = let
    passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
    toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
    passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
      (_: n: toPassiveServices n.config.myServices.monitoring.services)
      passiveNodes
      ) ++ lib.flatten (lib.attrsets.mapAttrsToList
      (_: n: toPassiveServices n.service)
      externalObjects);
  in {
    service = passiveServices;
    host = lib.lists.foldr
      (a: b: a//b)
      {}
      (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
      ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
  };
  emailCheck = host: hostFQDN: let
    allCfg = config.myEnv.monitoring.email_check;
    cfg = allCfg."${host}";
    reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
    to_email = cfg': host':
      let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
      in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
    mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
    mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
    command = if cfg.local
    then
      [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
    else
      [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
  in
    {
      service_description = "${hostFQDN} email service is active";
      use = "mail-service";
      host_name = hostFQDN;
      servicegroups = "webstatus-email";
      check_command = command;
    };
  otherObjects = map
    (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
    [ "ulminfo-fr" "phare" "eban" ];
  masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
  commonObjects = pkgs.callPackage ./objects_common.nix ({
    master = cfg.master;
    hostFQDN = config.hostEnv.fqdn;
    hostName = name;
    inherit mypluginsConfig;
  } // builtins.getAttr name commonConfig);
  hostObjects =
    let
      specific_file = ./. + "/objects_" + name + ".nix";
    in
      lib.attrsets.optionalAttrs
        (builtins.pathExists specific_file)
        (pkgs.callPackage specific_file {
          inherit config nodes emailCheck;
          hostFQDN = config.hostEnv.fqdn;
          hostName = name;
        });
  objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
    "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
  ) externalObjects;
in
{
  options = {
    myServices.monitoring = {
      enable = lib.mkOption {
        type = lib.types.bool;
        default = false;
        description = ''
          Whether to enable monitoring.
        '';
      };
      master = lib.mkOption {
        type = lib.types.bool;
        default = false;
        description = ''
          This instance is the master instance
        '';
      };
      hosts = lib.mkOption {
        readOnly = true;
        description = "Hosts list for this host";
        default = (commonObjects.host or {}) // (hostObjects.host or {});
      };
      services = lib.mkOption {
        readOnly = true;
        description = "Services list for this host";
        default = commonObjects.service ++ hostObjects.service;
      };
    };
  };

  config = lib.mkIf cfg.enable {
    services.nginx = lib.mkIf config.myServices.status.enable {
      virtualHosts."status.immae.eu".locations = objectsFiles // {
        "=/common/immae.cfg" = {
          alias = pkgs.writeText "immae.cfg" ''
            # put me for instance in /etc/naemon/module-conf.d/immae.cfg
            # Make sure that you have include_dir=module-conf.d in
            # naemon.cfg
            log_initial_states=1
            date_format=iso8601
            admin_email=${config.myEnv.monitoring.email}
            obsess_over_services=1
            ocsp_command=notify-master
          '';
        };
        "=/common/resource.cfg" = {
          alias = pkgs.writeText "resource.cfg" ''
            # Resource.cfg file
            # Replace this with path to monitoring plugins
            $USER1$=@@COMMON_PLUGINS@@
            # Replace this with a path to scripts from
            # https://git.immae.eu/cgit/perso/Immae/Config/Nix.git/tree/modules/private/monitoring/plugins
            $USER2$=@@IMMAE_PLUGINS@@
            $USER200$=https://status.immae.eu/
            $USER201$=@@TOKEN@@
          '';
        };
      };
    };

    services.duplyBackup.profiles.monitoring = {
      rootDir = config.services.naemon.varDir;
    };
    security.sudo.extraRules = let
      pluginsSudo = lib.lists.remove null (lib.attrsets.mapAttrsToList (k: v:
        if (v ? sudo)
        then ({ users = [ "naemon" ]; } // (v.sudo myplugins))
        else null) mypluginsConfig);
    in [
      {
        commands = [
          { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
          { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
        ];
        users = [ "naemon" ];
        runAs = "root";
      }
    ] ++ pluginsSudo;
    environment.etc."mdadm.conf" = {
      enable = true;
      mode = "0644";
      user = "root";
      text = "MAILADDR ${config.myEnv.monitoring.email}";
    };

    secrets.keys = [
      {
        dest = "naemon/id_rsa";
        user = "naemon";
        group = "naemon";
        permissions = "0400";
        text = config.myEnv.monitoring.ssh_secret_key;
      }
    ] ++ lib.optionals cfg.master (
      lib.mapAttrsToList (k: v:
      {
        dest = "${k}_access_key";
        user = "naemon";
        group = "naemon";
        permissions = "0400";
        text = ''
          export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
          export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
          export BASE_URL="${v.remote "immae-eldiron"}"
        '';
      }) config.myEnv.backup.remotes
    );
    # needed since extraResource is not in the closure
    systemd.services.naemon.path = [ myplugins ];
    services.naemon = {
      enable = true;
      extraConfig = ''
        use_syslog=1
        log_initial_states=1
        date_format=iso8601
        admin_email=${config.myEnv.monitoring.email}
      '' + lib.optionalString (!cfg.master) ''
        obsess_over_services=1
        ocsp_command=notify-master
      '' + lib.optionalString (cfg.master) ''
        broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
        broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
      '';
      extraResource = let
        resources = [hostObjects.resources or {}] ++ (lib.mapAttrsToList (k: v: v.resources or {}) mypluginsConfig);
        joined = lib.zipAttrsWith (n: v: if builtins.length (lib.unique v) == 1 then builtins.head v else abort "Non-unique resources names") resources;
        joinedStr = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "$" + "${k}$=${v}") joined);
      in ''
        $USER2$=${myplugins}
        ${joinedStr}
      '';
      objectDefs = toObjects commonObjects
        + toObjects hostObjects
        + lib.optionalString cfg.master (toObjects masterObjects)
        + lib.optionalString cfg.master (toObjects masterPassiveObjects)
        + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
    };
  };
}
Commit	Line	Data
e820134d	1	{ config, pkgs, lib, name, nodes, ... }:
3bc32d9e	2	let
e820134d	3	cfg = config.myServices.monitoring;
acab8301 IB	4	activatedPlugins = [ "memory" "command" "bandwidth" ]
	5	++ (if cfg.master then (masterObjects.activatedPlugins or []) else [])
	6	++ (if cfg.master then (lib.flatten (map (v: v.activatedPlugins or []) otherObjects)) else [])
	7	++ (hostObjects.activatedPlugins or [])
	8	++ (if cfg.master then ["notify-primary"] else ["notify-secondary"]);
	9	allPluginsConfig = import ./myplugins.nix {
	10	inherit pkgs lib config;
	11	sudo = "/run/wrappers/bin/sudo";
6ee77836	12	};
acab8301 IB	13	mypluginsConfig = lib.getAttrs activatedPlugins allPluginsConfig;
	14	myplugins = let
	15	mypluginsChunk = builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList (k: v: v.chunk or "") mypluginsConfig);
	16	in pkgs.runCommand "buildplugins" {
3bc32d9e IB	17	buildInputs = [ pkgs.makeWrapper pkgs.perl ];
	18	} ''
	19	mkdir $out
acab8301	20	${mypluginsChunk}
3bc32d9e	21	'';
eb071dd4 IB	22	toObjects = pkgs.callPackage ./to_objects.nix {};
eb071dd4 IB	23	commonConfig = {
6ee77836 IB	24	dilion = {
6ee77836 IB	25	processWarn = "250"; processAlert = "400";
c41d0de8	26	loadWarn = "1.0"; loadAlert = "1.2";
6ee77836 IB	27	interface = "eth0";
6ee77836 IB	28	};
eb071dd4 IB	29	eldiron = {
eb071dd4 IB	30	processWarn = "250"; processAlert = "400";
c41d0de8	31	loadWarn = "1.0"; loadAlert = "1.2";
2d7caffb	32	interface = "eth0";
eb071dd4 IB	33	};
eb071dd4 IB	34	backup-2 = {
25844101	35	processWarn = "60"; processAlert = "70";
eb071dd4	36	loadWarn = "1.0"; loadAlert = "2.0";
2d7caffb	37	interface = "ens3";
eb071dd4	38	};
e820134d IB	39	monitoring-1 = {
e820134d IB	40	processWarn = "50"; processAlert = "60";
7ad4966f IB	41	loadWarn = "4.0"; loadAlert = "6.0";
7ad4966f IB	42	load15Warn = "1.0"; load15Alert = "2.0";
2d7caffb	43	interface = "ens3";
e820134d	44	};
6ee77836 IB	45	quatresaisons = {
6ee77836 IB	46	processWarn = "250"; processAlert = "400";
c41d0de8	47	loadWarn = "1.0"; loadAlert = "1.2";
6ee77836 IB	48	interface = "eth0";
6ee77836 IB	49	};
eb071dd4	50	};
46a61a1b IB	51	externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
46a61a1b IB	52	(n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
e820134d	53	masterPassiveObjects = let
6ee77836	54	passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
e820134d IB	55	toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
	56	passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
	57	(_: n: toPassiveServices n.config.myServices.monitoring.services)
	58	passiveNodes
46a61a1b IB	59	) ++ lib.flatten (lib.attrsets.mapAttrsToList
	60	(_: n: toPassiveServices n.service)
	61	externalObjects);
e820134d	62	in {
71a2425e	63	service = passiveServices;
e820134d IB	64	host = lib.lists.foldr
	65	(a: b: a//b)
	66	{}
46a61a1b IB	67	(lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
46a61a1b IB	68	++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
e820134d	69	};
71a2425e IB	70	emailCheck = host: hostFQDN: let
	71	allCfg = config.myEnv.monitoring.email_check;
	72	cfg = allCfg."${host}";
	73	reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
	74	to_email = cfg': host':
	75	let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
	76	in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
	77	mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
	78	mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
	79	command = if cfg.local
	80	then
	81	[ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
	82	else
	83	[ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
	84	in
	85	{
	86	service_description = "${hostFQDN} email service is active";
	87	use = "mail-service";
	88	host_name = hostFQDN;
	89	servicegroups = "webstatus-email";
	90	check_command = command;
	91	};
	92	otherObjects = map
	93	(n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
2edbb2d8	94	[ "ulminfo-fr" "phare" "eban" ];
e820134d	95	masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
eb071dd4	96	commonObjects = pkgs.callPackage ./objects_common.nix ({
e820134d	97	master = cfg.master;
619e4f46	98	hostFQDN = config.hostEnv.fqdn;
e820134d	99	hostName = name;
acab8301	100	inherit mypluginsConfig;
eb071dd4 IB	101	} // builtins.getAttr name commonConfig);
	102	hostObjects =
	103	let
	104	specific_file = ./. + "/objects_" + name + ".nix";
9f202523	105	in
e820134d IB	106	lib.attrsets.optionalAttrs
	107	(builtins.pathExists specific_file)
	108	(pkgs.callPackage specific_file {
171d8e1a	109	inherit config nodes emailCheck;
619e4f46	110	hostFQDN = config.hostEnv.fqdn;
e820134d IB	111	hostName = name;
e820134d IB	112	});
46a61a1b IB	113	objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
	114	"=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
	115	) externalObjects;
3bc32d9e IB	116	in
	117	{
	118	options = {
9f202523 IB	119	myServices.monitoring = {
	120	enable = lib.mkOption {
	121	type = lib.types.bool;
	122	default = false;
	123	description = ''
	124	Whether to enable monitoring.
	125	'';
	126	};
e820134d IB	127	master = lib.mkOption {
	128	type = lib.types.bool;
	129	default = false;
	130	description = ''
	131	This instance is the master instance
	132	'';
	133	};
	134	hosts = lib.mkOption {
	135	readOnly = true;
	136	description = "Hosts list for this host";
	137	default = (commonObjects.host or {}) // (hostObjects.host or {});
	138	};
	139	services = lib.mkOption {
	140	readOnly = true;
	141	description = "Services list for this host";
	142	default = commonObjects.service ++ hostObjects.service;
	143	};
3bc32d9e IB	144	};
	145	};
	146
e820134d	147	config = lib.mkIf cfg.enable {
46a61a1b	148	services.nginx = lib.mkIf config.myServices.status.enable {
a0e80453 IB	149	virtualHosts."status.immae.eu".locations = objectsFiles // {
	150	"=/common/immae.cfg" = {
	151	alias = pkgs.writeText "immae.cfg" ''
	152	# put me for instance in /etc/naemon/module-conf.d/immae.cfg
	153	# Make sure that you have include_dir=module-conf.d in
	154	# naemon.cfg
	155	log_initial_states=1
	156	date_format=iso8601
	157	admin_email=${config.myEnv.monitoring.email}
	158	obsess_over_services=1
	159	ocsp_command=notify-master
	160	'';
	161	};
	162	"=/common/resource.cfg" = {
	163	alias = pkgs.writeText "resource.cfg" ''
	164	# Resource.cfg file
	165	# Replace this with path to monitoring plugins
	166	$USER1$=@@COMMON_PLUGINS@@
	167	# Replace this with a path to scripts from
	168	# https://git.immae.eu/cgit/perso/Immae/Config/Nix.git/tree/modules/private/monitoring/plugins
	169	$USER2$=@@IMMAE_PLUGINS@@
	170	$USER200$=https://status.immae.eu/
	171	$USER201$=@@TOKEN@@
	172	'';
	173	};
	174	};
46a61a1b IB	175	};
46a61a1b IB	176
d2e703c5	177	services.duplyBackup.profiles.monitoring = {
6a8252b1 IB	178	rootDir = config.services.naemon.varDir;
6a8252b1 IB	179	};
acab8301 IB	180	security.sudo.extraRules = let
	181	pluginsSudo = lib.lists.remove null (lib.attrsets.mapAttrsToList (k: v:
	182	if (v ? sudo)
	183	then ({ users = [ "naemon" ]; } // (v.sudo myplugins))
	184	else null) mypluginsConfig);
	185	in [
3bc32d9e IB	186	{
	187	commands = [
	188	{ command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
	189	{ command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
	190	];
	191	users = [ "naemon" ];
	192	runAs = "root";
	193	}
acab8301	194	] ++ pluginsSudo;
3bc32d9e IB	195	environment.etc."mdadm.conf" = {
	196	enable = true;
	197	mode = "0644";
	198	user = "root";
ab8f306d	199	text = "MAILADDR ${config.myEnv.monitoring.email}";
3bc32d9e IB	200	};
3bc32d9e IB	201
e820134d IB	202	secrets.keys = [
	203	{
	204	dest = "naemon/id_rsa";
	205	user = "naemon";
	206	group = "naemon";
171d8e1a	207	permissions = "0400";
e820134d IB	208	text = config.myEnv.monitoring.ssh_secret_key;
e820134d IB	209	}
5a61f6ad IB	210	] ++ lib.optionals cfg.master (
5a61f6ad IB	211	lib.mapAttrsToList (k: v:
171d8e1a	212	{
5a61f6ad	213	dest = "${k}_access_key";
171d8e1a IB	214	user = "naemon";
	215	group = "naemon";
	216	permissions = "0400";
	217	text = ''
5a61f6ad IB	218	export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
	219	export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
	220	export BASE_URL="${v.remote "immae-eldiron"}"
171d8e1a	221	'';
5a61f6ad	222	}) config.myEnv.backup.remotes
171d8e1a	223	);
3bc32d9e IB	224	# needed since extraResource is not in the closure
	225	systemd.services.naemon.path = [ myplugins ];
	226	services.naemon = {
	227	enable = true;
	228	extraConfig = ''
3bc32d9e IB	229	use_syslog=1
	230	log_initial_states=1
	231	date_format=iso8601
ab8f306d	232	admin_email=${config.myEnv.monitoring.email}
e820134d	233	'' + lib.optionalString (!cfg.master) ''
3bc32d9e IB	234	obsess_over_services=1
3bc32d9e IB	235	ocsp_command=notify-master
a97118c4 IB	236	'' + lib.optionalString (cfg.master) ''
	237	broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
	238	broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
3bc32d9e	239	'';
acab8301	240	extraResource = let
e64a4968	241	resources = [hostObjects.resources or {}] ++ (lib.mapAttrsToList (k: v: v.resources or {}) mypluginsConfig);
acab8301 IB	242	joined = lib.zipAttrsWith (n: v: if builtins.length (lib.unique v) == 1 then builtins.head v else abort "Non-unique resources names") resources;
	243	joinedStr = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "$" + "${k}$=${v}") joined);
	244	in ''
3bc32d9e	245	$USER2$=${myplugins}
acab8301	246	${joinedStr}
3bc32d9e	247	'';
e820134d IB	248	objectDefs = toObjects commonObjects
	249	+ toObjects hostObjects
	250	+ lib.optionalString cfg.master (toObjects masterObjects)
71a2425e IB	251	+ lib.optionalString cfg.master (toObjects masterPassiveObjects)
71a2425e IB	252	+ lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
3bc32d9e IB	253	};
	254	};
	255	}