]> git.immae.eu Git - perso/Immae/Config/Nix.git/blob - flakes/private/monitoring/flake.nix
Migrate to borg backup
[perso/Immae/Config/Nix.git] / flakes / private / monitoring / flake.nix
1 {
2 inputs = {
3 environment.url = "path:../environment";
4 secrets.url = "path:../../secrets";
5 naemon.url = "path:../../naemon";
6 nixpkgs-lib.url = "github:NixOS/nixpkgs?dir=lib";
7 };
8 outputs = { self, environment, nixpkgs-lib, secrets, naemon }: {
9 nagios-cli-config = ./nagios-cli.cfg;
10 lib = rec {
11 expandedObject = kind: object: objects:
12 if object ? "use"
13 then expandedObject kind objects.templates.${kind}.${object.use} objects // object
14 else object;
15
16 objectsCommon = import ./objects_common.nix;
17 toObjects = import ./to_objects.nix { inherit (nixpkgs-lib) lib; };
18
19 toMasterPassiveObject = svcTemplate: freshnessThresholdMultiplier: objects:
20 {
21 service = with nixpkgs-lib.lib; map (s:
22 {
23 host_name = (expandedObject "service" s objects).host_name;
24 use = svcTemplate;
25 retry_interval = "1";
26 freshness_threshold = let
27 fs = expandedObject "service" s objects;
28 in if builtins.isInt fs.check_interval
29 then builtins.ceil (freshnessThresholdMultiplier * 60 * fs.check_interval)
30 else fs.check_interval;
31 }
32 // filterAttrs (k: v: builtins.elem k ["service_description"] || builtins.substring 0 1 k == "_") s
33 // mapAttrs'
34 (n: nameValuePair (removePrefix "__passive_" n))
35 (filterAttrs (k: _: hasPrefix "__passive_" k) s)
36 ) objects.service;
37 host = objects.host;
38 };
39
40 emailCheck = allCfg: host: hostFQDN: let
41 cfg = allCfg."${host}";
42 reverseTargets = builtins.attrNames (nixpkgs-lib.lib.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
43 to_email = cfg': host':
44 let sep = if nixpkgs-lib.lib.hasInfix "+" cfg'.mail_address then "_" else "+";
45 in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
46 mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
47 mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
48 command = if cfg.local
49 then
50 [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
51 else
52 [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
53 in
54 {
55 service_description = "${hostFQDN} email service is active";
56 use = "mail-service";
57 host_name = hostFQDN;
58 servicegroups = "webstatus-email";
59 check_command = command;
60 };
61 };
62 nixosModule = self.nixosModules.monitoring;
63 nixosModules.monitoring = { config, pkgs, lib, ... }:
64 let
65 cfg = config.myServices.monitoring;
66 allPluginsConfig = import ./myplugins.nix {
67 inherit pkgs lib config;
68 sudo = "/run/wrappers/bin/sudo";
69 };
70 mypluginsConfig = lib.mapAttrs (n: v:
71 if builtins.isFunction v
72 then v (cfg.pluginsArgs."${n}" or {})
73 else v
74 ) (lib.getAttrs cfg.activatedPlugins allPluginsConfig);
75 myplugins = let
76 mypluginsChunk = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: v.chunk or "") mypluginsConfig);
77 in pkgs.runCommand "buildplugins" {
78 buildInputs = [ pkgs.makeWrapper pkgs.perl ];
79 } ''
80 mkdir $out
81 ${mypluginsChunk}
82 '';
83 objectsModule = with lib.types; submodule {
84 options = {
85 command = lib.mkOption {
86 type = attrsOf str;
87 default = {};
88 description = "Command definitions";
89 };
90
91 host = lib.mkOption {
92 type = attrsOf (attrsOf str);
93 default = {};
94 description = "Host definitions";
95 };
96 hostgroup = lib.mkOption {
97 type = attrsOf (attrsOf str);
98 default = {};
99 description = "Host group definitions";
100 };
101 hostdependency = lib.mkOption {
102 type = listOf (attrsOf str);
103 default = [];
104 description = "Host dependency definitions";
105 };
106
107 service = lib.mkOption {
108 type = listOf (attrsOf (oneOf [ str (listOf str) int ]));
109 # str -> string
110 # listOf str -> list to be concatenated with "!"
111 # int -> toString
112 default = [];
113 description = "Service definitions";
114 };
115 servicegroup = lib.mkOption {
116 type = attrsOf (attrsOf str);
117 default = {};
118 description = "Service group definitions";
119 };
120 servicedependency = lib.mkOption {
121 type = listOf (attrsOf str);
122 default = [];
123 description = "Service dependency definitions";
124 };
125
126 contact = lib.mkOption {
127 type = attrsOf (attrsOf str);
128 default = {};
129 description = "Contact definitions";
130 };
131 contactgroup = lib.mkOption {
132 type = attrsOf (attrsOf str);
133 default = {};
134 description = "Contact group definitions";
135 };
136
137 timeperiod = lib.mkOption {
138 type = attrsOf (attrsOf str);
139 default = {};
140 description = "Time period definitions";
141 };
142
143 templates = lib.mkOption {
144 description = "Template definitions";
145 default = {};
146 type = submodule {
147 options = {
148 service = lib.mkOption { type = attrsOf (attrsOf (either str int)); default = {}; };
149 contact = lib.mkOption { type = attrsOf (attrsOf str); default = {}; };
150 host = lib.mkOption { type = attrsOf (attrsOf str); default = {}; };
151 };
152 };
153 };
154 };
155 };
156 in
157 {
158 options = {
159 myServices.monitoring = {
160 enable = lib.mkOption {
161 type = lib.types.bool;
162 default = false;
163 description = ''
164 Whether to enable monitoring.
165 '';
166 };
167 smartdDisks = lib.mkOption {
168 type = lib.types.listOf lib.types.str;
169 default = [];
170 description = ''
171 List of smartd disks ids (symlinks in /dev/disk/by-id/) to monitor
172 '';
173 };
174 master = lib.mkOption {
175 type = lib.types.bool;
176 default = false;
177 description = ''
178 This instance is the master instance
179 '';
180 };
181 pluginsArgs = lib.mkOption {
182 default = {};
183 description = "Arguments to pass to the naemon plugin configuration";
184 type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified);
185 };
186 activatedPlugins = lib.mkOption {
187 default = [];
188 description = "List of naemon plugins to activate";
189 type = lib.types.listOf (lib.types.enum (builtins.attrNames allPluginsConfig));
190 };
191 fromMasterActivatedPlugins = lib.mkOption {
192 default = [];
193 description = "List of naemon plugins to activate from master";
194 type = lib.types.listOf (lib.types.str);
195 };
196 resources = lib.mkOption {
197 default = {};
198 description = "List of additionnal resources elements";
199 type = lib.types.attrsOf (lib.types.str);
200 };
201 objects = lib.mkOption {
202 default = {};
203 description = "Object definitions";
204 type = objectsModule;
205 };
206 fromMasterObjects = lib.mkOption {
207 default = {};
208 description = "Object definitions of checks that should be executed from master";
209 type = objectsModule;
210 };
211 };
212 };
213
214 imports = [
215 environment.nixosModule
216 secrets.nixosModule
217 naemon.nixosModule
218 ];
219 config = lib.mkIf cfg.enable {
220 myServices.monitoring.objects.command =
221 lib.foldr (v: o: o // (v.commands or {})) {} (builtins.attrValues mypluginsConfig);
222
223 security.sudo.extraRules = let
224 pluginsSudo = lib.lists.remove null (lib.mapAttrsToList (k: v:
225 if (v ? sudo)
226 then ({ users = [ "naemon" ]; } // (v.sudo myplugins))
227 else null) mypluginsConfig);
228 in pluginsSudo;
229
230 environment.etc.cnagios.source = "${pkgs.cnagios}/share/doc/cnagios";
231 environment.systemPackages = let
232 nagios-cli = pkgs.writeScriptBin "nagios-cli" ''
233 #!${pkgs.stdenv.shell}
234 sudo -u naemon ${pkgs.nagios-cli}/bin/nagios-cli -c ${self.nagios-cli-config}
235 '';
236 in [
237 pkgs.cnagios
238 nagios-cli
239 ];
240 secrets.keys = {
241 "naemon/id_rsa" = {
242 user = "naemon";
243 group = "naemon";
244 permissions = "0400";
245 text = config.myEnv.monitoring.ssh_secret_key;
246 };
247 "naemon/resources.cfg".keyDependencies = [ myplugins ];
248 };
249 services.naemon = {
250 enable = true;
251 extraConfig = ''
252 use_syslog=1
253 log_initial_states=1
254 date_format=iso8601
255 admin_email=${config.myEnv.monitoring.email}
256 '' + lib.optionalString (!cfg.master) ''
257 obsess_over_services=1
258 ocsp_command=notify-master
259 '';
260 extraResource = let
261 resources = [cfg.resources or {}] ++ (lib.mapAttrsToList (k: v: v.resources or {}) mypluginsConfig);
262 joined = lib.zipAttrsWith (n: v: if builtins.length (lib.unique v) == 1 then builtins.head v else abort "Non-unique resources names") resources;
263 joinedStr = builtins.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "$" + "${k}$=${v}") joined);
264 in ''
265 $USER2$=${myplugins}
266 ${joinedStr}
267 '';
268 objectDefs =
269 self.lib.toObjects cfg.objects;
270 };
271
272 myServices.monitoring.objects.service = builtins.map (d: {
273 service_description = "Disk /dev/disk/by-id/${d} is sane";
274 use = "local-service";
275 check_command = [ "check_smartctl" "/dev/disk/by-id/${d}" ];
276 __passive_servicegroups = "webstatus-resources";
277
278 check_interval = 60;
279 }) cfg.smartdDisks;
280
281 systemd = let
282 checkShortTimer = {
283 timerConfig = {
284 OnCalendar = "monthly";
285 RandomizedDelaySec = "3 weeks";
286 FixedRandomDelay = true;
287 };
288 wantedBy = [ "timers.target" ];
289 };
290 checkLongTimer = {
291 timerConfig = {
292 OnCalendar = "monthly";
293 RandomizedDelaySec = "3 weeks";
294 FixedRandomDelay = true;
295 };
296 wantedBy = [ "timers.target" ];
297 };
298 toSDTimers = id: {
299 "check-smartd-long-${id}" = checkLongTimer;
300 "check-smartd-short-${id}" = checkShortTimer;
301 };
302 toCheckService = id: type: {
303 description = "Run ${type} smartctl test for /dev/disk/by-id/${id}";
304 after = [ "multi-user.target" ];
305 serviceConfig = {
306 Type = "oneshot";
307 ExecStart = "${pkgs.smartmontools}/bin/smartctl -t ${type} /dev/disk/by-id/${id}";
308 };
309 };
310 toSDServices = id: {
311 "check-smartd-long-${id}" = toCheckService id "long";
312 "check-smartd-short-${id}" = toCheckService id "short";
313 };
314
315 in {
316 services = lib.attrsets.mergeAttrsList (builtins.map toSDServices cfg.smartdDisks);
317 timers = lib.attrsets.mergeAttrsList (builtins.map toSDTimers cfg.smartdDisks);
318 };
319 };
320 };
321 };
322 }