diff options
Diffstat (limited to 'modules/private/monitoring')
-rw-r--r-- | modules/private/monitoring/status.nix | 61 | ||||
-rwxr-xr-x | modules/private/monitoring/status/app.py | 409 |
2 files changed, 470 insertions, 0 deletions
diff --git a/modules/private/monitoring/status.nix b/modules/private/monitoring/status.nix new file mode 100644 index 0000000..ed4d681 --- /dev/null +++ b/modules/private/monitoring/status.nix | |||
@@ -0,0 +1,61 @@ | |||
1 | { config, pkgs, lib, name, ... }: | ||
2 | { | ||
3 | options = { | ||
4 | myServices.status = { | ||
5 | enable = lib.mkOption { | ||
6 | type = lib.types.bool; | ||
7 | default = false; | ||
8 | description = '' | ||
9 | Whether to enable status app. | ||
10 | ''; | ||
11 | }; | ||
12 | }; | ||
13 | }; | ||
14 | config = lib.mkIf config.myServices.status.enable { | ||
15 | secrets.keys = [ | ||
16 | { | ||
17 | dest = "naemon-status/environment"; | ||
18 | user = "naemon"; | ||
19 | group = "naemon"; | ||
20 | permission = "0400"; | ||
21 | text = '' | ||
22 | TOKENS=${builtins.concatStringsSep " " config.myEnv.monitoring.nrdp_tokens} | ||
23 | ''; | ||
24 | } | ||
25 | ]; | ||
26 | services.nginx = { | ||
27 | enable = true; | ||
28 | recommendedOptimisation = true; | ||
29 | recommendedGzipSettings = true; | ||
30 | recommendedProxySettings = true; | ||
31 | virtualHosts."status.immae.eu" = { | ||
32 | useACMEHost = name; | ||
33 | forceSSL = true; | ||
34 | locations."/".proxyPass = "http://unix:/run/naemon-status/socket.sock:/"; | ||
35 | }; | ||
36 | }; | ||
37 | security.acme.certs."${name}".extraDomains."status.immae.eu" = null; | ||
38 | |||
39 | myServices.certificates.enable = true; | ||
40 | networking.firewall.allowedTCPPorts = [ 80 443 18000 ]; | ||
41 | systemd.services.naemon-status = { | ||
42 | description = "Naemon status"; | ||
43 | after = [ "network.target" ]; | ||
44 | wantedBy = [ "multi-user.target" ]; | ||
45 | |||
46 | serviceConfig = { | ||
47 | EnvironmentFile = config.secrets.fullPaths."naemon-status/environment"; | ||
48 | Type = "simple"; | ||
49 | WorkingDirectory = "${./status}"; | ||
50 | ExecStart = let | ||
51 | python = pkgs.python3.withPackages (p: [ p.gunicorn p.flask p.flask_login ]); | ||
52 | in | ||
53 | "${python}/bin/gunicorn -w4 --bind unix:/run/naemon-status/socket.sock app:app"; | ||
54 | User = "naemon"; | ||
55 | RuntimeDirectory = "naemon-status"; | ||
56 | StandardOutput = "journal"; | ||
57 | StandardError = "inherit"; | ||
58 | }; | ||
59 | }; | ||
60 | }; | ||
61 | } | ||
diff --git a/modules/private/monitoring/status/app.py b/modules/private/monitoring/status/app.py new file mode 100755 index 0000000..b1d419c --- /dev/null +++ b/modules/private/monitoring/status/app.py | |||
@@ -0,0 +1,409 @@ | |||
1 | from flask import Flask, request, render_template_string, jsonify, make_response | ||
2 | from flask_login import LoginManager, UserMixin, login_required | ||
3 | import socket | ||
4 | import json | ||
5 | import time | ||
6 | import os | ||
7 | |||
8 | login_manager = LoginManager() | ||
9 | app = Flask(__name__) | ||
10 | login_manager.init_app(app) | ||
11 | |||
12 | STATUS = [ | ||
13 | "ok", | ||
14 | "warning", | ||
15 | "error", | ||
16 | "unknown" | ||
17 | ] | ||
18 | |||
19 | HOST_STATUS = [ | ||
20 | "up", | ||
21 | "down", | ||
22 | "unreachable", | ||
23 | ] | ||
24 | |||
25 | #### Push | ||
26 | AUTHORIZED_KEYS = os.environ.get("TOKENS", "").split() | ||
27 | COMMAND_FILE = "/var/run/naemon/naemon.cmd" | ||
28 | |||
29 | ERROR_NO_REQUEST_HANDLER="NO REQUEST HANDLER" | ||
30 | ERROR_NO_TOKEN_SUPPLIED="NO TOKEN" | ||
31 | ERROR_BAD_TOKEN_SUPPLIED="BAD TOKEN" | ||
32 | |||
33 | ERROR_BAD_COMMAND_FILE="BAD COMMAND FILE" | ||
34 | ERROR_COMMAND_FILE_OPEN_WRITE="COMMAND FILE UNWRITEABLE" | ||
35 | ERROR_COMMAND_FILE_OPEN="CANNOT OPEN COMMAND FILE" | ||
36 | ERROR_BAD_WRITE="WRITE ERROR" | ||
37 | |||
38 | ERROR_BAD_DATA="BAD DATA" | ||
39 | ERROR_BAD_JSON="BAD JSON" | ||
40 | |||
41 | ERROR_NO_CORRECT_STATUS="NO STATUS WAS CORRECT" | ||
42 | #### /Push | ||
43 | |||
44 | def get_lq(request): | ||
45 | # https://mathias-kettner.de/checkmk_livestatus.html | ||
46 | socket_path="/var/run/naemon/live" | ||
47 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | ||
48 | s.connect(socket_path) | ||
49 | s.send(request.encode()) | ||
50 | s.shutdown(socket.SHUT_WR) | ||
51 | chunks = [] | ||
52 | while len(chunks) == 0 or len(chunks[-1]) > 0: | ||
53 | chunks.append(s.recv(4096)) | ||
54 | s.close() | ||
55 | return b"".join(chunks).decode() | ||
56 | |||
57 | class Host: | ||
58 | def __init__(self, name, alias, status, webname): | ||
59 | self.name = name | ||
60 | self.alias = alias | ||
61 | self.webname = webname or alias | ||
62 | self.status = status | ||
63 | self.services = [] | ||
64 | |||
65 | @classmethod | ||
66 | def parse_hosts(cls, payload): | ||
67 | parsed = [cls.parse(p) for p in json.loads(payload)] | ||
68 | return {p.name: p for p in parsed} | ||
69 | |||
70 | @classmethod | ||
71 | def parse(cls, payload): | ||
72 | return cls(payload[0], payload[1], HOST_STATUS[payload[2]], payload[3].get("WEBSTATUS_NAME")) | ||
73 | |||
74 | def __repr__(self): | ||
75 | return "Host {}: {} ({})".format(self.name, self.alias, self.webname) | ||
76 | |||
77 | @classmethod | ||
78 | def query(cls): | ||
79 | answer = get_lq("""GET hosts | ||
80 | Filter: groups >= webstatus-hosts | ||
81 | Columns: name alias state custom_variables | ||
82 | OutputFormat: json | ||
83 | """) | ||
84 | return cls.parse_hosts(answer) | ||
85 | |||
86 | def fill_services(self, services): | ||
87 | self.services = [service for service in services if service.host == self.name] | ||
88 | |||
89 | class ServiceGroup: | ||
90 | def __init__(self, name, alias): | ||
91 | self.name = name | ||
92 | self.alias = alias | ||
93 | self.services = [] | ||
94 | |||
95 | @classmethod | ||
96 | def parse_groups(cls, payload): | ||
97 | parsed = [cls.parse(p) for p in json.loads(payload)] | ||
98 | return {p.name: p for p in parsed} | ||
99 | |||
100 | @classmethod | ||
101 | def parse(cls, payload): | ||
102 | return cls(payload[0], payload[1]) | ||
103 | |||
104 | @classmethod | ||
105 | def query(cls): | ||
106 | answer = get_lq("""GET servicegroups | ||
107 | Filter: name ~ ^webstatus- | ||
108 | Columns: name alias custom_variables | ||
109 | OutputFormat: json | ||
110 | """) | ||
111 | return cls.parse_groups(answer) | ||
112 | |||
113 | def fill_services(self, services): | ||
114 | self.services = [service for service in services if any([group == self.name for group in service.groups])] | ||
115 | |||
116 | def __repr__(self): | ||
117 | return "ServiceGroup {}: {}".format(self.name, self.alias) | ||
118 | |||
119 | class Service: | ||
120 | def __init__(self, name, host, groups, status, webname, url, description, infos): | ||
121 | self.name = name | ||
122 | self.host = host | ||
123 | self.groups = groups | ||
124 | self.status = status | ||
125 | self.webname = webname | ||
126 | self.url = url | ||
127 | self.description = description | ||
128 | self.infos = infos | ||
129 | |||
130 | @classmethod | ||
131 | def parse_services(cls, payload): | ||
132 | parsed = json.loads(payload) | ||
133 | return [cls.parse(p) for p in parsed if cls.valid(p[2])] | ||
134 | |||
135 | @staticmethod | ||
136 | def valid(groups): | ||
137 | return any([b.startswith("webstatus-") for b in groups]) | ||
138 | |||
139 | @classmethod | ||
140 | def parse(cls, payload): | ||
141 | return cls(payload[0], | ||
142 | payload[1], | ||
143 | payload[2], | ||
144 | STATUS[payload[3]], | ||
145 | payload[4].get("WEBSTATUS_NAME"), | ||
146 | payload[4].get("WEBSTATUS_URL"), | ||
147 | payload[5], | ||
148 | payload[6]) | ||
149 | |||
150 | @classmethod | ||
151 | def query(cls): | ||
152 | answer = get_lq("""GET services | ||
153 | Columns: display_name host_name groups state custom_variables description plugin_output | ||
154 | OutputFormat: json | ||
155 | """) | ||
156 | return cls.parse_services(answer) | ||
157 | |||
158 | def __repr__(self): | ||
159 | return "Service {}: {}".format(self.name, self.webname) | ||
160 | |||
161 | def get_infos(): | ||
162 | hosts = Host.query() | ||
163 | servicegroups = ServiceGroup.query() | ||
164 | services = Service.query() | ||
165 | |||
166 | for host in hosts: | ||
167 | hosts[host].fill_services(services) | ||
168 | for group in servicegroups: | ||
169 | servicegroups[group].fill_services(services) | ||
170 | return (hosts, servicegroups, services) | ||
171 | |||
172 | TEMPLATE='''<?xml version="1.0" encoding="UTF-8"?> | ||
173 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> | ||
174 | <html lang="en" xmlns="http://www.w3.org/1999/xhtml"> | ||
175 | <head> | ||
176 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> | ||
177 | <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
178 | <title>Status</title> | ||
179 | <meta name="referrer" content="no-referrer" /> | ||
180 | <style type="text/css"> | ||
181 | ul { | ||
182 | list-style: none; | ||
183 | margin: 0px; | ||
184 | } | ||
185 | ul li:nth-child(2n) { | ||
186 | background-color: rgb(240, 240, 240); | ||
187 | } | ||
188 | li.resource, li.service { | ||
189 | margin: 1px 0px; | ||
190 | } | ||
191 | span.status { | ||
192 | display: inline-block; | ||
193 | width: 150px; | ||
194 | text-align: center; | ||
195 | margin-right: 5px; | ||
196 | font-variant: small-caps; | ||
197 | font-size: 1.2em; | ||
198 | } | ||
199 | .status_ok,.status_up { | ||
200 | background-color: rgba(0, 255, 0, 0.5);; | ||
201 | } | ||
202 | .status_warning { | ||
203 | background-color: rgba(255, 255, 0, 0.5);; | ||
204 | } | ||
205 | .status_error,.status_down { | ||
206 | background-color: rgba(255, 0, 0, 0.5);; | ||
207 | } | ||
208 | .status_unknown,.status_unreachable { | ||
209 | background-color: rgba(0, 0, 255, 0.5);; | ||
210 | } | ||
211 | .infos { | ||
212 | margin-left: 40px; | ||
213 | color: rgb(100, 100, 100); | ||
214 | } | ||
215 | div#services { | ||
216 | column-count: auto; | ||
217 | column-width: 36em; | ||
218 | } | ||
219 | div.servicegroup { | ||
220 | -webkit-column-break-inside: avoid; | ||
221 | break-inside: avoid; | ||
222 | } | ||
223 | h3.servicegroup_title, h3.host_title { | ||
224 | margin: 1px 0px; | ||
225 | } | ||
226 | span.service_host, span.infos { | ||
227 | float: right; | ||
228 | display: inline-block; | ||
229 | color: rgb(100, 100, 100); | ||
230 | } | ||
231 | </style> | ||
232 | </head> | ||
233 | <body> | ||
234 | <h2>Hosts</h2> | ||
235 | {%- for host in hosts.values() %} | ||
236 | <h3 class="host_title"> | ||
237 | <span class="status status_{{ host.status }}">{{ host.status }}</span> | ||
238 | <span class="host">{{ host.webname }}</span> | ||
239 | </h3> | ||
240 | {%- for service in servicegroups["webstatus-resources"].services if service.host == host.name -%} | ||
241 | {%- if loop.first %} | ||
242 | <ul class="resources"> | ||
243 | {% endif %} | ||
244 | |||
245 | <li class="resource"> | ||
246 | <span class="status status_{{ service.status }}">{{ service.status }}</span> | ||
247 | <span class="description">{{ service.description }}</span> | ||
248 | <span class="infos">{{ service.infos }}</span> | ||
249 | </li> | ||
250 | |||
251 | {%- if loop.last %} | ||
252 | </ul> | ||
253 | {% endif %} | ||
254 | {% endfor %} | ||
255 | {%- endfor %} | ||
256 | |||
257 | <h2>Services</h2> | ||
258 | <div id="services"> | ||
259 | {%- for group in servicegroups.values() if group.services and group.name != "webstatus-resources" %} | ||
260 | <div class="servicegroup"> | ||
261 | <h3 class="servicegroup_title">{{ group.alias }}</h3> | ||
262 | {%- for service in group.services -%} | ||
263 | {%- if loop.first %} | ||
264 | <ul class="services"> | ||
265 | {% endif %} | ||
266 | |||
267 | <li class="service" title="{{ service.infos }}"> | ||
268 | <span class="status status_{{ service.status }}">{{ service.status }}</span> | ||
269 | <span class="description"> | ||
270 | {% if service.url and service.url.startswith("https://") %} | ||
271 | <a href="{{ service.url }}">{{ service.webname or service.description }}</a> | ||
272 | {% else %} | ||
273 | {{ service.webname or service.description }} | ||
274 | {% endif %} | ||
275 | </span> | ||
276 | <span class="service_host">{{ hosts[service.host].webname }}</span> | ||
277 | </li> | ||
278 | |||
279 | {%- if loop.last %} | ||
280 | </ul> | ||
281 | {% endif %} | ||
282 | {%- endfor -%} | ||
283 | </div> | ||
284 | {%- endfor %} | ||
285 | </div> | ||
286 | </body> | ||
287 | </html> | ||
288 | ''' | ||
289 | |||
290 | @login_manager.request_loader | ||
291 | def load_user_from_request(request): | ||
292 | api_key = request.headers.get('Token') | ||
293 | if api_key in AUTHORIZED_KEYS: | ||
294 | return UserMixin() | ||
295 | content = request.get_json(force=True, silent=True) | ||
296 | if content is not None and content.get("token") in AUTHORIZED_KEYS: | ||
297 | return UserMixin() | ||
298 | |||
299 | @app.route("/live", methods=["POST"]) | ||
300 | @login_required | ||
301 | def live(): | ||
302 | query = request.get_data() | ||
303 | result = get_lq(query.decode() + "\n") | ||
304 | resp = make_response(result) | ||
305 | resp.content_type = "text/plain" | ||
306 | return resp | ||
307 | |||
308 | @app.route("/", methods=["GET"]) | ||
309 | def get(): | ||
310 | (hosts, servicegroups, services) = get_infos() | ||
311 | resp = make_response(render_template_string(TEMPLATE, hosts=hosts, servicegroups=servicegroups)) | ||
312 | resp.content_type = "text/html" | ||
313 | return resp | ||
314 | |||
315 | @app.route("/", methods=["POST"]) | ||
316 | @login_required | ||
317 | def push(): | ||
318 | content = request.get_json(force=True, silent=True) | ||
319 | if content is None: | ||
320 | return ERROR_BAD_JSON | ||
321 | if content.get("cmd") != "submitcheck": | ||
322 | return render_error(ERROR_NO_REQUEST_HANDLER) | ||
323 | if "checkresult" not in content or not isinstance(content["checkresult"], list): | ||
324 | return render_error(ERROR_BAD_DATA) | ||
325 | |||
326 | checks = 0 | ||
327 | errors = 0 | ||
328 | for check in map(lambda x: CheckResult.from_json(x), content["checkresult"]): | ||
329 | if check is None: | ||
330 | errors += 1 | ||
331 | continue | ||
332 | try: | ||
333 | write_check_output(check) | ||
334 | except Exception as e: | ||
335 | return render_error(str(e)) | ||
336 | checks += 1 | ||
337 | return render_response(checks, errors) | ||
338 | |||
339 | def write_check_output(check): | ||
340 | if check.type== "service": | ||
341 | command = "[{time}] PROCESS_SERVICE_CHECK_RESULT;{hostname};{servicename};{state};{output}"; | ||
342 | else: | ||
343 | command = "[{time}] PROCESS_HOST_CHECK_RESULT;{hostname};{state};{output}"; | ||
344 | formatted = command.format( | ||
345 | time=int(time.time()), | ||
346 | hostname=check.hostname, | ||
347 | state=check.state, | ||
348 | output=check.output, | ||
349 | servicename=check.servicename, | ||
350 | ) | ||
351 | |||
352 | if not os.path.exists(COMMAND_FILE): | ||
353 | raise Exception(ERROR_BAD_COMMAND_FILE) | ||
354 | if not os.access(COMMAND_FILE, os.W_OK): | ||
355 | raise Exception(ERROR_COMMAND_FILE_OPEN_WRITE) | ||
356 | if not os.access(COMMAND_FILE, os.W_OK): | ||
357 | raise Exception(ERROR_COMMAND_FILE_OPEN_WRITE) | ||
358 | try: | ||
359 | with open(COMMAND_FILE, "w") as c: | ||
360 | c.write(formatted + "\n") | ||
361 | except Exception as e: | ||
362 | raise Exception(ERROR_BAD_WRITE) | ||
363 | |||
364 | def render_error(error): | ||
365 | return jsonify({ | ||
366 | "status": "error", | ||
367 | "message": error, | ||
368 | }) | ||
369 | |||
370 | def render_response(checks, errors): | ||
371 | if checks > 0: | ||
372 | return jsonify({ | ||
373 | "status": "ok", | ||
374 | "result": { | ||
375 | "checks": checks, | ||
376 | "errors": errors, | ||
377 | } | ||
378 | }) | ||
379 | else: | ||
380 | return jsonify({ | ||
381 | "status": "error", | ||
382 | "message": ERROR_NO_CORRECT_STATUS, | ||
383 | }) | ||
384 | |||
385 | class CheckResult: | ||
386 | def __init__(self, hostname, state, output, servicename, checktype): | ||
387 | self.hostname = hostname | ||
388 | self.state = state | ||
389 | self.output = output | ||
390 | self.servicename = servicename | ||
391 | self.type = checktype | ||
392 | |||
393 | @classmethod | ||
394 | def from_json(klass, j): | ||
395 | if not isinstance(j, dict): | ||
396 | return None | ||
397 | for key in ["hostname", "state", "output"]: | ||
398 | if key not in j or not isinstance(j[key], str): | ||
399 | return None | ||
400 | for key in ["servicename", "type"]: | ||
401 | if key in j and not isinstance(j[key], str): | ||
402 | return None | ||
403 | return klass( | ||
404 | j["hostname"], | ||
405 | j["state"], | ||
406 | j["output"], | ||
407 | j.get("servicename", ""), | ||
408 | j.get("type", "host")) | ||
409 | |||