diff options
Diffstat (limited to 'systems/zoldene/logging.nix')
-rw-r--r-- | systems/zoldene/logging.nix | 138 |
1 files changed, 138 insertions, 0 deletions
diff --git a/systems/zoldene/logging.nix b/systems/zoldene/logging.nix new file mode 100644 index 0000000..09ee104 --- /dev/null +++ b/systems/zoldene/logging.nix | |||
@@ -0,0 +1,138 @@ | |||
1 | { config, pkgs, name, ... }: | ||
2 | # Initialization | ||
3 | # CREATE INDEX ON LOGS (tag); | ||
4 | # CREATE INDEX ON LOGS (time); | ||
5 | # CREATE INDEX ON LOGS (((data->>'PRIORITY')::int)); | ||
6 | # CREATE INDEX ON LOGS ((data->>'_SYSTEMD_UNIT')); | ||
7 | # CREATE INDEX ON LOGS ((data->>'SYSLOG_IDENTIFIER')); | ||
8 | let | ||
9 | fluent-bit-config = { | ||
10 | pipeline = { | ||
11 | inputs = [ | ||
12 | { | ||
13 | name = "systemd"; | ||
14 | tag = "${name}.systemd"; | ||
15 | DB = "/var/lib/fluentbit/fluent-bit.db"; | ||
16 | } | ||
17 | ]; | ||
18 | outputs = [ | ||
19 | { | ||
20 | name = "loki"; | ||
21 | match = "${name}.systemd"; | ||
22 | line_format = "json"; | ||
23 | labels = "job=fluentbit, server=${name}, priority=$PRIORITY, syslog_identifier=$SYSLOG_IDENTIFIER, systemd_unit=$_SYSTEMD_UNIT"; | ||
24 | } | ||
25 | { | ||
26 | name = "pgsql"; | ||
27 | match = "*"; | ||
28 | host = "/run/postgresql"; | ||
29 | user = "fluentbit"; | ||
30 | table = "logs"; | ||
31 | database = "fluentbit"; | ||
32 | timestamp_key = "event_timestamp"; | ||
33 | } | ||
34 | ]; | ||
35 | }; | ||
36 | }; | ||
37 | yamlFormat = pkgs.formats.yaml {}; | ||
38 | psqlVersion = pkgs.postgresql_13.psqlSchema; | ||
39 | in | ||
40 | { | ||
41 | disko.devices.zpool.zfast.datasets."root/persist/var/lib/loki" = | ||
42 | { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/loki"; options.mountpoint = "legacy"; }; | ||
43 | disko.devices.zpool.zfast.datasets."root/persist/var/lib/fluentbit" = | ||
44 | { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/fluentbit"; options.mountpoint = "legacy"; }; | ||
45 | disko.devices.zpool.zfast.datasets."root/persist/var/lib/postgresql" = | ||
46 | { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/postgresql"; options.mountpoint = "legacy"; }; | ||
47 | disko.devices.zpool.zfast.datasets."root/persist/var/lib/postgresql/${psqlVersion}" = | ||
48 | { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/postgresql/${psqlVersion}"; options.mountpoint = "legacy"; }; | ||
49 | environment.persistence."/persist/zfast".directories = [ | ||
50 | { | ||
51 | directory = "/var/lib/postgresql"; | ||
52 | user = config.users.users.postgres.name; | ||
53 | group = config.users.users.postgres.group; | ||
54 | mode = "0755"; | ||
55 | } | ||
56 | { | ||
57 | directory = "/var/lib/fluentbit"; | ||
58 | user = config.users.users.fluentbit.name; | ||
59 | group = config.users.users.fluentbit.group; | ||
60 | mode = "0755"; | ||
61 | } | ||
62 | { | ||
63 | directory = "/var/lib/loki"; | ||
64 | user = config.users.users.loki.name; | ||
65 | group = config.users.users.loki.group; | ||
66 | mode = "0755"; | ||
67 | } | ||
68 | ]; | ||
69 | |||
70 | ids.uids.fluentbit = 500; | ||
71 | ids.gids.fluentbit = 500; | ||
72 | users.users.fluentbit = { | ||
73 | name = "fluentbit"; | ||
74 | home = "/var/lib/fluentbit"; | ||
75 | uid = config.ids.uids.fluentbit; | ||
76 | group = "fluentbit"; | ||
77 | isSystemUser = true; | ||
78 | extraGroups = [ "systemd-journal" ]; | ||
79 | }; | ||
80 | users.groups.fluentbit.gid = config.ids.gids.fluentbit; | ||
81 | |||
82 | services.loki = { | ||
83 | enable = true; | ||
84 | configuration = { | ||
85 | auth_enabled = false; | ||
86 | common = { | ||
87 | ring.kvstore.store = "inmemory"; | ||
88 | ring.instance_addr = "127.0.0.1"; | ||
89 | replication_factor = 1; | ||
90 | path_prefix = "/var/lib/loki"; | ||
91 | }; | ||
92 | server.log_level = "warn"; | ||
93 | limits_config = { | ||
94 | reject_old_samples = false; | ||
95 | ingestion_rate_mb = 100; | ||
96 | ingestion_burst_size_mb = 200; | ||
97 | per_stream_rate_limit = "100MB"; | ||
98 | per_stream_rate_limit_burst = "200MB"; | ||
99 | }; | ||
100 | |||
101 | schema_config.configs = [ | ||
102 | { | ||
103 | from = "2020-10-24"; | ||
104 | store = "boltdb-shipper"; | ||
105 | object_store = "filesystem"; | ||
106 | schema = "v11"; | ||
107 | index.prefix = "index_"; | ||
108 | index.period = "24h"; | ||
109 | } | ||
110 | ]; | ||
111 | }; | ||
112 | }; | ||
113 | services.postgresql = { | ||
114 | enable = true; | ||
115 | package = pkgs.postgresql_13; | ||
116 | ensureDatabases = [ "fluentbit" ]; | ||
117 | ensureUsers = [ | ||
118 | { | ||
119 | name = "fluentbit"; | ||
120 | ensurePermissions."DATABASE \"fluentbit\"" = "ALL PRIVILEGES"; | ||
121 | } | ||
122 | ]; | ||
123 | }; | ||
124 | |||
125 | environment.systemPackages = [ | ||
126 | pkgs.fluent-bit | ||
127 | ]; | ||
128 | systemd.services.fluent-bit = { | ||
129 | description = "Fluent-bit daemon"; | ||
130 | wantedBy = [ "multi-user.target" ]; | ||
131 | serviceConfig = { | ||
132 | ExecStart = "${pkgs.fluent-bit}/bin/fluent-bit -c ${yamlFormat.generate "fluent.yaml" fluent-bit-config}"; | ||
133 | User = "fluentbit"; | ||
134 | Group = "fluentbit"; | ||
135 | SupplementaryGroups = [ "systemd-journal" ]; | ||
136 | }; | ||
137 | }; | ||
138 | } | ||