aboutsummaryrefslogtreecommitdiff
path: root/systems/zoldene/logging.nix
blob: 2b6e331126a09d3b2507c6b46391e79a2be5d8c3 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
{ config, pkgs, name, ... }:
# Initialization
#   CREATE INDEX ON LOGS (tag);
#   CREATE INDEX ON LOGS (time);
#   CREATE INDEX ON LOGS (((data->>'PRIORITY')::int));
#   CREATE INDEX ON LOGS ((data->>'_SYSTEMD_UNIT'));
#   CREATE INDEX ON LOGS ((data->>'SYSLOG_IDENTIFIER'));
let
  fluent-bit-config = {
    pipeline = {
      inputs = [
        {
          name = "systemd";
          tag  = "${name}.systemd";
          DB   = "/var/lib/fluentbit/fluent-bit.db";
        }
      ];
      outputs = [
        {
          name = "loki";
          match = "${name}.systemd";
          line_format = "json";
          labels = "job=fluentbit, server=${name}, priority=$PRIORITY, syslog_identifier=$SYSLOG_IDENTIFIER, systemd_unit=$_SYSTEMD_UNIT";
        }
        {
          name = "pgsql";
          match = "*";
          host = "/run/postgresql";
          user = "fluentbit";
          table = "logs";
          database = "fluentbit";
          timestamp_key = "event_timestamp";
        }
      ];
    };
  };
  yamlFormat = pkgs.formats.yaml {};
  psqlVersion = pkgs.postgresql_16.psqlSchema;
in
{
  disko.devices.zpool.zfast.datasets."root/persist/var/lib/loki" =
    { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/loki"; options.mountpoint = "legacy"; };
  disko.devices.zpool.zfast.datasets."root/persist/var/lib/fluentbit" =
    { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/fluentbit"; options.mountpoint = "legacy"; };
  disko.devices.zpool.zfast.datasets."root/persist/var/lib/postgresql" =
    { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/postgresql"; options.mountpoint = "legacy"; };
  disko.devices.zpool.zfast.datasets."root/persist/var/lib/postgresql/${psqlVersion}" =
    { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/postgresql/${psqlVersion}"; options.mountpoint = "legacy"; };
  environment.persistence."/persist/zfast".directories = [
    {
      directory = "/var/lib/postgresql";
      user = config.users.users.postgres.name;
      group = config.users.users.postgres.group;
      mode = "0755";
    }
    {
      directory = "/var/lib/postgresql/${psqlVersion}";
      user = config.users.users.postgres.name;
      group = config.users.users.postgres.group;
      mode = "0755";
    }
    {
      directory = "/var/lib/fluentbit";
      user = config.users.users.fluentbit.name;
      group = config.users.users.fluentbit.group;
      mode = "0755";
    }
    {
      directory = "/var/lib/loki";
      user = config.users.users.loki.name;
      group = config.users.users.loki.group;
      mode = "0755";
    }
  ];

  ids.uids.fluentbit = 500;
  ids.gids.fluentbit = 500;
  users.users.fluentbit = {
    name = "fluentbit";
    home = "/var/lib/fluentbit";
    uid = config.ids.uids.fluentbit;
    group = "fluentbit";
    isSystemUser = true;
    extraGroups = [ "systemd-journal" ];
  };
  users.groups.fluentbit.gid = config.ids.gids.fluentbit;

  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;
      common = {
        ring.kvstore.store = "inmemory";
        ring.instance_addr = "127.0.0.1";
        replication_factor = 1;
        path_prefix = "/var/lib/loki";
      };
      server.log_level = "warn";
      limits_config = {
        reject_old_samples = false;
        ingestion_rate_mb = 100;
        ingestion_burst_size_mb = 200;
        per_stream_rate_limit = "100MB";
        per_stream_rate_limit_burst = "200MB";
      };

      schema_config.configs = [
        {
          from = "2020-10-24";
          store = "boltdb-shipper";
          object_store = "filesystem";
          schema = "v11";
          index.prefix = "index_";
          index.period = "24h";
        }
      ];
    };
  };
  systemd.services.postgresql.after = [
    "var-lib-postgresql.mount"
    "var-lib-postgresql-16.mount"
    "persist-zfast-var-lib-postgresql.mount"
    "persist-zfast-var-lib-postgresql-16.mount"
  ];
  systemd.services.postgresql.unitConfig = {
    BindsTo = [
      "var-lib-postgresql.mount"
      "var-lib-postgresql-16.mount"
      "persist-zfast-var-lib-postgresql.mount"
      "persist-zfast-var-lib-postgresql-16.mount"
    ];
  };

  services.postgresql = {
    enable = true;
    package = pkgs.postgresql_16;
    ensureDatabases = [ "fluentbit" ];
    ensureUsers = [
      {
        name = "fluentbit";
        ensureDBOwnership = true;
      }
    ];
  };

  environment.systemPackages = [
    pkgs.fluent-bit
  ];
  systemd.services.fluent-bit = {
    description = "Fluent-bit daemon";
    wantedBy = [ "multi-user.target" ];
    serviceConfig = {
      ExecStart = "${pkgs.fluent-bit}/bin/fluent-bit -c ${yamlFormat.generate "fluent.yaml" fluent-bit-config}";
      User = "fluentbit";
      Group = "fluentbit";
      SupplementaryGroups = [ "systemd-journal" ];
    };
  };
}