{ config, pkgs, name, ... }: # Initialization # CREATE INDEX ON LOGS (tag); # CREATE INDEX ON LOGS (time); # CREATE INDEX ON LOGS (((data->>'PRIORITY')::int)); # CREATE INDEX ON LOGS ((data->>'_SYSTEMD_UNIT')); # CREATE INDEX ON LOGS ((data->>'SYSLOG_IDENTIFIER')); let fluent-bit-config = { pipeline = { inputs = [ { name = "systemd"; tag = "${name}.systemd"; DB = "/var/lib/fluentbit/fluent-bit.db"; } ]; outputs = [ { name = "loki"; match = "${name}.systemd"; line_format = "json"; labels = "job=fluentbit, server=${name}, priority=$PRIORITY, syslog_identifier=$SYSLOG_IDENTIFIER, systemd_unit=$_SYSTEMD_UNIT"; } { name = "pgsql"; match = "*"; host = "/run/postgresql"; user = "fluentbit"; table = "logs"; database = "fluentbit"; timestamp_key = "event_timestamp"; } ]; }; }; yamlFormat = pkgs.formats.yaml {}; psqlVersion = pkgs.postgresql_16.psqlSchema; in { disko.devices.zpool.zfast.datasets."root/persist/var/lib/loki" = { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/loki"; options.mountpoint = "legacy"; }; disko.devices.zpool.zfast.datasets."root/persist/var/lib/fluentbit" = { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/fluentbit"; options.mountpoint = "legacy"; }; disko.devices.zpool.zfast.datasets."root/persist/var/lib/postgresql" = { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/postgresql"; options.mountpoint = "legacy"; }; disko.devices.zpool.zfast.datasets."root/persist/var/lib/postgresql/${psqlVersion}" = { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/postgresql/${psqlVersion}"; options.mountpoint = "legacy"; }; environment.persistence."/persist/zfast".directories = [ { directory = "/var/lib/postgresql"; user = config.users.users.postgres.name; group = config.users.users.postgres.group; mode = "0755"; } { directory = "/var/lib/fluentbit"; user = config.users.users.fluentbit.name; group = config.users.users.fluentbit.group; mode = "0755"; } { directory = "/var/lib/loki"; user = config.users.users.loki.name; group = config.users.users.loki.group; mode = "0755"; } ]; ids.uids.fluentbit = 500; ids.gids.fluentbit = 500; users.users.fluentbit = { name = "fluentbit"; home = "/var/lib/fluentbit"; uid = config.ids.uids.fluentbit; group = "fluentbit"; isSystemUser = true; extraGroups = [ "systemd-journal" ]; }; users.groups.fluentbit.gid = config.ids.gids.fluentbit; services.loki = { enable = true; configuration = { auth_enabled = false; common = { ring.kvstore.store = "inmemory"; ring.instance_addr = "127.0.0.1"; replication_factor = 1; path_prefix = "/var/lib/loki"; }; server.log_level = "warn"; limits_config = { reject_old_samples = false; ingestion_rate_mb = 100; ingestion_burst_size_mb = 200; per_stream_rate_limit = "100MB"; per_stream_rate_limit_burst = "200MB"; }; schema_config.configs = [ { from = "2020-10-24"; store = "boltdb-shipper"; object_store = "filesystem"; schema = "v11"; index.prefix = "index_"; index.period = "24h"; } ]; }; }; services.postgresql = { enable = true; package = pkgs.postgresql_16; ensureDatabases = [ "fluentbit" ]; ensureUsers = [ { name = "fluentbit"; ensureDBOwnership = true; } ]; }; environment.systemPackages = [ pkgs.fluent-bit ]; systemd.services.fluent-bit = { description = "Fluent-bit daemon"; wantedBy = [ "multi-user.target" ]; serviceConfig = { ExecStart = "${pkgs.fluent-bit}/bin/fluent-bit -c ${yamlFormat.generate "fluent.yaml" fluent-bit-config}"; User = "fluentbit"; Group = "fluentbit"; SupplementaryGroups = [ "systemd-journal" ]; }; }; }