path: root/systems/zoldene/logging.nix

{ config, pkgs, name, ... }:
# Initialization
#   CREATE INDEX ON LOGS (tag);
#   CREATE INDEX ON LOGS (time);
#   CREATE INDEX ON LOGS (((data->>'PRIORITY')::int));
#   CREATE INDEX ON LOGS ((data->>'_SYSTEMD_UNIT'));
#   CREATE INDEX ON LOGS ((data->>'SYSLOG_IDENTIFIER'));
let
  fluent-bit-config = {
    pipeline = {
      inputs = [
        {
          name = "systemd";
          tag  = "${name}.systemd";
          DB   = "/var/lib/fluentbit/fluent-bit.db";
        }
      ];
      outputs = [
        {
          name = "loki";
          match = "${name}.systemd";
          line_format = "json";
          labels = "job=fluentbit, server=${name}, priority=$PRIORITY, syslog_identifier=$SYSLOG_IDENTIFIER, systemd_unit=$_SYSTEMD_UNIT";
        }
        {
          name = "pgsql";
          match = "*";
          host = "/run/postgresql";
          user = "fluentbit";
          table = "logs";
          database = "fluentbit";
          timestamp_key = "event_timestamp";
        }
      ];
    };
  };
  yamlFormat = pkgs.formats.yaml {};
  psqlVersion = pkgs.postgresql_16.psqlSchema;
in
{
  disko.devices.zpool.zfast.datasets."root/persist/var/lib/loki" =
    { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/loki"; options.mountpoint = "legacy"; };
  disko.devices.zpool.zfast.datasets."root/persist/var/lib/fluentbit" =
    { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/fluentbit"; options.mountpoint = "legacy"; };
  disko.devices.zpool.zfast.datasets."root/persist/var/lib/postgresql" =
    { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/postgresql"; options.mountpoint = "legacy"; };
  disko.devices.zpool.zfast.datasets."root/persist/var/lib/postgresql/${psqlVersion}" =
    { type = "zfs_fs"; mountpoint = "/persist/zfast/var/lib/postgresql/${psqlVersion}"; options.mountpoint = "legacy"; };
  environment.persistence."/persist/zfast".directories = [
    {
      directory = "/var/lib/postgresql";
      user = config.users.users.postgres.name;
      group = config.users.users.postgres.group;
      mode = "0755";
    }
    {
      directory = "/var/lib/postgresql/${psqlVersion}";
      user = config.users.users.postgres.name;
      group = config.users.users.postgres.group;
      mode = "0755";
    }
    {
      directory = "/var/lib/fluentbit";
      user = config.users.users.fluentbit.name;
      group = config.users.users.fluentbit.group;
      mode = "0755";
    }
    {
      directory = "/var/lib/loki";
      user = config.users.users.loki.name;
      group = config.users.users.loki.group;
      mode = "0755";
    }
  ];

  ids.uids.fluentbit = 500;
  ids.gids.fluentbit = 500;
  users.users.fluentbit = {
    name = "fluentbit";
    home = "/var/lib/fluentbit";
    uid = config.ids.uids.fluentbit;
    group = "fluentbit";
    isSystemUser = true;
    extraGroups = [ "systemd-journal" ];
  };
  users.groups.fluentbit.gid = config.ids.gids.fluentbit;

  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;
      common = {
        ring.kvstore.store = "inmemory";
        ring.instance_addr = "127.0.0.1";
        replication_factor = 1;
        path_prefix = "/var/lib/loki";
      };
      server.log_level = "warn";
      limits_config = {
        reject_old_samples = false;
        ingestion_rate_mb = 100;
        ingestion_burst_size_mb = 200;
        per_stream_rate_limit = "100MB";
        per_stream_rate_limit_burst = "200MB";
        # Remove after 2024-07-08 see below
        allow_structured_metadata = false;
      };

      schema_config.configs = [
        {
          from = "2020-10-24";
          store = "boltdb-shipper";
          object_store = "filesystem";
          schema = "v11";
          index.prefix = "index_";
          index.period = "24h";
        }
        {
          from = "2024-07-08";
          store = "tsdb";
          object_store = "filesystem";
          schema = "v13";
          index.prefix = "index_";
          index.period = "24h";
        }
      ];
    };
  };
  systemd.services.postgresql.after = [
    "var-lib-postgresql.mount"
    "var-lib-postgresql-16.mount"
    "persist-zfast-var-lib-postgresql.mount"
    "persist-zfast-var-lib-postgresql-16.mount"
  ];
  systemd.services.postgresql.unitConfig = {
    BindsTo = [
      "var-lib-postgresql.mount"
      "var-lib-postgresql-16.mount"
      "persist-zfast-var-lib-postgresql.mount"
      "persist-zfast-var-lib-postgresql-16.mount"
    ];
  };

  services.postgresql = {
    enable = true;
    package = pkgs.postgresql_16;
    ensureDatabases = [ "fluentbit" ];
    ensureUsers = [
      {
        name = "fluentbit";
        ensureDBOwnership = true;
      }
    ];
  };

  environment.systemPackages = [
    pkgs.fluent-bit
  ];
  systemd.services.fluent-bit = {
    description = "Fluent-bit daemon";
    wantedBy = [ "multi-user.target" ];
    serviceConfig = {
      ExecStart = "${pkgs.fluent-bit}/bin/fluent-bit -c ${yamlFormat.generate "fluent.yaml" fluent-bit-config}";
      User = "fluentbit";
      Group = "fluentbit";
      SupplementaryGroups = [ "systemd-journal" ];
    };
  };
}