systems/eldiron/base.nix

   1 { config, pkgs, lib, ports, name, secrets, ... }:
   2 {
   3   # ssh-keyscan eldiron | nix-shell -p ssh-to-age --run ssh-to-age
   4   secrets.ageKeys = [ "age1dxr5lhvtnjssfaqpnf6qx80h8gfwkxg3tdf35m6n9wljmk7wadfs3kmahj" ];
   5   boot = {
   6     kernelModules = [ "kvm-intel" ];
   7     blacklistedKernelModules = [ "nvidiafb" ];
   8     loader.timeout = 1;
   9     loader.grub.devices = [ "/dev/sda" "/dev/sdc" ];
  10     kernel.sysctl = {
  11       # https://github.com/Netflix/security-bulletins/blob/master/advisories/third-party/2019-001.md
  12       "net.ipv4.tcp_sack" = 0;
  13     };
  14     supportedFilesystems = [ "zfs" ];
  15     kernelParams = ["zfs.zfs_arc_max=6442450944"];
  16     kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
  17     initrd.availableKernelModules = [ "ahci" "sd_mod" ];
  18     initrd.secrets = {
  19       "/boot/pass.key" = "/boot/pass.key";
  20     };
  21   };
  22   services.udev.extraRules = ''
  23     ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="c8:60:00:56:a0:88", NAME="eth0"
  24   '';
  25   nix.settings.max-jobs = 8;
  26   nixpkgs.config.permittedInsecurePackages = [
  27     "python-2.7.18.6" # for nagios-cli
  28     "nodejs-16.20.2" # for landing page building
  29   ];
  30
  31   nixpkgs.overlays = builtins.attrValues ports.overlays;
  32   powerManagement.cpuFreqGovernor = "ondemand";
  33
  34   security.acme.certs."${name}".postRun = builtins.concatStringsSep "\n" [
  35     (lib.optionalString config.services.websites.env.production.enable "/run/current-system/sw/bin/machinectl shell httpd-production /usr/bin/env systemctl reload httpd.service")
  36     (lib.optionalString config.services.websites.env.integration.enable "/run/current-system/sw/bin/machinectl shell httpd-integration /usr/bin/env systemctl reload httpd.service")
  37   ];
  38
  39   fileSystems = {
  40     # pools:
  41     #     zpool: ashift=12
  42     #     zfast: ashift=12
  43     # zfs:
  44     #     zpool/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
  45     #     zpool/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
  46     #     zpool/root/var: atime=on
  47     #     zfast/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
  48     #     zfast/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
  49     #     zfast/root/etc: ø
  50     #     zfast/root/nix: ø
  51     #     zfast/root/tmp: async=disabled
  52     #     zfast/root/var: atime=on
  53     #     zfast/root/var/lib: ø
  54     #     zfast/root/var/lib/mysql: logbias=throughput ; atime=off ; primarycache=metadata
  55     #     zfast/root/var/lib/postgresql: recordsize=8K ; atime=off ; logbias=throughput
  56     #     zfast/root/var/lib/postgresql/11.0: ø
  57     #     zfast/root/var/lib/postgresql/11.0/pg_wal: ø
  58     "/"     = { fsType = "zfs"; device = "zpool/root"; };
  59     "/boot" = { fsType = "ext4"; device = "/dev/disk/by-uuid/e6bb18fb-ff56-4b5f-ae9f-e60d40dc0622"; };
  60     "/etc"  = { fsType = "zfs"; device = "zpool/root/etc"; };
  61     "/nix"  = { fsType = "zfs"; device = "zfast/root/nix"; };
  62     "/tmp"  = { fsType = "zfs"; device = "zfast/root/tmp"; };
  63     "/var"  = { fsType = "zfs"; device = "zpool/root/var"; };
  64     "/var/lib/mysql" = { fsType = "zfs"; device = "zfast/root/var/lib/mysql"; };
  65     "/var/lib/postgresql"             = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql"; };
  66     "/var/lib/postgresql/11.0"        = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0"; };
  67     "/var/lib/postgresql/11.0/pg_wal" = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0/pg_wal"; };
  68   };
  69   swapDevices = [ { label = "swap1"; } { label = "swap2"; } ];
  70   hardware.enableRedistributableFirmware = true;
  71
  72   services.zfs = {
  73     autoScrub = {
  74       enable = false;
  75     };
  76   };
  77   networking = {
  78     hostId = "8262ca33"; # generated with head -c4 /dev/urandom | od -A none -t x4
  79     firewall.enable = true;
  80     firewall.allowedTCPPorts = [ config.myEnv.ports.zrepl_flony ];
  81     # FIXME: on next reboot, remove the /27 and the localCommands
  82     interfaces."eth0".ipv4.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
  83       (n: ips: map (ip: { address = ip; prefixLength = 32; }) (ips.ip4 or []))
  84       (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips))
  85       ++ [ { address = lib.head config.hostEnv.ips.main.ip4; prefixLength = 27; } ];
  86     interfaces."eth0".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
  87       (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
  88       config.hostEnv.ips);
  89     defaultGateway = "176.9.151.65";
  90     localCommands = ''
  91       # FIXME: Those commands were added by nixops and may not be
  92       # actually needed
  93       ip -6 addr add '2a01:4f8:160:3445::/64' dev 'eth0' || true
  94       ip -4 route change '176.9.151.64/27' via '176.9.151.65' dev 'eth0' || true
  95       ip -6 route add default via 'fe80::1' dev eth0 || true
  96     '';
  97     nameservers = [
  98       "213.133.98.98"
  99       "213.133.99.99"
 100       "213.133.100.100"
 101       "2a01:4f8:0:a0a1::add:1010"
 102       "2a01:4f8:0:a102::add:9999"
 103       "2a01:4f8:0:a111::add:9898"
 104     ];
 105   };
 106
 107   imports = [
 108     secrets.nixosModules.users-config-eldiron
 109     ./databases
 110     ./databases/mariadb.nix
 111     ./databases/openldap
 112     ./databases/postgresql.nix
 113     ./databases/redis.nix
 114
 115
 116     ./monitoring.nix
 117     ./ejabberd
 118     ./buildbot
 119     ./coturn.nix
 120     ./dns.nix
 121     ./borg_backup.nix
 122     ./duply_backup.nix
 123     ./gemini
 124     ./gitolite
 125     ./mail
 126     ./websites
 127     ./webstats
 128     ./irc.nix
 129     ./pub
 130     ./tasks
 131     ./ftp.nix
 132     ./mpd.nix
 133     ./vpn
 134   ];
 135
 136   services.borgBackup.enable = true;
 137   services.borgBackup.profiles.global = {
 138     bucket = "global";
 139     hash = false;
 140     remotes = [ "attilax" ];
 141     ignoredPaths = [
 142       "udev"
 143       "portables"
 144       "machines"
 145       "nixos"
 146       "nixos-containers"
 147     ];
 148   };
 149   myServices.buildbot.enable = true;
 150   myServices.databases.enable = true;
 151   myServices.gitolite.enable = true;
 152   myServices.monitoring.enable = true;
 153   myServices.irc.enable = true;
 154   myServices.pub.enable = true;
 155   myServices.tasks.enable = true;
 156   myServices.mpd.enable = true;
 157   myServices.dns.enable = true;
 158   myServices.websites.enable = true;
 159   myServices.gemini.enable = true;
 160   myServices.mail.enable = true;
 161   myServices.ejabberd.enable = true;
 162   myServices.vpn.enable = true;
 163   myServices.ftp.enable = true;
 164
 165   myServices.chatonsProperties.hostings.infogerance = {
 166     file.datetime = "2022-08-27T18:50:00";
 167     hosting = {
 168       name = "Infogérance";
 169       description = "Administration de serveurs";
 170       website = "https://www.immae.eu/";
 171       logo = "https://assets.immae.eu/logo.jpg";
 172       type = "HOSTEDSERVER";
 173       status.level = "OK";
 174       status.description = "OK";
 175       registration.load = "OPEN";
 176       install.type = "PACKAGE";
 177     };
 178   };
 179
 180   secrets.keys = {
 181     "ldap/pam_pgsql" = {
 182       user = "root";
 183       group = "root";
 184       permissions = "0400";
 185       text = ''
 186         database = immae
 187         user = immae_auth_read
 188         password = {{ .postgresql.immae_auth_read }}
 189         table = ldap_users
 190         user_column = login
 191         pw_type = function
 192         auth_query = SELECT ((mechanism = 'SSHA' AND password = encode(digest( %p || salt, 'sha1'), 'hex')) OR (mechanism = 'PLAIN' AND password = %p)) FROM ldap_users WHERE login = %u OR login || '@' || realm = %u
 193         #pwd_query = WITH newsalt as (select gen_random_bytes(4)) UPDATE ldap_users SET password = encode(digest( %p || (SELECT * FROM newsalt), 'sha1'), 'hex'), salt = (SELECT * FROM newsalt), mechanism = 'SSHA' WHERE login = %u OR login || '@' || realm = %u
 194       '';
 195     };
 196
 197     "zrepl_backup/identity" = {
 198       user = "root";
 199       group = "root";
 200       permissions = "0400";
 201       text = config.myEnv.zrepl_backup.ssh_key.private;
 202     };
 203     "zrepl/${name}.key" = {
 204       permissions = "0400";
 205       text = config.myEnv.zrepl_backup.certs."${name}".key;
 206       user = "root";
 207       group = "root";
 208     };
 209   } // builtins.listToAttrs (map (x: lib.attrsets.nameValuePair "zrepl/certificates/${x}.crt" {
 210     permissions = "0400";
 211     text = config.myEnv.zrepl_backup.certs."${x}".certificate;
 212     user = "root";
 213     group = "root";
 214   }) (builtins.attrNames config.myEnv.zrepl_backup.certs));
 215
 216   programs.ssh.knownHosts.dilion = {
 217     extraHostNames = ["dilion.immae.eu"];
 218     publicKey = config.myEnv.servers.dilion.hostKey;
 219   };
 220
 221   services.cron = {
 222     enable = true;
 223     mailto = "cron@immae.eu";
 224     systemCronJobs = [
 225       ''
 226         0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "immae.eu.*Recipient address rejected"
 227         # Need a way to blacklist properly
 228         # 0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "NOQUEUE:"
 229         0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtp -g "status=bounced"
 230       ''
 231     ];
 232   };
 233
 234   environment.systemPackages = [ pkgs.bindfs ];
 235
 236   environment.etc."mdadm.conf" = {
 237     enable = true;
 238     mode = "0644";
 239     user = "root";
 240     text = "MAILADDR ${config.myEnv.monitoring.email}";
 241   };
 242
 243   systemd.services.zrepl.path = [ pkgs.openssh ];
 244   services.zrepl = {
 245     enable = true;
 246     settings = {
 247       jobs = [
 248         {
 249           type = "push";
 250           # must not change
 251           name = "backup-to-dilion";
 252           filesystems."zpool/root" = true;
 253           filesystems."zpool/root/etc" = true;
 254           filesystems."zpool/root/var<" = true;
 255           connect = {
 256             address = "dilion.immae.eu:19000";
 257             type = "tls";
 258             server_cn = "dilion";
 259             ca = config.secrets.fullPaths."zrepl/certificates/dilion.crt";
 260             cert = config.secrets.fullPaths."zrepl/certificates/eldiron.crt";
 261             key = config.secrets.fullPaths."zrepl/eldiron.key";
 262           };
 263           snapshotting = {
 264             type = "periodic";
 265             prefix = "zrepl_";
 266             interval = "1h";
 267             # hooks = [
 268             #   {
 269             #     type = "mysql-lock-tables";
 270             #     dsn = "${config.myEnv.zrepl_backup.mysql.user}:${config.myEnv.zrepl_backup.mysql.password}@tcp(localhost)/";
 271             #     filesystems."zpool/root/var" = true;
 272             #   }
 273             #   {
 274             #     type = "command";
 275             #     path = pkgs.writeScript "redis-dump" ''
 276             #       #!${pkgs.stdenv.shell}
 277             #       ${pkgs.redis}/bin/redis-cli bgsave
 278             #     '';
 279             #     err_is_fatal = false;
 280             #     filesystems."zpool/root/var" = true;
 281             #   }
 282             # ];
 283           };
 284           send.encrypted = true;
 285           pruning.keep_sender = [
 286             { type = "regex"; regex = "^manual_.*"; }
 287             { type = "grid"; grid = "24x1h | 7x1d | 4x7d | 6x30d"; regex = "^zrepl_.*"; }
 288           ];
 289           pruning.keep_receiver = [
 290             { type = "regex"; regex = "^manual_.*"; }
 291             { type = "grid"; grid = "6x4h | 7x1d | 4x7d | 6x30d"; regex = "^zrepl_.*"; }
 292           ];
 293         }
 294         {
 295           type = "source";
 296           # must not change
 297           name = "backup-to-wd-zpool";
 298           serve.type = "tls";
 299           serve.listen = ":${builtins.toString config.myEnv.ports.zrepl_flony}";
 300           serve.ca = config.secrets.fullPaths."zrepl/certificates/flony.crt";
 301           serve.cert = config.secrets.fullPaths."zrepl/certificates/eldiron.crt";
 302           serve.key = config.secrets.fullPaths."zrepl/eldiron.key";
 303           serve.client_cns = [ "flony" ];
 304           filesystems."zpool/root" = true;
 305           filesystems."zpool/root/etc" = true;
 306           filesystems."zpool/root/var<" = true;
 307           filesystems."zfast/root/var<" = true;
 308           send.encrypted = true;
 309           snapshotting.type = "manual";
 310         }
 311       ];
 312     };
 313   };
 314
 315   environment.etc."fail2ban/filter.d/postgresql.conf".text = ''
 316     [Definition]
 317     failregex = <HOST> \S+ FATAL:  password authentication failed for user .+$
 318                 <HOST> \S+ FATAL:  PAM authentication failed for user.+$
 319                 <HOST> \S+ FATAL:  no pg_hba.conf entry for host.+$
 320   '';
 321   environment.etc."fail2ban/filter.d/mysqld-auth.local".text = ''
 322     [Definition]
 323     _daemon = mysql[-\w]*
 324   '';
 325   services.fail2ban.jails.dovecot = ''
 326     enabled = true
 327   '';
 328   services.fail2ban.jails.postfix-sasl = ''
 329     enabled = true
 330   '';
 331   services.fail2ban.jails.proftpd = ''
 332     enabled = true
 333   '';
 334   services.fail2ban.jails.postgresql = ''
 335     enabled = true
 336     port = 5432
 337     logpath  = %(syslog_daemon)s
 338     backend  = %(default_backend)s
 339     journalmatch = _SYSTEMD_UNIT=postgresql.service + _COMM=postgres
 340   '';
 341   services.fail2ban.jails.mysqld-auth = ''
 342     enabled = true
 343     journalmatch = _SYSTEMD_UNIT=mysql.service + _COMM=mysqld
 344   '';
 345   # This value determines the NixOS release with which your system is
 346   # to be compatible, in order to avoid breaking some software such as
 347   # database servers. You should change this only after NixOS release
 348   # notes say you should.
 349   # https://nixos.org/nixos/manual/release-notes.html
 350   system.stateVersion = "23.05"; # Did you read the comment?
 351
 352   security.pam.services.ldap.text = ''
 353     # Authentication from ldap for pgsql
 354     auth      required   ${pkgs.pam_pgsql}/lib/security/pam_pgsql.so config_file=/var/secrets/ldap/pam_pgsql
 355     account   required   ${pkgs.pam_pgsql}/lib/security/pam_pgsql.so config_file=/var/secrets/ldap/pam_pgsql
 356   '';
 357   services.saslauthd = {
 358     enable = true;
 359     mechanism = "pam";
 360   };
 361   environment.etc."sasl2/slapd.conf".text = ''
 362     mech_list: plain
 363     pwcheck_method: saslauthd
 364     saslauthd_path: /run/saslauthd/mux
 365   '';
 366 }