systems/eldiron/base.nix

   1 { config, pkgs, lib, php, name, secrets, ... }:
   2 {
   3   # ssh-keyscan eldiron | nix-shell -p ssh-to-age --run ssh-to-age
   4   secrets.ageKeys = [ "age1dxr5lhvtnjssfaqpnf6qx80h8gfwkxg3tdf35m6n9wljmk7wadfs3kmahj" ];
   5   boot = {
   6     kernelModules = [ "kvm-intel" ];
   7     blacklistedKernelModules = [ "nvidiafb" ];
   8     loader.timeout = 1;
   9     loader.grub.devices = [ "/dev/sda" "/dev/sdc" ];
  10     kernel.sysctl = {
  11       # https://github.com/Netflix/security-bulletins/blob/master/advisories/third-party/2019-001.md
  12       "net.ipv4.tcp_sack" = 0;
  13     };
  14     supportedFilesystems = [ "zfs" ];
  15     kernelParams = ["zfs.zfs_arc_max=6442450944"];
  16     kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
  17     initrd.availableKernelModules = [ "ahci" "sd_mod" ];
  18     initrd.secrets = {
  19       "/boot/pass.key" = "/boot/pass.key";
  20     };
  21   };
  22   services.udev.extraRules = ''
  23     ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="c8:60:00:56:a0:88", NAME="eth0"
  24   '';
  25   nix.settings.max-jobs = 8;
  26   nixpkgs.config.permittedInsecurePackages = [
  27     "python-2.7.18.6" # for nagios-cli
  28     "nodejs-16.20.2" # for landing page building
  29   ];
  30
  31   nixpkgs.overlays = [
  32     php.overlays.php
  33   ];
  34   powerManagement.cpuFreqGovernor = "powersave";
  35
  36   security.acme.certs."${name}".postRun = builtins.concatStringsSep "\n" [
  37     (lib.optionalString config.services.websites.env.production.enable "/run/current-system/sw/bin/machinectl shell httpd-production /usr/bin/env systemctl reload httpd.service")
  38     (lib.optionalString config.services.websites.env.integration.enable "/run/current-system/sw/bin/machinectl shell httpd-integration /usr/bin/env systemctl reload httpd.service")
  39   ];
  40
  41   fileSystems = {
  42     # pools:
  43     #     zpool: ashift=12
  44     #     zfast: ashift=12
  45     # zfs:
  46     #     zpool/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
  47     #     zpool/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
  48     #     zpool/root/var: atime=on
  49     #     zfast/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
  50     #     zfast/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
  51     #     zfast/root/etc: ø
  52     #     zfast/root/nix: ø
  53     #     zfast/root/tmp: async=disabled
  54     #     zfast/root/var: atime=on
  55     #     zfast/root/var/lib: ø
  56     #     zfast/root/var/lib/mysql: logbias=throughput ; atime=off ; primarycache=metadata
  57     #     zfast/root/var/lib/postgresql: recordsize=8K ; atime=off ; logbias=throughput
  58     #     zfast/root/var/lib/postgresql/11.0: ø
  59     #     zfast/root/var/lib/postgresql/11.0/pg_wal: ø
  60     "/"     = { fsType = "zfs"; device = "zpool/root"; };
  61     "/boot" = { fsType = "ext4"; device = "/dev/disk/by-uuid/e6bb18fb-ff56-4b5f-ae9f-e60d40dc0622"; };
  62     "/etc"  = { fsType = "zfs"; device = "zpool/root/etc"; };
  63     "/nix"  = { fsType = "zfs"; device = "zfast/root/nix"; };
  64     "/tmp"  = { fsType = "zfs"; device = "zfast/root/tmp"; };
  65     "/var"  = { fsType = "zfs"; device = "zpool/root/var"; };
  66     "/var/lib/mysql" = { fsType = "zfs"; device = "zfast/root/var/lib/mysql"; };
  67     "/var/lib/postgresql"             = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql"; };
  68     "/var/lib/postgresql/11.0"        = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0"; };
  69     "/var/lib/postgresql/11.0/pg_wal" = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0/pg_wal"; };
  70   };
  71   swapDevices = [ { label = "swap1"; } { label = "swap2"; } ];
  72   hardware.enableRedistributableFirmware = true;
  73
  74   services.zfs = {
  75     autoScrub = {
  76       enable = false;
  77     };
  78   };
  79   networking = {
  80     hostId = "8262ca33"; # generated with head -c4 /dev/urandom | od -A none -t x4
  81     firewall.enable = true;
  82     firewall.allowedTCPPorts = [ config.myEnv.ports.zrepl_flony ];
  83     # FIXME: on next reboot, remove the /27 and the localCommands
  84     interfaces."eth0".ipv4.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
  85       (n: ips: map (ip: { address = ip; prefixLength = 32; }) (ips.ip4 or []))
  86       (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips))
  87       ++ [ { address = lib.head config.hostEnv.ips.main.ip4; prefixLength = 27; } ];
  88     interfaces."eth0".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
  89       (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
  90       config.hostEnv.ips);
  91     defaultGateway = "176.9.151.65";
  92     localCommands = ''
  93       # FIXME: Those commands were added by nixops and may not be
  94       # actually needed
  95       ip -6 addr add '2a01:4f8:160:3445::/64' dev 'eth0' || true
  96       ip -4 route change '176.9.151.64/27' via '176.9.151.65' dev 'eth0' || true
  97       ip -6 route add default via 'fe80::1' dev eth0 || true
  98     '';
  99     nameservers = [
 100       "213.133.98.98"
 101       "213.133.99.99"
 102       "213.133.100.100"
 103       "2a01:4f8:0:a0a1::add:1010"
 104       "2a01:4f8:0:a102::add:9999"
 105       "2a01:4f8:0:a111::add:9898"
 106     ];
 107   };
 108
 109   imports = [
 110     secrets.nixosModules.users-config-eldiron
 111     ./databases
 112     ./databases/mariadb.nix
 113     ./databases/openldap
 114     ./databases/postgresql.nix
 115     ./databases/redis.nix
 116
 117
 118     ./monitoring.nix
 119     ./ejabberd
 120     ./buildbot
 121     ./coturn.nix
 122     ./dns.nix
 123     ./duply_backup.nix
 124     ./gemini
 125     ./gitolite
 126     ./mail
 127     ./websites
 128     ./webstats
 129     ./irc.nix
 130     ./pub
 131     ./tasks
 132     ./ftp.nix
 133     ./mpd.nix
 134     ./vpn
 135   ];
 136
 137   myServices.buildbot.enable = true;
 138   myServices.databases.enable = true;
 139   myServices.gitolite.enable = true;
 140   myServices.monitoring.enable = true;
 141   myServices.irc.enable = true;
 142   myServices.pub.enable = true;
 143   myServices.tasks.enable = true;
 144   myServices.mpd.enable = true;
 145   myServices.dns.enable = true;
 146   myServices.websites.enable = true;
 147   myServices.gemini.enable = true;
 148   myServices.mail.enable = true;
 149   myServices.ejabberd.enable = true;
 150   myServices.vpn.enable = true;
 151   myServices.ftp.enable = true;
 152
 153   myServices.chatonsProperties.hostings.infogerance = {
 154     file.datetime = "2022-08-27T18:50:00";
 155     hosting = {
 156       name = "Infogérance";
 157       description = "Administration de serveurs";
 158       website = "https://www.immae.eu/";
 159       logo = "https://assets.immae.eu/logo.jpg";
 160       type = "HOSTEDSERVER";
 161       status.level = "OK";
 162       status.description = "OK";
 163       registration.load = "OPEN";
 164       install.type = "PACKAGE";
 165     };
 166   };
 167
 168   services.netdata.enable = true;
 169   services.netdata.config.global."memory mode" = "none";
 170   services.netdata.config.health."enabled" = "no";
 171   services.netdata.config.web.mode = "none";
 172   users.users."${config.services.netdata.user}".extraGroups = [ "keys" ];
 173   services.netdata.configDir."stream.conf" = config.secrets.fullPaths."netdata-stream.conf";
 174   secrets.keys = {
 175     "ldap/pam_pgsql" = {
 176       user = "root";
 177       group = "root";
 178       permissions = "0400";
 179       text = ''
 180         database = immae
 181         user = immae_auth_read
 182         password = {{ .postgresql.immae_auth_read }}
 183         table = ldap_users
 184         user_column = login
 185         pw_type = function
 186         auth_query = SELECT (mechanism = 'SSHA' AND password = encode(digest( %p || salt, 'sha1'), 'hex')) FROM ldap_users WHERE login = %u OR login || '@' || realm = %u
 187         #pwd_query = WITH newsalt as (select gen_random_bytes(4)) UPDATE ldap_users SET password = encode(digest( %p || (SELECT * FROM newsalt), 'sha1'), 'hex'), salt = (SELECT * FROM newsalt), mechanism = 'SSHA' WHERE login = %u OR login || '@' || realm = %u
 188       '';
 189     };
 190
 191     "netdata-stream.conf" = {
 192       user = config.services.netdata.user;
 193       group = config.services.netdata.group;
 194       permissions = "0400";
 195       text = ''
 196         [stream]
 197             enabled = yes
 198             destination = ${config.myEnv.monitoring.netdata_aggregator}
 199             api key = ${config.myEnv.monitoring.netdata_keys.eldiron}
 200       '';
 201     };
 202     "zrepl_backup/identity" = {
 203       user = "root";
 204       group = "root";
 205       permissions = "0400";
 206       text = config.myEnv.zrepl_backup.ssh_key.private;
 207     };
 208     "zrepl/${name}.key" = {
 209       permissions = "0400";
 210       text = config.myEnv.zrepl_backup.certs."${name}".key;
 211       user = "root";
 212       group = "root";
 213     };
 214   } // builtins.listToAttrs (map (x: lib.attrsets.nameValuePair "zrepl/certificates/${x}.crt" {
 215     permissions = "0400";
 216     text = config.myEnv.zrepl_backup.certs."${x}".certificate;
 217     user = "root";
 218     group = "root";
 219   }) (builtins.attrNames config.myEnv.zrepl_backup.certs));
 220
 221   programs.ssh.knownHosts.dilion = {
 222     extraHostNames = ["dilion.immae.eu"];
 223     publicKey = config.myEnv.servers.dilion.hostKey;
 224   };
 225
 226   services.cron = {
 227     enable = true;
 228     mailto = "cron@immae.eu";
 229     systemCronJobs = [
 230       ''
 231         0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "immae.eu.*Recipient address rejected"
 232         # Need a way to blacklist properly
 233         # 0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "NOQUEUE:"
 234         0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtp -g "status=bounced"
 235       ''
 236     ];
 237   };
 238
 239   environment.systemPackages = [ pkgs.bindfs ];
 240
 241   environment.etc."mdadm.conf" = {
 242     enable = true;
 243     mode = "0644";
 244     user = "root";
 245     text = "MAILADDR ${config.myEnv.monitoring.email}";
 246   };
 247
 248   systemd.services.zrepl.path = [ pkgs.openssh ];
 249   services.zrepl = {
 250     enable = true;
 251     settings = {
 252       jobs = [
 253         {
 254           type = "push";
 255           # must not change
 256           name = "backup-to-dilion";
 257           filesystems."zpool/root" = true;
 258           filesystems."zpool/root/etc" = true;
 259           filesystems."zpool/root/var<" = true;
 260           connect = {
 261             address = "dilion.immae.eu:19000";
 262             type = "tls";
 263             server_cn = "dilion";
 264             ca = config.secrets.fullPaths."zrepl/certificates/dilion.crt";
 265             cert = config.secrets.fullPaths."zrepl/certificates/eldiron.crt";
 266             key = config.secrets.fullPaths."zrepl/eldiron.key";
 267           };
 268           snapshotting = {
 269             type = "periodic";
 270             prefix = "zrepl_";
 271             interval = "1h";
 272             # hooks = [
 273             #   {
 274             #     type = "mysql-lock-tables";
 275             #     dsn = "${config.myEnv.zrepl_backup.mysql.user}:${config.myEnv.zrepl_backup.mysql.password}@tcp(localhost)/";
 276             #     filesystems."zpool/root/var" = true;
 277             #   }
 278             #   {
 279             #     type = "command";
 280             #     path = pkgs.writeScript "redis-dump" ''
 281             #       #!${pkgs.stdenv.shell}
 282             #       ${pkgs.redis}/bin/redis-cli bgsave
 283             #     '';
 284             #     err_is_fatal = false;
 285             #     filesystems."zpool/root/var" = true;
 286             #   }
 287             # ];
 288           };
 289           send.encrypted = true;
 290           pruning.keep_sender = [
 291             { type = "regex"; regex = "^manual_.*"; }
 292             { type = "grid"; grid = "24x1h | 7x1d | 4x7d | 6x30d"; regex = "^zrepl_.*"; }
 293           ];
 294           pruning.keep_receiver = [
 295             { type = "regex"; regex = "^manual_.*"; }
 296             { type = "grid"; grid = "6x4h | 7x1d | 4x7d | 6x30d"; regex = "^zrepl_.*"; }
 297           ];
 298         }
 299         {
 300           type = "source";
 301           # must not change
 302           name = "backup-to-wd-zpool";
 303           serve.type = "tls";
 304           serve.listen = ":${builtins.toString config.myEnv.ports.zrepl_flony}";
 305           serve.ca = config.secrets.fullPaths."zrepl/certificates/flony.crt";
 306           serve.cert = config.secrets.fullPaths."zrepl/certificates/eldiron.crt";
 307           serve.key = config.secrets.fullPaths."zrepl/eldiron.key";
 308           serve.client_cns = [ "flony" ];
 309           filesystems."zpool/root" = true;
 310           filesystems."zpool/root/etc" = true;
 311           filesystems."zpool/root/var<" = true;
 312           filesystems."zfast/root/var<" = true;
 313           send.encrypted = true;
 314           snapshotting.type = "manual";
 315         }
 316       ];
 317     };
 318   };
 319
 320   environment.etc."fail2ban/filter.d/postgresql.conf".text = ''
 321     [Definition]
 322     failregex = <HOST> \S+ FATAL:  password authentication failed for user .+$
 323                 <HOST> \S+ FATAL:  PAM authentication failed for user.+$
 324                 <HOST> \S+ FATAL:  no pg_hba.conf entry for host.+$
 325   '';
 326   environment.etc."fail2ban/filter.d/mysqld-auth.local".text = ''
 327     [Definition]
 328     _daemon = mysql[-\w]*
 329   '';
 330   services.fail2ban.jails.dovecot = ''
 331     enabled = true
 332   '';
 333   services.fail2ban.jails.postfix-sasl = ''
 334     enabled = true
 335   '';
 336   services.fail2ban.jails.proftpd = ''
 337     enabled = true
 338   '';
 339   services.fail2ban.jails.postgresql = ''
 340     enabled = true
 341     port = 5432
 342     logpath  = %(syslog_daemon)s
 343     backend  = %(default_backend)s
 344     journalmatch = _SYSTEMD_UNIT=postgresql.service + _COMM=postgres
 345   '';
 346   services.fail2ban.jails.mysqld-auth = ''
 347     enabled = true
 348     journalmatch = _SYSTEMD_UNIT=mysql.service + _COMM=mysqld
 349   '';
 350   # This value determines the NixOS release with which your system is
 351   # to be compatible, in order to avoid breaking some software such as
 352   # database servers. You should change this only after NixOS release
 353   # notes say you should.
 354   # https://nixos.org/nixos/manual/release-notes.html
 355   system.stateVersion = "23.05"; # Did you read the comment?
 356
 357   security.pam.services.ldap.text = ''
 358     # Authentication from ldap for pgsql
 359     auth      required   ${pkgs.pam_pgsql}/lib/security/pam_pgsql.so config_file=/var/secrets/ldap/pam_pgsql
 360     account   required   ${pkgs.pam_pgsql}/lib/security/pam_pgsql.so config_file=/var/secrets/ldap/pam_pgsql
 361   '';
 362   services.saslauthd = {
 363     enable = true;
 364     mechanism = "pam";
 365   };
 366   environment.etc."sasl2/slapd.conf".text = ''
 367     mech_list: plain
 368     pwcheck_method: saslauthd
 369     saslauthd_path: /run/saslauthd/mux
 370   '';
 371 }