modules/private/system/eldiron.nix

   1 { config, pkgs, lib, ... }:
   2 {
   3   deployment = {
   4     targetUser = "root";
   5     targetHost = lib.head config.hostEnv.ips.main.ip4;
   6   };
   7   # ssh-keyscan eldiron | nix-shell -p ssh-to-age --run ssh-to-age
   8   secrets.ageKeys = [ "age1dxr5lhvtnjssfaqpnf6qx80h8gfwkxg3tdf35m6n9wljmk7wadfs3kmahj" ];
   9   boot = {
  10     kernelModules = [ "kvm-intel" ];
  11     blacklistedKernelModules = [ "nvidiafb" ];
  12     loader.timeout = 1;
  13     loader.grub.devices = [ "/dev/sda" "/dev/sdb" ];
  14     kernel.sysctl = {
  15       # https://github.com/Netflix/security-bulletins/blob/master/advisories/third-party/2019-001.md
  16       "net.ipv4.tcp_sack" = 0;
  17     };
  18     supportedFilesystems = [ "zfs" ];
  19     kernelParams = ["zfs.zfs_arc_max=6442450944"];
  20     kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
  21     initrd.availableKernelModules = [ "ahci" "sd_mod" ];
  22     initrd.secrets = {
  23       "/boot/pass.key" = "/boot/pass.key";
  24     };
  25   };
  26   services.udev.extraRules = ''
  27     ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="c8:60:00:56:a0:88", NAME="eth0"
  28   '';
  29   nix.settings.max-jobs = 8;
  30   powerManagement.cpuFreqGovernor = "powersave";
  31   myEnv = import ../../../nixops/secrets/environment.nix;
  32
  33   fileSystems = {
  34     # pools:
  35     #     zpool: ashift=12
  36     #     zfast: ashift=12
  37     # zfs:
  38     #     zpool/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
  39     #     zpool/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
  40     #     zpool/root/var: atime=on
  41     #     zfast/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
  42     #     zfast/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
  43     #     zfast/root/etc: ø
  44     #     zfast/root/nix: ø
  45     #     zfast/root/tmp: async=disabled
  46     #     zfast/root/var: atime=on
  47     #     zfast/root/var/lib: ø
  48     #     zfast/root/var/lib/mysql: logbias=throughput ; atime=off ; primarycache=metadata
  49     #     zfast/root/var/lib/postgresql: recordsize=8K ; atime=off ; logbias=throughput
  50     #     zfast/root/var/lib/postgresql/11.0: ø
  51     #     zfast/root/var/lib/postgresql/11.0/pg_wal: ø
  52     "/"     = { fsType = "zfs"; device = "zpool/root"; };
  53     "/boot" = { fsType = "ext4"; device = "/dev/disk/by-uuid/e6bb18fb-ff56-4b5f-ae9f-e60d40dc0622"; };
  54     "/etc"  = { fsType = "zfs"; device = "zpool/root/etc"; };
  55     "/nix"  = { fsType = "zfs"; device = "zfast/root/nix"; };
  56     "/tmp"  = { fsType = "zfs"; device = "zfast/root/tmp"; };
  57     "/var"  = { fsType = "zfs"; device = "zpool/root/var"; };
  58     "/var/lib/mysql" = { fsType = "zfs"; device = "zfast/root/var/lib/mysql"; };
  59     "/var/lib/postgresql"             = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql"; };
  60     "/var/lib/postgresql/11.0"        = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0"; };
  61     "/var/lib/postgresql/11.0/pg_wal" = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0/pg_wal"; };
  62   };
  63   swapDevices = [ { label = "swap1"; } { label = "swap2"; } ];
  64   hardware.enableRedistributableFirmware = true;
  65
  66   services.zfs = {
  67     autoScrub = {
  68       enable = false;
  69     };
  70   };
  71   networking = {
  72     hostId = "8262ca33"; # generated with head -c4 /dev/urandom | od -A none -t x4
  73     firewall.enable = true;
  74     firewall.allowedTCPPorts = [ config.myEnv.ports.zrepl_flony ];
  75     # FIXME: on next reboot, remove the /27 and the localCommands
  76     interfaces."eth0".ipv4.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
  77       (n: ips: map (ip: { address = ip; prefixLength = 32; }) (ips.ip4 or []))
  78       (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips))
  79       ++ [ { address = lib.head config.hostEnv.ips.main.ip4; prefixLength = 27; } ];
  80     interfaces."eth0".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
  81       (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
  82       config.hostEnv.ips);
  83     defaultGateway = "176.9.151.65";
  84     localCommands = ''
  85       # FIXME: Those commands were added by nixops and may not be
  86       # actually needed
  87       ip -6 addr add '2a01:4f8:160:3445::/64' dev 'eth0' || true
  88       ip -4 route change '176.9.151.64/27' via '176.9.151.65' dev 'eth0' || true
  89       ip -6 route add default via 'fe80::1' dev eth0 || true
  90     '';
  91     nameservers = [
  92       "213.133.98.98"
  93       "213.133.99.99"
  94       "213.133.100.100"
  95       "2a01:4f8:0:a0a1::add:1010"
  96       "2a01:4f8:0:a102::add:9999"
  97       "2a01:4f8:0:a111::add:9898"
  98     ];
  99   };
 100
 101   imports = builtins.attrValues (import ../..);
 102
 103   myServices.buildbot.enable = true;
 104   myServices.databases.enable = true;
 105   myServices.gitolite.enable = true;
 106   myServices.monitoring.enable = true;
 107   myServices.irc.enable = true;
 108   myServices.pub.enable = true;
 109   myServices.tasks.enable = true;
 110   myServices.mpd.enable = true;
 111   myServices.dns.enable = true;
 112   myServices.certificates.enable = true;
 113   myServices.websites.enable = true;
 114   myServices.gemini.enable = true;
 115   myServices.mail.enable = true;
 116   myServices.ejabberd.enable = true;
 117   myServices.vpn.enable = true;
 118   myServices.ftp.enable = true;
 119
 120   services.netdata.enable = true;
 121   services.netdata.config.global."memory mode" = "none";
 122   services.netdata.config.health."enabled" = "no";
 123   services.netdata.config.web.mode = "none";
 124   users.users."${config.services.netdata.user}".extraGroups = [ "keys" ];
 125   services.netdata.configDir."stream.conf" = config.secrets.fullPaths."netdata-stream.conf";
 126   secrets.keys = {
 127     "netdata-stream.conf" = {
 128       user = config.services.netdata.user;
 129       group = config.services.netdata.group;
 130       permissions = "0400";
 131       text = ''
 132         [stream]
 133             enabled = yes
 134             destination = ${config.myEnv.monitoring.netdata_aggregator}
 135             api key = ${config.myEnv.monitoring.netdata_keys.eldiron}
 136       '';
 137     };
 138     "zrepl_backup/identity" = {
 139       user = "root";
 140       group = "root";
 141       permissions = "0400";
 142       text = config.myEnv.zrepl_backup.ssh_key.private;
 143     };
 144   };
 145   programs.ssh.knownHosts.dilion = {
 146     extraHostNames = ["dilion.immae.eu"];
 147     publicKey = let
 148       profile = config.myEnv.rsync_backup.profiles.dilion;
 149     in
 150       "${profile.host_key_type} ${profile.host_key}";
 151   };
 152
 153   services.cron = {
 154     enable = true;
 155     mailto = "cron@immae.eu";
 156     systemCronJobs = [
 157       ''
 158         0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "immae.eu.*Recipient address rejected"
 159         # Need a way to blacklist properly
 160         # 0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "NOQUEUE:"
 161         0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtp -g "status=bounced"
 162       ''
 163     ];
 164   };
 165
 166   environment.systemPackages = [ pkgs.bindfs ];
 167
 168   immaeServices.zrepl = {
 169     enable = true;
 170     config = let
 171       redis_dump = pkgs.writeScript "redis-dump" ''
 172         #! ${pkgs.stdenv.shell}
 173         ${pkgs.redis}/bin/redis-cli bgsave
 174       '';
 175     in ''
 176       jobs:
 177         - type: push
 178           # must not change
 179           name: "backup-to-dilion"
 180           filesystems:
 181             "zpool/root": true
 182             "zpool/root/etc": true
 183             "zpool/root/var<": true
 184           connect:
 185             address: dilion.immae.eu:19000
 186             type: tls
 187             server_cn: dilion
 188             ca: ${config.secrets.fullPaths."zrepl/certificates/dilion.crt"}
 189             cert: ${config.secrets.fullPaths."zrepl/certificates/eldiron.crt"}
 190             key: ${config.secrets.fullPaths."zrepl/eldiron.key"}
 191           snapshotting:
 192             type: periodic
 193             prefix: zrepl_
 194             interval: 1h
 195             #hooks:
 196             #  - type: mysql-lock-tables
 197             #    dsn: "${config.myEnv.zrepl_backup.mysql.user}:${config.myEnv.zrepl_backup.mysql.password}@tcp(localhost)/"
 198             #    filesystems:
 199             #      "zpool/root/var": true
 200             #  - type: command
 201             #    path: ${redis_dump}
 202             #    err_is_fatal: false
 203             #    filesystems:
 204             #      "zpool/root/var": true
 205           send:
 206             encrypted: true
 207           pruning:
 208             keep_sender:
 209               - type: regex
 210                 regex: "^manual_.*"
 211               - type: grid
 212                 grid: 24x1h | 7x1d | 4x7d | 6x30d
 213                 regex: "^zrepl_.*"
 214             keep_receiver:
 215               - type: regex
 216                 regex: "^manual_.*"
 217               - type: grid
 218                 grid: 6x4h | 7x1d | 4x7d | 6x30d
 219                 regex: "^zrepl_.*"
 220         - type: source
 221           # must not change
 222           name: "backup-to-wd-zpool"
 223           serve:
 224             type: tls
 225             listen: :${builtins.toString config.myEnv.ports.zrepl_flony}
 226             ca: ${config.secrets.fullPaths."zrepl/certificates/flony.crt"}
 227             cert: ${config.secrets.fullPaths."zrepl/certificates/eldiron.crt"}
 228             key: ${config.secrets.fullPaths."zrepl/eldiron.key"}
 229             client_cns:
 230               - flony
 231           filesystems:
 232             "zpool/root": true
 233             "zpool/root/etc": true
 234             "zpool/root/var<": true
 235             "zfast/root/var<": true
 236           send:
 237             encrypted: true
 238           snapshotting:
 239             type: manual
 240     '';
 241   };
 242   # This value determines the NixOS release with which your system is
 243   # to be compatible, in order to avoid breaking some software such as
 244   # database servers. You should change this only after NixOS release
 245   # notes say you should.
 246   # https://nixos.org/nixos/manual/release-notes.html
 247   system.stateVersion = "20.03"; # Did you read the comment?
 248 }