[perso/Immae/Config/Nix.git] / modules / private / system / eldiron.nix

{ privateFiles }:
{ config, pkgs, lib, ... }:
{
  deployment = {
    targetUser = "root";
    targetHost = config.hostEnv.ips.main.ip4;
    substituteOnDestination = true;
  };
  boot = {
    kernelModules = [ "kvm-intel" ];
    blacklistedKernelModules = [ "nvidiafb" ];
    loader.timeout = 1;
    loader.grub.devices = [ "/dev/sda" "/dev/sdb" ];
    kernel.sysctl = {
      # https://github.com/Netflix/security-bulletins/blob/master/advisories/third-party/2019-001.md
      "net.ipv4.tcp_sack" = 0;
    };
    supportedFilesystems = [ "zfs" ];
    kernelParams = ["zfs.zfs_arc_max=6442450944"];
    kernelPackages = pkgs.linuxPackages_latest;
    initrd.availableKernelModules = [ "ahci" "sd_mod" ];
    initrd.secrets = {
      "/boot/pass.key" = "/boot/pass.key";
    };
  };
  services.udev.extraRules = ''
    ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="c8:60:00:56:a0:88", NAME="eth0"
  '';
  nix.maxJobs = 8;
  powerManagement.cpuFreqGovernor = "powersave";
  myEnv = import "${privateFiles}/environment.nix" // { inherit privateFiles; };

  fileSystems = {
    # pools:
    #     zpool: ashift=12
    #     zfast: ashift=12
    # zfs:
    #     zpool/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
    #     zpool/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
    #     zpool/root/var: atime=on
    #     zfast/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
    #     zfast/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
    #     zfast/root/etc: ø
    #     zfast/root/nix: ø
    #     zfast/root/tmp: async=disabled
    #     zfast/root/var: atime=on
    #     zfast/root/var/lib: ø
    #     zfast/root/var/lib/mysql: logbias=throughput ; atime=off ; primarycache=metadata
    #     zfast/root/var/lib/postgresql: recordsize=8K ; atime=off ; logbias=throughput
    #     zfast/root/var/lib/postgresql/11.0: ø
    #     zfast/root/var/lib/postgresql/11.0/pg_wal: ø
    "/"     = { fsType = "zfs"; device = "zpool/root"; };
    "/boot" = { fsType = "ext4"; device = "/dev/disk/by-uuid/e6bb18fb-ff56-4b5f-ae9f-e60d40dc0622"; };
    "/etc"  = { fsType = "zfs"; device = "zpool/root/etc"; };
    "/nix"  = { fsType = "zfs"; device = "zfast/root/nix"; };
    "/tmp"  = { fsType = "zfs"; device = "zfast/root/tmp"; };
    "/var"  = { fsType = "zfs"; device = "zpool/root/var"; };
    "/var/lib/mysql" = { fsType = "zfs"; device = "zfast/root/var/lib/mysql"; };
    "/var/lib/postgresql"             = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql"; };
    "/var/lib/postgresql/11.0"        = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0"; };
    "/var/lib/postgresql/11.0/pg_wal" = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0/pg_wal"; };
  };
  swapDevices = [ { label = "swap1"; } { label = "swap2"; } ];
  hardware.enableRedistributableFirmware = true;

  services.zfs = {
    autoScrub = {
      enable = false;
    };
  };
  networking = {
    hostId = "8262ca33"; # generated with head -c4 /dev/urandom | od -A none -t x4
    firewall.enable = true;
    # FIXME: on next reboot, remove the /27 and the localCommands
    interfaces."eth0".ipv4.addresses =
      pkgs.lib.attrsets.mapAttrsToList
        (n: ips: { address = ips.ip4; prefixLength = 32; })
        (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips)
      ++ [ { address = config.hostEnv.ips.main.ip4; prefixLength = 27; } ];
    interfaces."eth0".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
      (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
      config.hostEnv.ips);
    defaultGateway = "176.9.151.65";
    localCommands = ''
      # FIXME: Those commands were added by nixops and may not be
      # actually needed
      ip -6 addr add '2a01:4f8:160:3445::/64' dev 'eth0' || true
      ip -4 route change '176.9.151.64/27' via '176.9.151.65' dev 'eth0' || true
      ip -6 route add default via 'fe80::1' dev eth0 || true
    '';
    nameservers = [
      "213.133.98.98"
      "213.133.99.99"
      "213.133.100.100"
      "2a01:4f8:0:a0a1::add:1010"
      "2a01:4f8:0:a102::add:9999"
      "2a01:4f8:0:a111::add:9898"
    ];
  };

  imports = builtins.attrValues (import ../..);

  myServices.buildbot.enable = true;
  myServices.databases.enable = true;
  myServices.gitolite.enable = true;
  myServices.monitoring.enable = true;
  myServices.irc.enable = true;
  myServices.pub.enable = true;
  myServices.tasks.enable = true;
  myServices.mpd.enable = true;
  myServices.dns.enable = true;
  myServices.certificates.enable = true;
  myServices.websites.enable = true;
  myServices.mail.enable = true;
  myServices.ejabberd.enable = true;
  myServices.vpn.enable = true;
  services.pure-ftpd.enable = true;
  services.duplyBackup.enable = false;
  services.duplyBackup.profiles.oldies.rootDir = "/var/lib/oldies";

  services.netdata.enable = true;
  services.netdata.config.global."memory mode" = "none";
  services.netdata.config.health."enabled" = "no";
  services.netdata.config.web.mode = "none";
  users.users."${config.services.netdata.user}".extraGroups = [ "keys" ];
  environment.etc."netdata/stream.conf".source = "/var/secrets/netdata-stream.conf";
  secrets.keys = [
    {
      dest = "netdata-stream.conf";
      user = config.services.netdata.user;
      group = config.services.netdata.group;
      permissions = "0400";
      text = ''
        [stream]
            enabled = yes
            destination = ${config.myEnv.monitoring.netdata_aggregator}
            api key = ${config.myEnv.monitoring.netdata_keys.eldiron}
      '';
    }
    {
      dest = "zrepl_backup/identity";
      user = "root";
      group = "root";
      permissions = "0400";
      text = config.myEnv.zrepl_backup.ssh_key.private;
    }
  ];
  programs.ssh.knownHosts.dilion = {
    hostNames = ["dilion.immae.eu"];
    publicKey = let
      profile = config.myEnv.rsync_backup.profiles.dilion;
    in
      "${profile.host_key_type} ${profile.host_key}";
  };

  services.cron = {
    enable = true;
    mailto = "cron@immae.eu";
    systemCronJobs = [
      ''
        0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "immae.eu.*Recipient address rejected"
        # Need a way to blacklist properly
        # 0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "NOQUEUE:"
        0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtp -g "status=bounced"
      ''
    ];
  };

  fileSystems."/var/lib/pub/immae/devtools" = {
    device = "/run/current-system/sw/bin/bindfs#/var/lib/ftp/devtools.immae.eu/";
    fsType = "fuse";
    options = [ "force-user=pub" "create-for-user=wwwrun" "create-for-group=wwwrun" ];
  };
  environment.systemPackages = [ pkgs.bindfs ];

  services.zrepl = {
    enable = true;
    config = let
      redis_dump = pkgs.writeScript "redis-dump" ''
        #! ${pkgs.stdenv.shell}
        ${pkgs.redis}/bin/redis-cli bgsave
      '';
    in ''
      jobs:
        - type: push
          # must not change
          name: "backup-to-dilion"
          filesystems:
            "zpool/root": true
            "zpool/root/etc": true
            "zpool/root/var<": true
          connect:
            type: ssh+stdinserver
            host: dilion.immae.eu
            user: backup
            port: 22
            identity_file: ${config.secrets.fullPaths."zrepl_backup/identity"}
          snapshotting:
            type: periodic
            prefix: zrepl_
            interval: 1h
            #hooks:
            #  - type: mysql-lock-tables
            #    dsn: "${config.myEnv.zrepl_backup.mysql.user}:${config.myEnv.zrepl_backup.mysql.password}@tcp(localhost)/"
            #    filesystems:
            #      "zpool/root/var": true
            #  - type: command
            #    path: ${redis_dump}
            #    err_is_fatal: false
            #    filesystems:
            #      "zpool/root/var": true
          send:
            encrypted: true
          pruning:
            keep_sender:
              - type: regex
                regex: "^manual_.*"
              - type: grid
                grid: 24x1h | 7x1d | 4x7d | 6x30d
                regex: "^zrepl_.*"
            keep_receiver:
              - type: regex
                regex: "^manual_.*"
              - type: grid
                grid: 6x4h | 7x1d | 4x7d | 6x30d
                regex: "^zrepl_.*"
    '';
  };
  # This value determines the NixOS release with which your system is
  # to be compatible, in order to avoid breaking some software such as
  # database servers. You should change this only after NixOS release
  # notes say you should.
  # https://nixos.org/nixos/manual/release-notes.html
  system.stateVersion = "20.03"; # Did you read the comment?
}
Commit	Line	Data
f807d917	1	{ privateFiles }:
7e214bf9	2	{ config, pkgs, lib, ... }:
f807d917	3	{
34abd6af IB	4	deployment = {
	5	targetUser = "root";
	6	targetHost = config.hostEnv.ips.main.ip4;
	7	substituteOnDestination = true;
	8	};
	9	boot = {
	10	kernelModules = [ "kvm-intel" ];
	11	blacklistedKernelModules = [ "nvidiafb" ];
	12	loader.timeout = 1;
	13	loader.grub.devices = [ "/dev/sda" "/dev/sdb" ];
	14	kernel.sysctl = {
	15	# https://github.com/Netflix/security-bulletins/blob/master/advisories/third-party/2019-001.md
	16	"net.ipv4.tcp_sack" = 0;
	17	};
	18	supportedFilesystems = [ "zfs" ];
	19	kernelParams = ["zfs.zfs_arc_max=6442450944"];
	20	kernelPackages = pkgs.linuxPackages_latest;
	21	initrd.availableKernelModules = [ "ahci" "sd_mod" ];
	22	initrd.secrets = {
	23	"/boot/pass.key" = "/boot/pass.key";
	24	};
	25	};
	26	services.udev.extraRules = ''
	27	ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="c8:60:00:56:a0:88", NAME="eth0"
	28	'';
	29	nix.maxJobs = 8;
	30	powerManagement.cpuFreqGovernor = "powersave";
ab8f306d	31	myEnv = import "${privateFiles}/environment.nix" // { inherit privateFiles; };
f807d917	32
7e214bf9	33	fileSystems = {
ef909e24 IB	34	# pools:
	35	# zpool: ashift=12
	36	# zfast: ashift=12
	37	# zfs:
	38	# zpool/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
	39	# zpool/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
	40	# zpool/root/var: atime=on
	41	# zfast/: acltype=posixacl ; xattr=sa ; atime=off ; mountpoint=legacy
	42	# zfast/root: encryption=on ; keyformat=passphrase ; keylocation=file:///boot/pass.key
	43	# zfast/root/etc: ø
	44	# zfast/root/nix: ø
	45	# zfast/root/tmp: async=disabled
	46	# zfast/root/var: atime=on
	47	# zfast/root/var/lib: ø
	48	# zfast/root/var/lib/mysql: logbias=throughput ; atime=off ; primarycache=metadata
	49	# zfast/root/var/lib/postgresql: recordsize=8K ; atime=off ; logbias=throughput
	50	# zfast/root/var/lib/postgresql/11.0: ø
	51	# zfast/root/var/lib/postgresql/11.0/pg_wal: ø
34abd6af	52	"/" = { fsType = "zfs"; device = "zpool/root"; };
7e214bf9 IB	53	"/boot" = { fsType = "ext4"; device = "/dev/disk/by-uuid/e6bb18fb-ff56-4b5f-ae9f-e60d40dc0622"; };
7e214bf9 IB	54	"/etc" = { fsType = "zfs"; device = "zpool/root/etc"; };
ef909e24 IB	55	"/nix" = { fsType = "zfs"; device = "zfast/root/nix"; };
ef909e24 IB	56	"/tmp" = { fsType = "zfs"; device = "zfast/root/tmp"; };
7e214bf9	57	"/var" = { fsType = "zfs"; device = "zpool/root/var"; };
ef909e24 IB	58	"/var/lib/mysql" = { fsType = "zfs"; device = "zfast/root/var/lib/mysql"; };
	59	"/var/lib/postgresql" = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql"; };
	60	"/var/lib/postgresql/11.0" = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0"; };
	61	"/var/lib/postgresql/11.0/pg_wal" = { fsType = "zfs"; device = "zfast/root/var/lib/postgresql/11.0/pg_wal"; };
7e214bf9	62	};
34abd6af IB	63	swapDevices = [ { label = "swap1"; } { label = "swap2"; } ];
34abd6af IB	64	hardware.enableRedistributableFirmware = true;
7e214bf9 IB	65
7e214bf9 IB	66	services.zfs = {
7e214bf9	67	autoScrub = {
05a3b252	68	enable = false;
7e214bf9 IB	69	};
7e214bf9 IB	70	};
f807d917	71	networking = {
31e11cdf	72	hostId = "8262ca33"; # generated with head -c4 /dev/urandom \| od -A none -t x4
f807d917	73	firewall.enable = true;
05a3b252	74	# FIXME: on next reboot, remove the /27 and the localCommands
34abd6af IB	75	interfaces."eth0".ipv4.addresses =
	76	pkgs.lib.attrsets.mapAttrsToList
	77	(n: ips: { address = ips.ip4; prefixLength = 32; })
	78	(pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips)
	79	++ [ { address = config.hostEnv.ips.main.ip4; prefixLength = 27; } ];
f807d917 IB	80	interfaces."eth0".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
f807d917 IB	81	(n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
619e4f46	82	config.hostEnv.ips);
34abd6af IB	83	defaultGateway = "176.9.151.65";
	84	localCommands = ''
	85	# FIXME: Those commands were added by nixops and may not be
	86	# actually needed
	87	ip -6 addr add '2a01:4f8:160:3445::/64' dev 'eth0' \|\| true
	88	ip -4 route change '176.9.151.64/27' via '176.9.151.65' dev 'eth0' \|\| true
	89	ip -6 route add default via 'fe80::1' dev eth0 \|\| true
	90	'';
	91	nameservers = [
	92	"213.133.98.98"
	93	"213.133.99.99"
	94	"213.133.100.100"
	95	"2a01:4f8:0:a0a1::add:1010"
	96	"2a01:4f8:0:a102::add:9999"
	97	"2a01:4f8:0:a111::add:9898"
	98	];
f807d917 IB	99	};
	100
	101	imports = builtins.attrValues (import ../..);
	102
	103	myServices.buildbot.enable = true;
	104	myServices.databases.enable = true;
	105	myServices.gitolite.enable = true;
3bc32d9e	106	myServices.monitoring.enable = true;
f807d917 IB	107	myServices.irc.enable = true;
	108	myServices.pub.enable = true;
	109	myServices.tasks.enable = true;
8415083e IB	110	myServices.mpd.enable = true;
	111	myServices.dns.enable = true;
	112	myServices.certificates.enable = true;
	113	myServices.websites.enable = true;
	114	myServices.mail.enable = true;
3f453c7d	115	myServices.ejabberd.enable = true;
ea9c6fe8	116	myServices.vpn.enable = true;
f807d917	117	services.pure-ftpd.enable = true;
5b42284e	118	services.duplyBackup.enable = false;
213c323a	119	services.duplyBackup.profiles.oldies.rootDir = "/var/lib/oldies";
f807d917	120
e43fdf34 IB	121	services.netdata.enable = true;
	122	services.netdata.config.global."memory mode" = "none";
	123	services.netdata.config.health."enabled" = "no";
	124	services.netdata.config.web.mode = "none";
c081a03a IB	125	users.users."${config.services.netdata.user}".extraGroups = [ "keys" ];
c081a03a IB	126	environment.etc."netdata/stream.conf".source = "/var/secrets/netdata-stream.conf";
8bf83d7a	127	secrets.keys = [
c081a03a IB	128	{
	129	dest = "netdata-stream.conf";
	130	user = config.services.netdata.user;
	131	group = config.services.netdata.group;
	132	permissions = "0400";
	133	text = ''
	134	[stream]
	135	enabled = yes
	136	destination = ${config.myEnv.monitoring.netdata_aggregator}
	137	api key = ${config.myEnv.monitoring.netdata_keys.eldiron}
	138	'';
	139	}
8bf83d7a	140	{
5dda316b	141	dest = "zrepl_backup/identity";
8bf83d7a IB	142	user = "root";
	143	group = "root";
	144	permissions = "0400";
5dda316b	145	text = config.myEnv.zrepl_backup.ssh_key.private;
8bf83d7a IB	146	}
	147	];
	148	programs.ssh.knownHosts.dilion = {
	149	hostNames = ["dilion.immae.eu"];
	150	publicKey = let
	151	profile = config.myEnv.rsync_backup.profiles.dilion;
	152	in
	153	"${profile.host_key_type} ${profile.host_key}";
	154	};
	155
f807d917 IB	156	services.cron = {
f807d917 IB	157	enable = true;
4227853a	158	mailto = "cron@immae.eu";
f807d917 IB	159	systemCronJobs = [
f807d917 IB	160	''
fb7611c1 IB	161	0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "immae.eu.*Recipient address rejected"
	162	# Need a way to blacklist properly
	163	# 0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtpd -g "NOQUEUE:"
	164	0 0 * * * root journalctl -q --since="25 hours ago" -u postfix -t postfix/smtp -g "status=bounced"
f807d917 IB	165	''
	166	];
	167	};
	168
f831eb7e IB	169	fileSystems."/var/lib/pub/immae/devtools" = {
	170	device = "/run/current-system/sw/bin/bindfs#/var/lib/ftp/devtools.immae.eu/";
	171	fsType = "fuse";
	172	options = [ "force-user=pub" "create-for-user=wwwrun" "create-for-group=wwwrun" ];
	173	};
740a6506	174	environment.systemPackages = [ pkgs.bindfs ];
f831eb7e	175
5dda316b IB	176	services.zrepl = {
	177	enable = true;
	178	config = let
	179	redis_dump = pkgs.writeScript "redis-dump" ''
	180	#! ${pkgs.stdenv.shell}
	181	${pkgs.redis}/bin/redis-cli bgsave
	182	'';
	183	in ''
	184	jobs:
	185	- type: push
	186	# must not change
	187	name: "backup-to-dilion"
	188	filesystems:
	189	"zpool/root": true
	190	"zpool/root/etc": true
	191	"zpool/root/var<": true
	192	connect:
	193	type: ssh+stdinserver
	194	host: dilion.immae.eu
	195	user: backup
	196	port: 22
	197	identity_file: ${config.secrets.fullPaths."zrepl_backup/identity"}
	198	snapshotting:
	199	type: periodic
	200	prefix: zrepl_
8b150321 IB	201	interval: 1h
	202	#hooks:
	203	# - type: mysql-lock-tables
	204	# dsn: "${config.myEnv.zrepl_backup.mysql.user}:${config.myEnv.zrepl_backup.mysql.password}@tcp(localhost)/"
	205	# filesystems:
	206	# "zpool/root/var": true
	207	# - type: command
	208	# path: ${redis_dump}
	209	# err_is_fatal: false
	210	# filesystems:
	211	# "zpool/root/var": true
5dda316b IB	212	send:
	213	encrypted: true
	214	pruning:
	215	keep_sender:
5dda316b IB	216	- type: regex
	217	regex: "^manual_.*"
	218	- type: grid
8b150321	219	grid: 24x1h \| 7x1d \| 4x7d \| 6x30d
5dda316b IB	220	regex: "^zrepl_.*"
	221	keep_receiver:
	222	- type: regex
	223	regex: "^manual_.*"
	224	- type: grid
8b150321	225	grid: 6x4h \| 7x1d \| 4x7d \| 6x30d
5dda316b IB	226	regex: "^zrepl_.*"
	227	'';
	228	};
f807d917 IB	229	# This value determines the NixOS release with which your system is
	230	# to be compatible, in order to avoid breaking some software such as
	231	# database servers. You should change this only after NixOS release
	232	# notes say you should.
61ebe69b	233	# https://nixos.org/nixos/manual/release-notes.html
d43e0c61	234	system.stateVersion = "20.03"; # Did you read the comment?
f807d917	235	}