First attempt at making declarative VMs

In order to make buildbot more secure, the builds need to happen inside
VMs so that they can be thrown out on demand when not needed.

This commit implements this facility on dilion, and also defines
declaratively some previous VMs which used to run on the machine.

4 files changed, 328 insertions, 0 deletions

diff --git a/modules/private/system/dilion/vms.nix b/modules/private/system/dilion/vms.nix
new file mode 100644
index 0000000..8d5a57b
--- /dev/null
+++ b/modules/private/system/dilion/vms.nix
@@ -0,0 +1,146 @@
+# inspired from https://nixos.wiki/wiki/Virtualization_in_NixOS
+{ config, pkgs, lib, ... }@args:
+let
+  networks = {
+    immae = {
+      bridgeNumber = "1";
+      ipRange = "192.168.100";
+    };
+  };
+  guests = {
+    caldance = {
+      pool = "zfspool";
+      cpus = "1";
+      memory = "2";
+      network = "immae";
+      diskSize = "10GiB";
+      extraDevicesXML = ''
+        <filesystem type="mount">
+          <source dir="/var/lib/caldance"/>
+          <target dir="home"/>
+        </filesystem>
+      '';
+    };
+    buildbot = {
+      pool = "zfspool";
+      cpus = "1";
+      memory = "3";
+      network = "immae";
+      diskSize = "10GiB";
+      destroyVolumeOnExit = true;
+      preStart = ''
+        if ! ${pkgs.libvirt}/bin/virsh pool-info --pool niximages &> /dev/null; then
+          pool-create-as --name niximages --type dir --target /etc/libvirtd/base-images/
+        fi
+        if ! ${pkgs.libvirt}/bin/virsh pool-info --pool buildbot-disks &> /dev/null; then
+          mkdir -p /var/lib/libvirt/images/buildbot-disks
+          pool-create-as --name buildbot-disks --type dir --target /var/lib/libvirt/images/buildbot-disks
+        fi
+      '';
+    };
+  };
+  toImage = f: "${import ./vms/base_image.nix f (args // { myEnv = config.myEnv; })}/nixos.qcow2";
+in
+{
+  environment.etc."libvirtd/base-images/nixos.qcow2".source = toImage ./vms/base_configuration.nix;
+  environment.etc."libvirtd/base-images/buildbot.qcow2".source = toImage ./vms/buildbot_configuration.nix;
+  systemd.services = lib.mapAttrs' (name: guest: lib.nameValuePair "libvirtd-guest-${name}" {
+    after = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
+    requires = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = "yes";
+    };
+    script =
+      let
+        xml = pkgs.writeText "libvirt-guest-${name}.xml"
+        ''
+          <domain type="kvm">
+            <name>${name}</name>
+            <uuid>UUID</uuid>
+            <memory unit="GiB">${guest.memory}</memory>
+            <vcpu>${guest.cpus}</vcpu>
+            <os>
+              <type arch="x86_64">hvm</type>
+            </os>
+            <devices>
+              <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
+              <disk type="volume">
+                <source pool="${guest.pool}" volume="guest-${name}" />
+                <target dev="vda" bus="virtio"/>
+              </disk>
+              ${guest.extraDevicesXML or ""}
+              <input type="keyboard" bus="usb"/>
+              <graphics type="vnc" port="-1" autoport="yes"/>
+              <interface type="network">
+                <source network="${guest.network}" />
+              </interface>
+            </devices>
+            <features>
+              <acpi/>
+            </features>
+          </domain>
+        '';
+      in
+      guest.preStart or "" + ''
+        if ! ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
+          ${pkgs.libvirt}/bin/virsh vol-create-as --pool ${guest.pool} --name 'guest-${name}' --capacity '${guest.diskSize}'
+          volume_path=$(${pkgs.libvirt}/bin/virsh vol-path --pool ${guest.pool} --vol 'guest-${name}')
+          ${pkgs.qemu}/bin/qemu-img convert /etc/libvirtd/base-images/nixos.qcow2 $volume_path
+        fi
+        uuid="$(${pkgs.libvirt}/bin/virsh domuuid '${name}' || true)"
+        ${pkgs.libvirt}/bin/virsh define <(sed "s/UUID/$uuid/" '${xml}')
+        ${pkgs.libvirt}/bin/virsh start '${name}'
+      '';
+    preStop = ''
+      ${pkgs.libvirt}/bin/virsh shutdown '${name}'
+      let "timeout = $(date +%s) + 10"
+      while [ "$(${pkgs.libvirt}/bin/virsh list --name | grep --count '^${name}$')" -gt 0 ]; do
+        if [ "$(date +%s)" -ge "$timeout" ]; then
+          # Meh, we warned it...
+          ${pkgs.libvirt}/bin/virsh destroy '${name}'
+        else
+          # The machine is still running, let's give it some time to shut down
+          sleep 0.5
+        fi
+      done
+    '' + lib.optionalString (guest.destroyVolumeOnExit or false) ''
+        if ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
+          ${pkgs.libvirt}/bin/virsh vol-wipe --pool ${guest.pool} --vol 'guest-${name}' || true
+          ${pkgs.libvirt}/bin/virsh vol-delete --pool ${guest.pool} --vol 'guest-${name}'
+        fi
+    '';
+  }) guests // (lib.mapAttrs' (name: network: lib.nameValuePair "libvirtd-network-${name}" {
+    after = [ "libvirtd.service" ];
+    requires = [ "libvirtd.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = "yes";
+    };
+    script = let
+      xml = pkgs.writeText "libvirt-network-${name}.xml" ''
+        <network>
+          <name>${name}</name>
+          <uuid>UUID</uuid>
+          <forward mode='nat' />
+          <bridge name='virbr${network.bridgeNumber}' />
+          <domain name='${name}' localOnly='yes'/>
+          <ip address='${network.ipRange}.1' netmask='255.255.255.0'>
+            <dhcp>
+              <range start='${network.ipRange}.2' end='${network.ipRange}.254'/>
+            </dhcp>
+          </ip>
+        </network>
+      '';
+    in ''
+      uuid="$(${pkgs.libvirt}/bin/virsh net-uuid '${name}' || true)"
+      ${pkgs.libvirt}/bin/virsh net-define <(sed "s/UUID/$uuid/" '${xml}')
+      ${pkgs.libvirt}/bin/virsh net-start '${name}'
+    '';
+    preStop = ''
+      ${pkgs.libvirt}/bin/virsh net-destroy '${name}'
+    '';
+  }) networks);
+}
diff --git a/modules/private/system/dilion/vms/base_configuration.nix b/modules/private/system/dilion/vms/base_configuration.nix
new file mode 100644
index 0000000..e2caba2
--- /dev/null
+++ b/modules/private/system/dilion/vms/base_configuration.nix
@@ -0,0 +1,21 @@
+{ lib, config, ... }@args:
+{
+  options.myEnv = (import ../../../environment.nix (args // { name = "dummy"; })).options.myEnv;
+  config = {
+    fileSystems."/".device = "/dev/disk/by-label/nixos";
+    boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "virtio_balloon" "virtio_blk" "virtio_pci" "virtio_ring" ];
+    boot.loader = {
+      grub = {
+        version = 2;
+        device = "/dev/vda";
+      };
+      timeout = 0;
+    };
+    services.openssh.enable = true;
+    networking.firewall.allowedTCPPorts = [ 22 ];
+    users = {
+      mutableUsers = false;
+      users.root.openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.immae_dilion ];
+    };
+  };
+}
diff --git a/modules/private/system/dilion/vms/base_image.nix b/modules/private/system/dilion/vms/base_image.nix
new file mode 100644
index 0000000..8de8560
--- /dev/null
+++ b/modules/private/system/dilion/vms/base_image.nix
@@ -0,0 +1,94 @@
+configuration_file: { pkgs ? import <nixpkgs> {}, system ? builtins.currentSystem, myEnv, ... }:
+let
+  config = (import <nixpkgs/nixos/lib/eval-config.nix> {
+    inherit system;
+    modules = [ {
+      myEnv = myEnv;
+      imports = [ configuration_file ];
+
+      # We want our template image to be as small as possible, but the deployed image should be able to be
+      # of any size. Hence we resize on the first boot.
+      systemd.services.resize-main-fs = {
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig.Type = "oneshot";
+        script =
+          ''
+            # Resize main partition to fill whole disk
+            echo ", +" | ${pkgs.utillinux}/bin/sfdisk /dev/vda --no-reread -N 1
+            ${pkgs.parted}/bin/partprobe
+            # Resize filesystem
+            ${pkgs.e2fsprogs}/bin/resize2fs /dev/vda1
+          '';
+      };
+    } ];
+  }).config;
+in pkgs.vmTools.runInLinuxVM (
+  pkgs.runCommand "nixos-base-image"
+    {
+      memSize = 768;
+      preVM =
+        ''
+          mkdir $out
+          diskImage=image.qcow2
+          ${pkgs.vmTools.qemu}/bin/qemu-img create -f qcow2 $diskImage 2G
+          mv closure xchg/
+        '';
+      postVM =
+        ''
+          echo compressing VM image...
+          ${pkgs.vmTools.qemu}/bin/qemu-img convert -c $diskImage -O qcow2 $out/nixos.qcow2
+        '';
+      buildInputs = [ pkgs.utillinux pkgs.perl pkgs.parted pkgs.e2fsprogs ];
+      exportReferencesGraph =
+        [ "closure" config.system.build.toplevel ];
+    }
+    ''
+      # Create the partition
+      parted /dev/vda mklabel msdos
+      parted /dev/vda -- mkpart primary ext4 1M -1s
+
+      # Format the partition
+      mkfs.ext4 -L nixos /dev/vda1
+      mkdir /mnt
+      mount /dev/vda1 /mnt
+
+      for dir in dev proc sys; do
+        mkdir /mnt/$dir
+        mount --bind /$dir /mnt/$dir
+      done
+
+      storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure)
+      echo filling Nix store...
+      mkdir -p /mnt/nix/store
+      set -f
+      cp -prd $storePaths /mnt/nix/store
+      # The permissions will be set up incorrectly if the host machine is not running NixOS
+      chown -R 0:30000 /mnt/nix/store
+
+      mkdir -p /mnt/etc/nix
+      echo 'build-users-group = ' > /mnt/etc/nix/nix.conf
+
+      # Register the paths in the Nix database.
+      export USER=root
+      printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \
+          chroot /mnt ${config.nix.package.out}/bin/nix-store --load-db
+
+      # Create the system profile to allow nixos-rebuild to work.
+      chroot /mnt ${config.nix.package.out}/bin/nix-env \
+          -p /nix/var/nix/profiles/system --set ${config.system.build.toplevel}
+
+      # `nixos-rebuild' requires an /etc/NIXOS.
+      mkdir -p /mnt/etc/nixos
+      touch /mnt/etc/NIXOS
+
+      # `switch-to-configuration' requires a /bin/sh
+      mkdir -p /mnt/bin
+      ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh
+
+      # Generate the GRUB menu.
+      chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot
+
+      umount /mnt/{proc,dev,sys}
+      umount /mnt
+    ''
+)
diff --git a/modules/private/system/dilion/vms/buildbot_configuration.nix b/modules/private/system/dilion/vms/buildbot_configuration.nix
new file mode 100644
index 0000000..05b02d4
--- /dev/null
+++ b/modules/private/system/dilion/vms/buildbot_configuration.nix
@@ -0,0 +1,67 @@
+{ pkgs, config, lib, ... }:
+{
+  imports = [
+    <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
+    ./base_configuration.nix
+  ];
+  systemd.services.buildbot-worker.serviceConfig.ExecStartPre = let
+    cfg = config.services.buildbot-worker;
+    script = pkgs.writeScript "decode-dmi" ''
+      #!${pkgs.stdenv.shell}
+
+      mkdir -vp "${cfg.buildbotDir}"
+      varfile=${cfg.buildbotDir}/variables
+      rm $varfile || true
+      echo "[DEFAULT]" > $varfile
+      strings=$(${pkgs.dmidecode}/bin/dmidecode --oem-string count)
+      for i in $(seq 1 $strings); do
+        ${pkgs.dmidecode}/bin/dmidecode --oem-string $i >> $varfile
+      done
+      chown -R ${cfg.user}:${cfg.group} ${cfg.buildbotDir}
+    '';
+    in
+      lib.mkForce ["+${script}"];
+  systemd.services.buildbot-worker.serviceConfig.ExecStart = let
+      cfg = config.services.buildbot-worker;
+      tacFile = pkgs.writeText "buildbot-worker.tac" ''
+        import os
+        from io import open
+
+        from buildbot_worker.bot import Worker
+        from twisted.application import service
+
+        basedir = '${cfg.buildbotDir}'
+
+        # note: this line is matched against to check that this is a worker
+        # directory; do not edit it.
+        application = service.Application('buildbot-worker')
+
+        import configparser
+        config = config = configparser.ConfigParser()
+        config.read("${cfg.buildbotDir}/variables")
+        master_url_split = config["DEFAULT"]["buildbot_master_url"].split(':')
+        buildmaster_host = master_url_split[0]
+        port = int(master_url_split[1])
+        workername = config["DEFAULT"]["buildbot_worker_name"]
+
+        with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
+            passwd = passwd_file.read().strip('\r\n')
+        keepalive = ${toString cfg.keepalive}
+        umask = None
+        maxdelay = 300
+        numcpus = None
+        allow_shutdown = None
+
+        s = Worker(buildmaster_host, port, workername, passwd, basedir,
+                  keepalive, umask=umask, maxdelay=maxdelay,
+                  numcpus=numcpus, allow_shutdown=allow_shutdown)
+        s.setServiceParent(application)
+      '';
+    in
+      lib.mkForce "${cfg.package.pythonModule.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
+  services.buildbot-worker = {
+    enable = true;
+    workerPass = config.myEnv.buildbot.workerPassword;
+    packages = [ pkgs.git pkgs.gzip pkgs.openssh ];
+  };
+}