First attempt at making declarative VMs

In order to make buildbot more secure, the builds need to happen inside
VMs so that they can be thrown out on demand when not needed.

This commit implements this facility on dilion, and also defines
declaratively some previous VMs which used to run on the machine.

10 files changed, 684 insertions, 10 deletions

diff --git a/modules/private/buildbot/common/libvirt.py b/modules/private/buildbot/common/libvirt.py
new file mode 100644
index 0000000..85fd908
--- /dev/null
+++ b/modules/private/buildbot/common/libvirt.py
@@ -0,0 +1,306 @@
+# This file was part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Portions Copyright Buildbot Team Members
+# Portions Copyright 2010 Isotoma Limited
+
+
+import os
+
+from twisted.internet import defer
+from twisted.internet import threads
+from twisted.internet import utils
+from twisted.python import failure
+from twisted.python import log
+
+from buildbot import config
+from buildbot.util.eventual import eventually
+from buildbot.worker import AbstractLatentWorker
+
+try:
+    import libvirt
+except ImportError:
+    libvirt = None
+
+import random
+import string
+
+def random_string_generator():
+    chars = string.ascii_letters
+    return ''.join(random.choice(chars) for x in range(6))
+
+class WorkQueue:
+
+    """
+    I am a class that turns parallel access into serial access.
+
+    I exist because we want to run libvirt access in threads as we don't
+    trust calls not to block, but under load libvirt doesn't seem to like
+    this kind of threaded use.
+    """
+
+    def __init__(self):
+        self.queue = []
+
+    def _process(self):
+        log.msg("Looking to start a piece of work now...")
+
+        # Is there anything to do?
+        if not self.queue:
+            log.msg("_process called when there is no work")
+            return
+
+        # Peek at the top of the stack - get a function to call and
+        # a deferred to fire when its all over
+        d, next_operation, args, kwargs = self.queue[0]
+
+        # Start doing some work - expects a deferred
+        try:
+            d2 = next_operation(*args, **kwargs)
+        except Exception:
+            d2 = defer.fail()
+
+        # Whenever a piece of work is done, whether it worked or not
+        # call this to schedule the next piece of work
+        @d2.addBoth
+        def _work_done(res):
+            log.msg("Completed a piece of work")
+            self.queue.pop(0)
+            if self.queue:
+                log.msg("Preparing next piece of work")
+                eventually(self._process)
+            return res
+
+        # When the work is done, trigger d
+        d2.chainDeferred(d)
+
+    def execute(self, cb, *args, **kwargs):
+        kickstart_processing = not self.queue
+        d = defer.Deferred()
+        self.queue.append((d, cb, args, kwargs))
+        if kickstart_processing:
+            self._process()
+        return d
+
+    def executeInThread(self, cb, *args, **kwargs):
+        return self.execute(threads.deferToThread, cb, *args, **kwargs)
+
+
+# A module is effectively a singleton class, so this is OK
+queue = WorkQueue()
+
+
+class Domain:
+
+    """
+    I am a wrapper around a libvirt Domain object
+    """
+
+    def __init__(self, connection, domain):
+        self.connection = connection
+        self.domain = domain
+
+    def name(self):
+        return queue.executeInThread(self.domain.name)
+
+    def create(self):
+        return queue.executeInThread(self.domain.create)
+
+    def shutdown(self):
+        return queue.executeInThread(self.domain.shutdown)
+
+    def destroy(self):
+        return queue.executeInThread(self.domain.destroy)
+
+class Volume:
+    def __init__(self, connection, volume):
+        self.connection = connection
+        self.volume = volume
+
+    @defer.inlineCallbacks
+    def destroy(self):
+        yield queue.executeInThread(self.volume.wipe)
+        yield queue.executeInThread(self.volume.delete)
+
+class Pool:
+    VolumeClass = Volume
+    def __init__(self, connection, pool):
+        self.connection = connection
+        self.pool = pool
+
+    @defer.inlineCallbacks
+    def create_volume(self, xml):
+        res = yield queue.executeInThread(self.pool.createXML, xml)
+        return self.VolumeClass(self.connection, res)
+
+class Connection:
+
+    """
+    I am a wrapper around a libvirt Connection object.
+    """
+
+    DomainClass = Domain
+    PoolClass = Pool
+
+    def __init__(self, uri):
+        self.uri = uri
+        self.connection = libvirt.open(uri)
+
+    @defer.inlineCallbacks
+    def create(self, xml):
+        """ I take libvirt XML and start a new VM """
+        res = yield queue.executeInThread(self.connection.createXML, xml, 0)
+        return self.DomainClass(self, res)
+
+    @defer.inlineCallbacks
+    def lookup_pool(self, name):
+        res = yield queue.executeInThread(self.connection.storagePoolLookupByName, name)
+        return self.PoolClass(self, res)
+
+class LibVirtWorker(AbstractLatentWorker):
+
+    def __init__(self, name, password, connection, master_url, base_image=None, **kwargs):
+        super().__init__(name, password, **kwargs)
+        if not libvirt:
+            config.error(
+                "The python module 'libvirt' is needed to use a LibVirtWorker")
+
+        self.master_url = master_url
+        self.random_name = random_string_generator()
+        self.connection = connection
+        self.base_image = base_image
+
+        self.domain = None
+        self.domain_name = "buildbot-" + self.workername + "-" + self.random_name
+        self.volume = None
+        self.volume_name = "buildbot-" + self.workername + "-" + self.random_name
+        self.pool_name = "buildbot-disks"
+
+    def reconfigService(self, *args, **kwargs):
+        if 'build_wait_timeout' not in kwargs:
+            kwargs['build_wait_timeout'] = 0
+        return super().reconfigService(*args, **kwargs)
+
+    def canStartBuild(self):
+        if self.domain and not self.isConnected():
+            log.msg(
+                "Not accepting builds as existing domain but worker not connected")
+            return False
+
+        return super().canStartBuild()
+
+    @defer.inlineCallbacks
+    def _prepare_image(self):
+        log.msg("Creating temporary image {}".format(self.volume_name))
+        pool = yield self.connection.lookup_pool(self.pool_name)
+        vol_xml = """
+            <volume type='file'>
+                <name>{vol_name}</name>
+                <capacity unit='G'>10</capacity>
+                <target>
+                    <format type='qcow2'/>
+                    <permissions>
+                        <mode>0600</mode>
+                        <owner>0</owner>
+                        <group>0</group>
+                    </permissions>
+                </target>
+                <backingStore>
+                    <path>/etc/libvirtd/base-images/buildbot.qcow2</path>
+                    <format type='qcow2'/>
+                </backingStore>
+            </volume>
+        """.format(vol_name = self.volume_name)
+        self.volume = yield pool.create_volume(vol_xml)
+
+    @defer.inlineCallbacks
+    def start_instance(self, build):
+        """
+        I start a new instance of a VM.
+
+        If a base_image is specified, I will make a clone of that otherwise i will
+        use image directly.
+
+        If i'm not given libvirt domain definition XML, I will look for my name
+        in the list of defined virtual machines and start that.
+        """
+        domain_xml = """
+            <domain type="kvm">
+                <name>{domain_name}</name>
+                <memory unit="GiB">2</memory>
+                <vcpu>1</vcpu>
+                <sysinfo type='smbios'>
+                    <oemStrings>
+                        <entry>buildbot_master_url={master_url}</entry>
+                        <entry>buildbot_worker_name={worker_name}</entry>
+                    </oemStrings>
+                </sysinfo>
+                <os>
+                    <type arch="x86_64">hvm</type>
+                    <smbios mode='sysinfo'/>
+                </os>
+                <devices>
+                    <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
+                    <disk type="volume" device="disk">
+                        <driver name='qemu' type='qcow2' />
+                        <source type="volume" pool="{pool_name}" volume="{volume_name}" />
+                        <backingStore type='volume'>
+                        <format type='qcow2'/>
+                        <source type="volume" pool="niximages" volume="buildbot.qcow2" />
+                        </backingStore>
+                        <target dev="vda" bus="virtio"/>
+                    </disk>
+                    <input type="keyboard" bus="usb"/>
+                    <graphics type="vnc" port="-1" autoport="yes"/>
+                    <interface type="network">
+                        <source network="immae" />
+                    </interface>
+                </devices>
+            </domain>
+        """.format(volume_name = self.volume_name, master_url = self.master_url, pool_name =
+                self.pool_name, domain_name = self.domain_name, worker_name = self.workername)
+
+        yield self._prepare_image()
+
+        try:
+            self.domain = yield self.connection.create(domain_xml)
+        except Exception:
+            log.err(failure.Failure(),
+                    ("Cannot start a VM ({}), failing gracefully and triggering"
+                     "a new build check").format(self.workername))
+            self.domain = None
+            return False
+
+        return [self.domain_name]
+
+    def stop_instance(self, fast=False):
+        """
+        I attempt to stop a running VM.
+        I make sure any connection to the worker is removed.
+        If the VM was using a cloned image, I remove the clone
+        When everything is tidied up, I ask that bbot looks for work to do
+        """
+
+        log.msg("Attempting to stop '{}'".format(self.workername))
+        if self.domain is None:
+            log.msg("I don't think that domain is even running, aborting")
+            return defer.succeed(None)
+
+        domain = self.domain
+        self.domain = None
+
+        d = domain.destroy()
+        if self.volume is not None:
+            self.volume.destroy()
+
+        return d
diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix
index d6753e5..ac34845 100644
--- a/modules/private/buildbot/default.nix
+++ b/modules/private/buildbot/default.nix
@@ -107,7 +107,12 @@ in
                 project_env = with lib.attrsets;
                   mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment //
                   mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) //
-                  { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; };
+                  {
+                    BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}";
+                    BUILDBOT_WORKER_PORT = builtins.toString project.workerPort;
+                    BUILDBOT_HOST = config.hostEnv.fqdn;
+                    BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system";
+                  };
                 in builtins.concatStringsSep "\n"
                   (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env);
             }
@@ -126,6 +131,13 @@ in
         permissions = "0600";
         user = "buildbot";
         group = "buildbot";
+        text = config.myEnv.buildbot.workerPassword;
+        dest = "buildbot/worker_password";
+      }
+      {
+        permissions = "0600";
+        user = "buildbot";
+        group = "buildbot";
         text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key";
         dest = "buildbot/ssh_key";
       }
@@ -135,6 +147,7 @@ in
       restart = true;
       paths = [
         "/var/secrets/buildbot/ldap"
+        "/var/secrets/buildbot/worker_password"
         "/var/secrets/buildbot/ssh_key"
         "/var/secrets/buildbot/${project.name}/environment_file"
       ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets;
@@ -144,6 +157,7 @@ in
       description = "buildbot slice";
     };
 
+    networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects;
     systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
       description = "Buildbot Continuous Integration Server ${project.name}.";
       after = [ "network-online.target" ];
@@ -196,6 +210,7 @@ in
       buildbot_secrets=${varDir}/${project.name}/secrets
       install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
       install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap
+      install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/worker_password $buildbot_secrets/worker_password
       ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
         (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets
       )}
@@ -213,6 +228,7 @@ in
         });
         HOME = "${varDir}/${project.name}";
         PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
+          pkgs.python3Packages.libvirt
           pkgs.python3Packages.wokkel
           pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
           pkgs.python3Packages.buildbot-worker
diff --git a/modules/private/buildbot/projects/test/__init__.py b/modules/private/buildbot/projects/test/__init__.py
index e6b8d51..e2f6f82 100644
--- a/modules/private/buildbot/projects/test/__init__.py
+++ b/modules/private/buildbot/projects/test/__init__.py
@@ -1,5 +1,6 @@
 from buildbot.plugins import *
 from buildbot_common.build_helpers import *
+import buildbot_common.libvirt as ilibvirt
 import os
 from buildbot.util import bytes2unicode
 import json
@@ -10,11 +11,13 @@ class E():
     PROJECT       = "test"
     BUILDBOT_URL  = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
     SOCKET        = "unix:/run/buildbot/{}.sock".format(PROJECT)
-    PB_SOCKET     = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT)
+    PB_SOCKET     = os.environ["BUILDBOT_WORKER_PORT"]
+    WORKER_HOST   = "{}:{}".format(os.environ["BUILDBOT_HOST"], PB_SOCKET)
     RELEASE_PATH  = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT)
     RELEASE_URL   = "https://release.immae.eu/{}".format(PROJECT)
     GIT_URL       = "https://git.immae.eu/perso/Immae/TestProject.git"
     SSH_KEY_PATH  = "/var/lib/buildbot/buildbot_key"
+    LIBVIRT_URL   = os.environ["BUILDBOT_VIRT_URL"] + "?keyfile=" + SSH_KEY_PATH
     PUPPET_HOST   = "root@backup-1.v.immae.eu"
     LDAP_HOST     = "ldap.immae.eu"
     LDAP_DN       = "cn=buildbot,ou=services,dc=immae,dc=eu"
@@ -70,8 +73,14 @@ def configure(c):
     c["www"]["change_hook_dialects"]["base"] = {
             "custom_class": CustomBase
             }
-    c['workers'].append(worker.LocalWorker("generic-worker-test"))
-    c['workers'].append(worker.LocalWorker("deploy-worker-test"))
+    c['workers'].append(ilibvirt.LibVirtWorker("test-build",
+                                             open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
+                                             ilibvirt.Connection(E.LIBVIRT_URL),
+                                             E.WORKER_HOST))
+    c['workers'].append(ilibvirt.LibVirtWorker("test-deploy",
+                                             open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
+                                             ilibvirt.Connection(E.LIBVIRT_URL),
+                                             E.WORKER_HOST))
 
     c['schedulers'].append(hook_scheduler("TestProject", timer=1))
     c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"]))
@@ -109,7 +118,7 @@ def factory():
         logEnviron=False, command=["echo", package]))
     factory.addSteps(package_and_upload(package, package_dest, package_url))
 
-    return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory)
+    return util.BuilderConfig(name="TestProject_build", workernames=["test-build"], factory=factory)
 
 
 def compute_build_infos():
@@ -143,7 +152,7 @@ def deploy_factory():
         ldap_password=util.Secret("ldap")))
     factory.addStep(steps.MasterShellCommand(command=[
         "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host]))
-    return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory)
+    return util.BuilderConfig(name="TestProject_deploy", workernames=["test-deploy"], factory=factory)
 
 from twisted.internet import defer
 from buildbot.process.buildstep import FAILURE
diff --git a/modules/private/environment.nix b/modules/private/environment.nix
index 719bf8f..f0af572 100644
--- a/modules/private/environment.nix
+++ b/modules/private/environment.nix
@@ -228,6 +228,7 @@ in
       '';
       type = submodule {
         options = {
+          rootKeys = mkOption { type = attrsOf str; description = "Keys of root users"; };
           ldap = mkOption {
             description = ''
               LDAP credentials for cn=ssh,ou=services,dc=immae,dc=eu dn
@@ -804,6 +805,7 @@ in
       description = "Buildbot configuration";
       type = submodule {
         options = {
+          workerPassword = mkOption { description = "Buildbot worker password"; type = str; };
           user = mkOption {
             description = "Buildbot user";
             type = submodule {
@@ -855,6 +857,7 @@ in
                     '';
                 };
                 pythonPathHome = mkOption { type = bool; description = "Whether to add project’s python home to python path"; };
+                workerPort = mkOption { type = port; description = "Port for the worker"; };
                 secrets = mkOption {
                   type = attrsOf str;
                   description = "Secrets for the project to dump as files";
diff --git a/modules/private/gitolite/default.nix b/modules/private/gitolite/default.nix
index 6b573e3..e54ee8a 100644
--- a/modules/private/gitolite/default.nix
+++ b/modules/private/gitolite/default.nix
@@ -74,7 +74,7 @@ in {
     # Installation: https://git.immae.eu/mantisbt/view.php?id=93
     services.gitolite = {
       enable = true;
-      adminPubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXqRbiHw7QoHADNIEuo4nUT9fSOIEBMdJZH0bkQAxXyJFyCM1IMz0pxsHV0wu9tdkkr36bPEUj2aV5bkYLBN6nxcV2Y49X8bjOSCPfx3n6Own1h+NeZVBj4ZByrFmqCbTxUJIZ2bZKcWOFncML39VmWdsVhNjg0X4NBBehqXRIKr2gt3E/ESAxTYJFm0BnU0baciw9cN0bsRGqvFgf5h2P48CIAfwhVcGmPQnnAwabnosYQzRWxR0OygH5Kd8mePh6FheIRIigfXsDO8f/jdxwut8buvNIf3m5EBr3tUbTsvM+eV3M5vKGt7sk8T64DVtepTSdOOWtp+47ktsnHOMh immae@immae.eu";
+      adminPubkey = config.myEnv.sshd.rootKeys.immae_dilion;
     };
   };
 }
diff --git a/modules/private/system/dilion.nix b/modules/private/system/dilion.nix
index be8269e..a59d607 100644
--- a/modules/private/system/dilion.nix
+++ b/modules/private/system/dilion.nix
@@ -76,12 +76,24 @@
   };
 
   myServices.ssh.modules = [ config.myServices.ssh.predefinedModules.regular ];
-  imports = builtins.attrValues (import ../..);
+  imports = builtins.attrValues (import ../..) ++ [ ./dilion/vms.nix ];
 
   system.nssModules = [ pkgs.libvirt ];
   system.nssDatabases.hosts = lib.mkForce [ "files" "libvirt_guest" "mymachines" "dns" "myhostname" ];
   programs.zsh.enable = true;
 
+  users.users.libvirt = {
+    hashedPassword = "!";
+    shell = pkgs.bashInteractive;
+    isSystemUser = true;
+    group = "libvirtd";
+    packages = [ pkgs.netcat-openbsd ];
+    openssh.authorizedKeys.keyFiles = [
+      "${privateFiles}/buildbot_ssh_key.pub"
+    ];
+    openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.ismael_flony ];
+  };
+
   users.users.backup = {
     hashedPassword = "!";
     isSystemUser = true;
@@ -118,7 +130,7 @@
     after = [ "network.target" ];
 
     serviceConfig = {
-      ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:22";
+      ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:caldance:22";
     };
   };
 
@@ -170,7 +182,7 @@
     recommendedGzipSettings = true;
     recommendedProxySettings = true;
     upstreams = {
-      caldance.servers."nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:3031" = {};
+      caldance.servers."caldance:3031" = {};
     };
     virtualHosts = {
       "dev.immae.eu" = {
diff --git a/modules/private/system/dilion/vms.nix b/modules/private/system/dilion/vms.nix
new file mode 100644
index 0000000..8d5a57b
--- /dev/null
+++ b/modules/private/system/dilion/vms.nix
@@ -0,0 +1,146 @@
+# inspired from https://nixos.wiki/wiki/Virtualization_in_NixOS
+{ config, pkgs, lib, ... }@args:
+let
+  networks = {
+    immae = {
+      bridgeNumber = "1";
+      ipRange = "192.168.100";
+    };
+  };
+  guests = {
+    caldance = {
+      pool = "zfspool";
+      cpus = "1";
+      memory = "2";
+      network = "immae";
+      diskSize = "10GiB";
+      extraDevicesXML = ''
+        <filesystem type="mount">
+          <source dir="/var/lib/caldance"/>
+          <target dir="home"/>
+        </filesystem>
+      '';
+    };
+    buildbot = {
+      pool = "zfspool";
+      cpus = "1";
+      memory = "3";
+      network = "immae";
+      diskSize = "10GiB";
+      destroyVolumeOnExit = true;
+      preStart = ''
+        if ! ${pkgs.libvirt}/bin/virsh pool-info --pool niximages &> /dev/null; then
+          pool-create-as --name niximages --type dir --target /etc/libvirtd/base-images/
+        fi
+        if ! ${pkgs.libvirt}/bin/virsh pool-info --pool buildbot-disks &> /dev/null; then
+          mkdir -p /var/lib/libvirt/images/buildbot-disks
+          pool-create-as --name buildbot-disks --type dir --target /var/lib/libvirt/images/buildbot-disks
+        fi
+      '';
+    };
+  };
+  toImage = f: "${import ./vms/base_image.nix f (args // { myEnv = config.myEnv; })}/nixos.qcow2";
+in
+{
+  environment.etc."libvirtd/base-images/nixos.qcow2".source = toImage ./vms/base_configuration.nix;
+  environment.etc."libvirtd/base-images/buildbot.qcow2".source = toImage ./vms/buildbot_configuration.nix;
+  systemd.services = lib.mapAttrs' (name: guest: lib.nameValuePair "libvirtd-guest-${name}" {
+    after = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
+    requires = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = "yes";
+    };
+    script =
+      let
+        xml = pkgs.writeText "libvirt-guest-${name}.xml"
+        ''
+          <domain type="kvm">
+            <name>${name}</name>
+            <uuid>UUID</uuid>
+            <memory unit="GiB">${guest.memory}</memory>
+            <vcpu>${guest.cpus}</vcpu>
+            <os>
+              <type arch="x86_64">hvm</type>
+            </os>
+            <devices>
+              <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
+              <disk type="volume">
+                <source pool="${guest.pool}" volume="guest-${name}" />
+                <target dev="vda" bus="virtio"/>
+              </disk>
+              ${guest.extraDevicesXML or ""}
+              <input type="keyboard" bus="usb"/>
+              <graphics type="vnc" port="-1" autoport="yes"/>
+              <interface type="network">
+                <source network="${guest.network}" />
+              </interface>
+            </devices>
+            <features>
+              <acpi/>
+            </features>
+          </domain>
+        '';
+      in
+      guest.preStart or "" + ''
+        if ! ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
+          ${pkgs.libvirt}/bin/virsh vol-create-as --pool ${guest.pool} --name 'guest-${name}' --capacity '${guest.diskSize}'
+          volume_path=$(${pkgs.libvirt}/bin/virsh vol-path --pool ${guest.pool} --vol 'guest-${name}')
+          ${pkgs.qemu}/bin/qemu-img convert /etc/libvirtd/base-images/nixos.qcow2 $volume_path
+        fi
+        uuid="$(${pkgs.libvirt}/bin/virsh domuuid '${name}' || true)"
+        ${pkgs.libvirt}/bin/virsh define <(sed "s/UUID/$uuid/" '${xml}')
+        ${pkgs.libvirt}/bin/virsh start '${name}'
+      '';
+    preStop = ''
+      ${pkgs.libvirt}/bin/virsh shutdown '${name}'
+      let "timeout = $(date +%s) + 10"
+      while [ "$(${pkgs.libvirt}/bin/virsh list --name | grep --count '^${name}$')" -gt 0 ]; do
+        if [ "$(date +%s)" -ge "$timeout" ]; then
+          # Meh, we warned it...
+          ${pkgs.libvirt}/bin/virsh destroy '${name}'
+        else
+          # The machine is still running, let's give it some time to shut down
+          sleep 0.5
+        fi
+      done
+    '' + lib.optionalString (guest.destroyVolumeOnExit or false) ''
+        if ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
+          ${pkgs.libvirt}/bin/virsh vol-wipe --pool ${guest.pool} --vol 'guest-${name}' || true
+          ${pkgs.libvirt}/bin/virsh vol-delete --pool ${guest.pool} --vol 'guest-${name}'
+        fi
+    '';
+  }) guests // (lib.mapAttrs' (name: network: lib.nameValuePair "libvirtd-network-${name}" {
+    after = [ "libvirtd.service" ];
+    requires = [ "libvirtd.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = "yes";
+    };
+    script = let
+      xml = pkgs.writeText "libvirt-network-${name}.xml" ''
+        <network>
+          <name>${name}</name>
+          <uuid>UUID</uuid>
+          <forward mode='nat' />
+          <bridge name='virbr${network.bridgeNumber}' />
+          <domain name='${name}' localOnly='yes'/>
+          <ip address='${network.ipRange}.1' netmask='255.255.255.0'>
+            <dhcp>
+              <range start='${network.ipRange}.2' end='${network.ipRange}.254'/>
+            </dhcp>
+          </ip>
+        </network>
+      '';
+    in ''
+      uuid="$(${pkgs.libvirt}/bin/virsh net-uuid '${name}' || true)"
+      ${pkgs.libvirt}/bin/virsh net-define <(sed "s/UUID/$uuid/" '${xml}')
+      ${pkgs.libvirt}/bin/virsh net-start '${name}'
+    '';
+    preStop = ''
+      ${pkgs.libvirt}/bin/virsh net-destroy '${name}'
+    '';
+  }) networks);
+}
diff --git a/modules/private/system/dilion/vms/base_configuration.nix b/modules/private/system/dilion/vms/base_configuration.nix
new file mode 100644
index 0000000..e2caba2
--- /dev/null
+++ b/modules/private/system/dilion/vms/base_configuration.nix
@@ -0,0 +1,21 @@
+{ lib, config, ... }@args:
+{
+  options.myEnv = (import ../../../environment.nix (args // { name = "dummy"; })).options.myEnv;
+  config = {
+    fileSystems."/".device = "/dev/disk/by-label/nixos";
+    boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "virtio_balloon" "virtio_blk" "virtio_pci" "virtio_ring" ];
+    boot.loader = {
+      grub = {
+        version = 2;
+        device = "/dev/vda";
+      };
+      timeout = 0;
+    };
+    services.openssh.enable = true;
+    networking.firewall.allowedTCPPorts = [ 22 ];
+    users = {
+      mutableUsers = false;
+      users.root.openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.immae_dilion ];
+    };
+  };
+}
diff --git a/modules/private/system/dilion/vms/base_image.nix b/modules/private/system/dilion/vms/base_image.nix
new file mode 100644
index 0000000..8de8560
--- /dev/null
+++ b/modules/private/system/dilion/vms/base_image.nix
@@ -0,0 +1,94 @@
+configuration_file: { pkgs ? import <nixpkgs> {}, system ? builtins.currentSystem, myEnv, ... }:
+let
+  config = (import <nixpkgs/nixos/lib/eval-config.nix> {
+    inherit system;
+    modules = [ {
+      myEnv = myEnv;
+      imports = [ configuration_file ];
+
+      # We want our template image to be as small as possible, but the deployed image should be able to be
+      # of any size. Hence we resize on the first boot.
+      systemd.services.resize-main-fs = {
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig.Type = "oneshot";
+        script =
+          ''
+            # Resize main partition to fill whole disk
+            echo ", +" | ${pkgs.utillinux}/bin/sfdisk /dev/vda --no-reread -N 1
+            ${pkgs.parted}/bin/partprobe
+            # Resize filesystem
+            ${pkgs.e2fsprogs}/bin/resize2fs /dev/vda1
+          '';
+      };
+    } ];
+  }).config;
+in pkgs.vmTools.runInLinuxVM (
+  pkgs.runCommand "nixos-base-image"
+    {
+      memSize = 768;
+      preVM =
+        ''
+          mkdir $out
+          diskImage=image.qcow2
+          ${pkgs.vmTools.qemu}/bin/qemu-img create -f qcow2 $diskImage 2G
+          mv closure xchg/
+        '';
+      postVM =
+        ''
+          echo compressing VM image...
+          ${pkgs.vmTools.qemu}/bin/qemu-img convert -c $diskImage -O qcow2 $out/nixos.qcow2
+        '';
+      buildInputs = [ pkgs.utillinux pkgs.perl pkgs.parted pkgs.e2fsprogs ];
+      exportReferencesGraph =
+        [ "closure" config.system.build.toplevel ];
+    }
+    ''
+      # Create the partition
+      parted /dev/vda mklabel msdos
+      parted /dev/vda -- mkpart primary ext4 1M -1s
+
+      # Format the partition
+      mkfs.ext4 -L nixos /dev/vda1
+      mkdir /mnt
+      mount /dev/vda1 /mnt
+
+      for dir in dev proc sys; do
+        mkdir /mnt/$dir
+        mount --bind /$dir /mnt/$dir
+      done
+
+      storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure)
+      echo filling Nix store...
+      mkdir -p /mnt/nix/store
+      set -f
+      cp -prd $storePaths /mnt/nix/store
+      # The permissions will be set up incorrectly if the host machine is not running NixOS
+      chown -R 0:30000 /mnt/nix/store
+
+      mkdir -p /mnt/etc/nix
+      echo 'build-users-group = ' > /mnt/etc/nix/nix.conf
+
+      # Register the paths in the Nix database.
+      export USER=root
+      printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \
+          chroot /mnt ${config.nix.package.out}/bin/nix-store --load-db
+
+      # Create the system profile to allow nixos-rebuild to work.
+      chroot /mnt ${config.nix.package.out}/bin/nix-env \
+          -p /nix/var/nix/profiles/system --set ${config.system.build.toplevel}
+
+      # `nixos-rebuild' requires an /etc/NIXOS.
+      mkdir -p /mnt/etc/nixos
+      touch /mnt/etc/NIXOS
+
+      # `switch-to-configuration' requires a /bin/sh
+      mkdir -p /mnt/bin
+      ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh
+
+      # Generate the GRUB menu.
+      chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot
+
+      umount /mnt/{proc,dev,sys}
+      umount /mnt
+    ''
+)
diff --git a/modules/private/system/dilion/vms/buildbot_configuration.nix b/modules/private/system/dilion/vms/buildbot_configuration.nix
new file mode 100644
index 0000000..05b02d4
--- /dev/null
+++ b/modules/private/system/dilion/vms/buildbot_configuration.nix
@@ -0,0 +1,67 @@
+{ pkgs, config, lib, ... }:
+{
+  imports = [
+    <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
+    ./base_configuration.nix
+  ];
+  systemd.services.buildbot-worker.serviceConfig.ExecStartPre = let
+    cfg = config.services.buildbot-worker;
+    script = pkgs.writeScript "decode-dmi" ''
+      #!${pkgs.stdenv.shell}
+
+      mkdir -vp "${cfg.buildbotDir}"
+      varfile=${cfg.buildbotDir}/variables
+      rm $varfile || true
+      echo "[DEFAULT]" > $varfile
+      strings=$(${pkgs.dmidecode}/bin/dmidecode --oem-string count)
+      for i in $(seq 1 $strings); do
+        ${pkgs.dmidecode}/bin/dmidecode --oem-string $i >> $varfile
+      done
+      chown -R ${cfg.user}:${cfg.group} ${cfg.buildbotDir}
+    '';
+    in
+      lib.mkForce ["+${script}"];
+  systemd.services.buildbot-worker.serviceConfig.ExecStart = let
+      cfg = config.services.buildbot-worker;
+      tacFile = pkgs.writeText "buildbot-worker.tac" ''
+        import os
+        from io import open
+
+        from buildbot_worker.bot import Worker
+        from twisted.application import service
+
+        basedir = '${cfg.buildbotDir}'
+
+        # note: this line is matched against to check that this is a worker
+        # directory; do not edit it.
+        application = service.Application('buildbot-worker')
+
+        import configparser
+        config = config = configparser.ConfigParser()
+        config.read("${cfg.buildbotDir}/variables")
+        master_url_split = config["DEFAULT"]["buildbot_master_url"].split(':')
+        buildmaster_host = master_url_split[0]
+        port = int(master_url_split[1])
+        workername = config["DEFAULT"]["buildbot_worker_name"]
+
+        with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
+            passwd = passwd_file.read().strip('\r\n')
+        keepalive = ${toString cfg.keepalive}
+        umask = None
+        maxdelay = 300
+        numcpus = None
+        allow_shutdown = None
+
+        s = Worker(buildmaster_host, port, workername, passwd, basedir,
+                  keepalive, umask=umask, maxdelay=maxdelay,
+                  numcpus=numcpus, allow_shutdown=allow_shutdown)
+        s.setServiceParent(application)
+      '';
+    in
+      lib.mkForce "${cfg.package.pythonModule.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
+  services.buildbot-worker = {
+    enable = true;
+    workerPass = config.myEnv.buildbot.workerPassword;
+    packages = [ pkgs.git pkgs.gzip pkgs.openssh ];
+  };
+}