13 files changed, 686 insertions, 12 deletions

diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md
index 1697299..50eeca4 100644
--- a/DOCUMENTATION.md
+++ b/DOCUMENTATION.md
@@ -180,7 +180,7 @@ Things to look at during upgrades:
 Upgrade to latest unstable
 -------------------
 
-- Nothing in particular yet
+- Weechat: https://specs.weechat.org/specs/001285-follow-xdg-base-dir-spec.html
 
 Etherpad-lite
 -------------
diff --git a/environments/immae-eu.nix b/environments/immae-eu.nix
index ca9e5b5..df57e55 100644
--- a/environments/immae-eu.nix
+++ b/environments/immae-eu.nix
@@ -123,7 +123,7 @@ let
     sshfs ncdu procps-ng
 
     # other tools
-    pgloader s3cmd lftp jq cpulimit libxslt gandi-cli
+    pgloader s3cmd lftp jq cpulimit libxslt gandi-cli bubblewrap
 
     # Terraform + AWS
     terraform_0_12 awscli
diff --git a/modules/private/buildbot/common/libvirt.py b/modules/private/buildbot/common/libvirt.py
new file mode 100644
index 0000000..85fd908
--- /dev/null
+++ b/modules/private/buildbot/common/libvirt.py
@@ -0,0 +1,306 @@
+# This file was part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Portions Copyright Buildbot Team Members
+# Portions Copyright 2010 Isotoma Limited
+
+
+import os
+
+from twisted.internet import defer
+from twisted.internet import threads
+from twisted.internet import utils
+from twisted.python import failure
+from twisted.python import log
+
+from buildbot import config
+from buildbot.util.eventual import eventually
+from buildbot.worker import AbstractLatentWorker
+
+try:
+    import libvirt
+except ImportError:
+    libvirt = None
+
+import random
+import string
+
+def random_string_generator():
+    chars = string.ascii_letters
+    return ''.join(random.choice(chars) for x in range(6))
+
+class WorkQueue:
+
+    """
+    I am a class that turns parallel access into serial access.
+
+    I exist because we want to run libvirt access in threads as we don't
+    trust calls not to block, but under load libvirt doesn't seem to like
+    this kind of threaded use.
+    """
+
+    def __init__(self):
+        self.queue = []
+
+    def _process(self):
+        log.msg("Looking to start a piece of work now...")
+
+        # Is there anything to do?
+        if not self.queue:
+            log.msg("_process called when there is no work")
+            return
+
+        # Peek at the top of the stack - get a function to call and
+        # a deferred to fire when its all over
+        d, next_operation, args, kwargs = self.queue[0]
+
+        # Start doing some work - expects a deferred
+        try:
+            d2 = next_operation(*args, **kwargs)
+        except Exception:
+            d2 = defer.fail()
+
+        # Whenever a piece of work is done, whether it worked or not
+        # call this to schedule the next piece of work
+        @d2.addBoth
+        def _work_done(res):
+            log.msg("Completed a piece of work")
+            self.queue.pop(0)
+            if self.queue:
+                log.msg("Preparing next piece of work")
+                eventually(self._process)
+            return res
+
+        # When the work is done, trigger d
+        d2.chainDeferred(d)
+
+    def execute(self, cb, *args, **kwargs):
+        kickstart_processing = not self.queue
+        d = defer.Deferred()
+        self.queue.append((d, cb, args, kwargs))
+        if kickstart_processing:
+            self._process()
+        return d
+
+    def executeInThread(self, cb, *args, **kwargs):
+        return self.execute(threads.deferToThread, cb, *args, **kwargs)
+
+
+# A module is effectively a singleton class, so this is OK
+queue = WorkQueue()
+
+
+class Domain:
+
+    """
+    I am a wrapper around a libvirt Domain object
+    """
+
+    def __init__(self, connection, domain):
+        self.connection = connection
+        self.domain = domain
+
+    def name(self):
+        return queue.executeInThread(self.domain.name)
+
+    def create(self):
+        return queue.executeInThread(self.domain.create)
+
+    def shutdown(self):
+        return queue.executeInThread(self.domain.shutdown)
+
+    def destroy(self):
+        return queue.executeInThread(self.domain.destroy)
+
+class Volume:
+    def __init__(self, connection, volume):
+        self.connection = connection
+        self.volume = volume
+
+    @defer.inlineCallbacks
+    def destroy(self):
+        yield queue.executeInThread(self.volume.wipe)
+        yield queue.executeInThread(self.volume.delete)
+
+class Pool:
+    VolumeClass = Volume
+    def __init__(self, connection, pool):
+        self.connection = connection
+        self.pool = pool
+
+    @defer.inlineCallbacks
+    def create_volume(self, xml):
+        res = yield queue.executeInThread(self.pool.createXML, xml)
+        return self.VolumeClass(self.connection, res)
+
+class Connection:
+
+    """
+    I am a wrapper around a libvirt Connection object.
+    """
+
+    DomainClass = Domain
+    PoolClass = Pool
+
+    def __init__(self, uri):
+        self.uri = uri
+        self.connection = libvirt.open(uri)
+
+    @defer.inlineCallbacks
+    def create(self, xml):
+        """ I take libvirt XML and start a new VM """
+        res = yield queue.executeInThread(self.connection.createXML, xml, 0)
+        return self.DomainClass(self, res)
+
+    @defer.inlineCallbacks
+    def lookup_pool(self, name):
+        res = yield queue.executeInThread(self.connection.storagePoolLookupByName, name)
+        return self.PoolClass(self, res)
+
+class LibVirtWorker(AbstractLatentWorker):
+
+    def __init__(self, name, password, connection, master_url, base_image=None, **kwargs):
+        super().__init__(name, password, **kwargs)
+        if not libvirt:
+            config.error(
+                "The python module 'libvirt' is needed to use a LibVirtWorker")
+
+        self.master_url = master_url
+        self.random_name = random_string_generator()
+        self.connection = connection
+        self.base_image = base_image
+
+        self.domain = None
+        self.domain_name = "buildbot-" + self.workername + "-" + self.random_name
+        self.volume = None
+        self.volume_name = "buildbot-" + self.workername + "-" + self.random_name
+        self.pool_name = "buildbot-disks"
+
+    def reconfigService(self, *args, **kwargs):
+        if 'build_wait_timeout' not in kwargs:
+            kwargs['build_wait_timeout'] = 0
+        return super().reconfigService(*args, **kwargs)
+
+    def canStartBuild(self):
+        if self.domain and not self.isConnected():
+            log.msg(
+                "Not accepting builds as existing domain but worker not connected")
+            return False
+
+        return super().canStartBuild()
+
+    @defer.inlineCallbacks
+    def _prepare_image(self):
+        log.msg("Creating temporary image {}".format(self.volume_name))
+        pool = yield self.connection.lookup_pool(self.pool_name)
+        vol_xml = """
+            <volume type='file'>
+                <name>{vol_name}</name>
+                <capacity unit='G'>10</capacity>
+                <target>
+                    <format type='qcow2'/>
+                    <permissions>
+                        <mode>0600</mode>
+                        <owner>0</owner>
+                        <group>0</group>
+                    </permissions>
+                </target>
+                <backingStore>
+                    <path>/etc/libvirtd/base-images/buildbot.qcow2</path>
+                    <format type='qcow2'/>
+                </backingStore>
+            </volume>
+        """.format(vol_name = self.volume_name)
+        self.volume = yield pool.create_volume(vol_xml)
+
+    @defer.inlineCallbacks
+    def start_instance(self, build):
+        """
+        I start a new instance of a VM.
+
+        If a base_image is specified, I will make a clone of that otherwise i will
+        use image directly.
+
+        If i'm not given libvirt domain definition XML, I will look for my name
+        in the list of defined virtual machines and start that.
+        """
+        domain_xml = """
+            <domain type="kvm">
+                <name>{domain_name}</name>
+                <memory unit="GiB">2</memory>
+                <vcpu>1</vcpu>
+                <sysinfo type='smbios'>
+                    <oemStrings>
+                        <entry>buildbot_master_url={master_url}</entry>
+                        <entry>buildbot_worker_name={worker_name}</entry>
+                    </oemStrings>
+                </sysinfo>
+                <os>
+                    <type arch="x86_64">hvm</type>
+                    <smbios mode='sysinfo'/>
+                </os>
+                <devices>
+                    <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
+                    <disk type="volume" device="disk">
+                        <driver name='qemu' type='qcow2' />
+                        <source type="volume" pool="{pool_name}" volume="{volume_name}" />
+                        <backingStore type='volume'>
+                        <format type='qcow2'/>
+                        <source type="volume" pool="niximages" volume="buildbot.qcow2" />
+                        </backingStore>
+                        <target dev="vda" bus="virtio"/>
+                    </disk>
+                    <input type="keyboard" bus="usb"/>
+                    <graphics type="vnc" port="-1" autoport="yes"/>
+                    <interface type="network">
+                        <source network="immae" />
+                    </interface>
+                </devices>
+            </domain>
+        """.format(volume_name = self.volume_name, master_url = self.master_url, pool_name =
+                self.pool_name, domain_name = self.domain_name, worker_name = self.workername)
+
+        yield self._prepare_image()
+
+        try:
+            self.domain = yield self.connection.create(domain_xml)
+        except Exception:
+            log.err(failure.Failure(),
+                    ("Cannot start a VM ({}), failing gracefully and triggering"
+                     "a new build check").format(self.workername))
+            self.domain = None
+            return False
+
+        return [self.domain_name]
+
+    def stop_instance(self, fast=False):
+        """
+        I attempt to stop a running VM.
+        I make sure any connection to the worker is removed.
+        If the VM was using a cloned image, I remove the clone
+        When everything is tidied up, I ask that bbot looks for work to do
+        """
+
+        log.msg("Attempting to stop '{}'".format(self.workername))
+        if self.domain is None:
+            log.msg("I don't think that domain is even running, aborting")
+            return defer.succeed(None)
+
+        domain = self.domain
+        self.domain = None
+
+        d = domain.destroy()
+        if self.volume is not None:
+            self.volume.destroy()
+
+        return d
diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix
index d6753e5..ac34845 100644
--- a/modules/private/buildbot/default.nix
+++ b/modules/private/buildbot/default.nix
@@ -107,7 +107,12 @@ in
                 project_env = with lib.attrsets;
                   mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment //
                   mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) //
-                  { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; };
+                  {
+                    BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}";
+                    BUILDBOT_WORKER_PORT = builtins.toString project.workerPort;
+                    BUILDBOT_HOST = config.hostEnv.fqdn;
+                    BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system";
+                  };
                 in builtins.concatStringsSep "\n"
                   (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env);
             }
@@ -126,6 +131,13 @@ in
         permissions = "0600";
         user = "buildbot";
         group = "buildbot";
+        text = config.myEnv.buildbot.workerPassword;
+        dest = "buildbot/worker_password";
+      }
+      {
+        permissions = "0600";
+        user = "buildbot";
+        group = "buildbot";
         text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key";
         dest = "buildbot/ssh_key";
       }
@@ -135,6 +147,7 @@ in
       restart = true;
       paths = [
         "/var/secrets/buildbot/ldap"
+        "/var/secrets/buildbot/worker_password"
         "/var/secrets/buildbot/ssh_key"
         "/var/secrets/buildbot/${project.name}/environment_file"
       ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets;
@@ -144,6 +157,7 @@ in
       description = "buildbot slice";
     };
 
+    networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects;
     systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
       description = "Buildbot Continuous Integration Server ${project.name}.";
       after = [ "network-online.target" ];
@@ -196,6 +210,7 @@ in
       buildbot_secrets=${varDir}/${project.name}/secrets
       install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
       install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap
+      install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/worker_password $buildbot_secrets/worker_password
       ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
         (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets
       )}
@@ -213,6 +228,7 @@ in
         });
         HOME = "${varDir}/${project.name}";
         PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
+          pkgs.python3Packages.libvirt
           pkgs.python3Packages.wokkel
           pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
           pkgs.python3Packages.buildbot-worker
diff --git a/modules/private/buildbot/projects/test/__init__.py b/modules/private/buildbot/projects/test/__init__.py
index e6b8d51..e2f6f82 100644
--- a/modules/private/buildbot/projects/test/__init__.py
+++ b/modules/private/buildbot/projects/test/__init__.py
@@ -1,5 +1,6 @@
 from buildbot.plugins import *
 from buildbot_common.build_helpers import *
+import buildbot_common.libvirt as ilibvirt
 import os
 from buildbot.util import bytes2unicode
 import json
@@ -10,11 +11,13 @@ class E():
     PROJECT       = "test"
     BUILDBOT_URL  = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
     SOCKET        = "unix:/run/buildbot/{}.sock".format(PROJECT)
-    PB_SOCKET     = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT)
+    PB_SOCKET     = os.environ["BUILDBOT_WORKER_PORT"]
+    WORKER_HOST   = "{}:{}".format(os.environ["BUILDBOT_HOST"], PB_SOCKET)
     RELEASE_PATH  = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT)
     RELEASE_URL   = "https://release.immae.eu/{}".format(PROJECT)
     GIT_URL       = "https://git.immae.eu/perso/Immae/TestProject.git"
     SSH_KEY_PATH  = "/var/lib/buildbot/buildbot_key"
+    LIBVIRT_URL   = os.environ["BUILDBOT_VIRT_URL"] + "?keyfile=" + SSH_KEY_PATH
     PUPPET_HOST   = "root@backup-1.v.immae.eu"
     LDAP_HOST     = "ldap.immae.eu"
     LDAP_DN       = "cn=buildbot,ou=services,dc=immae,dc=eu"
@@ -70,8 +73,14 @@ def configure(c):
     c["www"]["change_hook_dialects"]["base"] = {
             "custom_class": CustomBase
             }
-    c['workers'].append(worker.LocalWorker("generic-worker-test"))
-    c['workers'].append(worker.LocalWorker("deploy-worker-test"))
+    c['workers'].append(ilibvirt.LibVirtWorker("test-build",
+                                             open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
+                                             ilibvirt.Connection(E.LIBVIRT_URL),
+                                             E.WORKER_HOST))
+    c['workers'].append(ilibvirt.LibVirtWorker("test-deploy",
+                                             open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
+                                             ilibvirt.Connection(E.LIBVIRT_URL),
+                                             E.WORKER_HOST))
 
     c['schedulers'].append(hook_scheduler("TestProject", timer=1))
     c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"]))
@@ -109,7 +118,7 @@ def factory():
         logEnviron=False, command=["echo", package]))
     factory.addSteps(package_and_upload(package, package_dest, package_url))
 
-    return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory)
+    return util.BuilderConfig(name="TestProject_build", workernames=["test-build"], factory=factory)
 
 
 def compute_build_infos():
@@ -143,7 +152,7 @@ def deploy_factory():
         ldap_password=util.Secret("ldap")))
     factory.addStep(steps.MasterShellCommand(command=[
         "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host]))
-    return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory)
+    return util.BuilderConfig(name="TestProject_deploy", workernames=["test-deploy"], factory=factory)
 
 from twisted.internet import defer
 from buildbot.process.buildstep import FAILURE
diff --git a/modules/private/environment.nix b/modules/private/environment.nix
index 719bf8f..f0af572 100644
--- a/modules/private/environment.nix
+++ b/modules/private/environment.nix
@@ -228,6 +228,7 @@ in
       '';
       type = submodule {
         options = {
+          rootKeys = mkOption { type = attrsOf str; description = "Keys of root users"; };
           ldap = mkOption {
             description = ''
               LDAP credentials for cn=ssh,ou=services,dc=immae,dc=eu dn
@@ -804,6 +805,7 @@ in
       description = "Buildbot configuration";
       type = submodule {
         options = {
+          workerPassword = mkOption { description = "Buildbot worker password"; type = str; };
           user = mkOption {
             description = "Buildbot user";
             type = submodule {
@@ -855,6 +857,7 @@ in
                     '';
                 };
                 pythonPathHome = mkOption { type = bool; description = "Whether to add project’s python home to python path"; };
+                workerPort = mkOption { type = port; description = "Port for the worker"; };
                 secrets = mkOption {
                   type = attrsOf str;
                   description = "Secrets for the project to dump as files";
diff --git a/modules/private/gitolite/default.nix b/modules/private/gitolite/default.nix
index 6b573e3..e54ee8a 100644
--- a/modules/private/gitolite/default.nix
+++ b/modules/private/gitolite/default.nix
@@ -74,7 +74,7 @@ in {
     # Installation: https://git.immae.eu/mantisbt/view.php?id=93
     services.gitolite = {
       enable = true;
-      adminPubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXqRbiHw7QoHADNIEuo4nUT9fSOIEBMdJZH0bkQAxXyJFyCM1IMz0pxsHV0wu9tdkkr36bPEUj2aV5bkYLBN6nxcV2Y49X8bjOSCPfx3n6Own1h+NeZVBj4ZByrFmqCbTxUJIZ2bZKcWOFncML39VmWdsVhNjg0X4NBBehqXRIKr2gt3E/ESAxTYJFm0BnU0baciw9cN0bsRGqvFgf5h2P48CIAfwhVcGmPQnnAwabnosYQzRWxR0OygH5Kd8mePh6FheIRIigfXsDO8f/jdxwut8buvNIf3m5EBr3tUbTsvM+eV3M5vKGt7sk8T64DVtepTSdOOWtp+47ktsnHOMh immae@immae.eu";
+      adminPubkey = config.myEnv.sshd.rootKeys.immae_dilion;
     };
   };
 }
diff --git a/modules/private/system/dilion.nix b/modules/private/system/dilion.nix
index be8269e..a59d607 100644
--- a/modules/private/system/dilion.nix
+++ b/modules/private/system/dilion.nix
@@ -76,12 +76,24 @@
   };
 
   myServices.ssh.modules = [ config.myServices.ssh.predefinedModules.regular ];
-  imports = builtins.attrValues (import ../..);
+  imports = builtins.attrValues (import ../..) ++ [ ./dilion/vms.nix ];
 
   system.nssModules = [ pkgs.libvirt ];
   system.nssDatabases.hosts = lib.mkForce [ "files" "libvirt_guest" "mymachines" "dns" "myhostname" ];
   programs.zsh.enable = true;
 
+  users.users.libvirt = {
+    hashedPassword = "!";
+    shell = pkgs.bashInteractive;
+    isSystemUser = true;
+    group = "libvirtd";
+    packages = [ pkgs.netcat-openbsd ];
+    openssh.authorizedKeys.keyFiles = [
+      "${privateFiles}/buildbot_ssh_key.pub"
+    ];
+    openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.ismael_flony ];
+  };
+
   users.users.backup = {
     hashedPassword = "!";
     isSystemUser = true;
@@ -118,7 +130,7 @@
     after = [ "network.target" ];
 
     serviceConfig = {
-      ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:22";
+      ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:caldance:22";
     };
   };
 
@@ -170,7 +182,7 @@
     recommendedGzipSettings = true;
     recommendedProxySettings = true;
     upstreams = {
-      caldance.servers."nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:3031" = {};
+      caldance.servers."caldance:3031" = {};
     };
     virtualHosts = {
       "dev.immae.eu" = {
diff --git a/modules/private/system/dilion/vms.nix b/modules/private/system/dilion/vms.nix
new file mode 100644
index 0000000..8d5a57b
--- /dev/null
+++ b/modules/private/system/dilion/vms.nix
@@ -0,0 +1,146 @@
+# inspired from https://nixos.wiki/wiki/Virtualization_in_NixOS
+{ config, pkgs, lib, ... }@args:
+let
+  networks = {
+    immae = {
+      bridgeNumber = "1";
+      ipRange = "192.168.100";
+    };
+  };
+  guests = {
+    caldance = {
+      pool = "zfspool";
+      cpus = "1";
+      memory = "2";
+      network = "immae";
+      diskSize = "10GiB";
+      extraDevicesXML = ''
+        <filesystem type="mount">
+          <source dir="/var/lib/caldance"/>
+          <target dir="home"/>
+        </filesystem>
+      '';
+    };
+    buildbot = {
+      pool = "zfspool";
+      cpus = "1";
+      memory = "3";
+      network = "immae";
+      diskSize = "10GiB";
+      destroyVolumeOnExit = true;
+      preStart = ''
+        if ! ${pkgs.libvirt}/bin/virsh pool-info --pool niximages &> /dev/null; then
+          pool-create-as --name niximages --type dir --target /etc/libvirtd/base-images/
+        fi
+        if ! ${pkgs.libvirt}/bin/virsh pool-info --pool buildbot-disks &> /dev/null; then
+          mkdir -p /var/lib/libvirt/images/buildbot-disks
+          pool-create-as --name buildbot-disks --type dir --target /var/lib/libvirt/images/buildbot-disks
+        fi
+      '';
+    };
+  };
+  toImage = f: "${import ./vms/base_image.nix f (args // { myEnv = config.myEnv; })}/nixos.qcow2";
+in
+{
+  environment.etc."libvirtd/base-images/nixos.qcow2".source = toImage ./vms/base_configuration.nix;
+  environment.etc."libvirtd/base-images/buildbot.qcow2".source = toImage ./vms/buildbot_configuration.nix;
+  systemd.services = lib.mapAttrs' (name: guest: lib.nameValuePair "libvirtd-guest-${name}" {
+    after = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
+    requires = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = "yes";
+    };
+    script =
+      let
+        xml = pkgs.writeText "libvirt-guest-${name}.xml"
+        ''
+          <domain type="kvm">
+            <name>${name}</name>
+            <uuid>UUID</uuid>
+            <memory unit="GiB">${guest.memory}</memory>
+            <vcpu>${guest.cpus}</vcpu>
+            <os>
+              <type arch="x86_64">hvm</type>
+            </os>
+            <devices>
+              <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
+              <disk type="volume">
+                <source pool="${guest.pool}" volume="guest-${name}" />
+                <target dev="vda" bus="virtio"/>
+              </disk>
+              ${guest.extraDevicesXML or ""}
+              <input type="keyboard" bus="usb"/>
+              <graphics type="vnc" port="-1" autoport="yes"/>
+              <interface type="network">
+                <source network="${guest.network}" />
+              </interface>
+            </devices>
+            <features>
+              <acpi/>
+            </features>
+          </domain>
+        '';
+      in
+      guest.preStart or "" + ''
+        if ! ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
+          ${pkgs.libvirt}/bin/virsh vol-create-as --pool ${guest.pool} --name 'guest-${name}' --capacity '${guest.diskSize}'
+          volume_path=$(${pkgs.libvirt}/bin/virsh vol-path --pool ${guest.pool} --vol 'guest-${name}')
+          ${pkgs.qemu}/bin/qemu-img convert /etc/libvirtd/base-images/nixos.qcow2 $volume_path
+        fi
+        uuid="$(${pkgs.libvirt}/bin/virsh domuuid '${name}' || true)"
+        ${pkgs.libvirt}/bin/virsh define <(sed "s/UUID/$uuid/" '${xml}')
+        ${pkgs.libvirt}/bin/virsh start '${name}'
+      '';
+    preStop = ''
+      ${pkgs.libvirt}/bin/virsh shutdown '${name}'
+      let "timeout = $(date +%s) + 10"
+      while [ "$(${pkgs.libvirt}/bin/virsh list --name | grep --count '^${name}$')" -gt 0 ]; do
+        if [ "$(date +%s)" -ge "$timeout" ]; then
+          # Meh, we warned it...
+          ${pkgs.libvirt}/bin/virsh destroy '${name}'
+        else
+          # The machine is still running, let's give it some time to shut down
+          sleep 0.5
+        fi
+      done
+    '' + lib.optionalString (guest.destroyVolumeOnExit or false) ''
+        if ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
+          ${pkgs.libvirt}/bin/virsh vol-wipe --pool ${guest.pool} --vol 'guest-${name}' || true
+          ${pkgs.libvirt}/bin/virsh vol-delete --pool ${guest.pool} --vol 'guest-${name}'
+        fi
+    '';
+  }) guests // (lib.mapAttrs' (name: network: lib.nameValuePair "libvirtd-network-${name}" {
+    after = [ "libvirtd.service" ];
+    requires = [ "libvirtd.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = "yes";
+    };
+    script = let
+      xml = pkgs.writeText "libvirt-network-${name}.xml" ''
+        <network>
+          <name>${name}</name>
+          <uuid>UUID</uuid>
+          <forward mode='nat' />
+          <bridge name='virbr${network.bridgeNumber}' />
+          <domain name='${name}' localOnly='yes'/>
+          <ip address='${network.ipRange}.1' netmask='255.255.255.0'>
+            <dhcp>
+              <range start='${network.ipRange}.2' end='${network.ipRange}.254'/>
+            </dhcp>
+          </ip>
+        </network>
+      '';
+    in ''
+      uuid="$(${pkgs.libvirt}/bin/virsh net-uuid '${name}' || true)"
+      ${pkgs.libvirt}/bin/virsh net-define <(sed "s/UUID/$uuid/" '${xml}')
+      ${pkgs.libvirt}/bin/virsh net-start '${name}'
+    '';
+    preStop = ''
+      ${pkgs.libvirt}/bin/virsh net-destroy '${name}'
+    '';
+  }) networks);
+}
diff --git a/modules/private/system/dilion/vms/base_configuration.nix b/modules/private/system/dilion/vms/base_configuration.nix
new file mode 100644
index 0000000..e2caba2
--- /dev/null
+++ b/modules/private/system/dilion/vms/base_configuration.nix
@@ -0,0 +1,21 @@
+{ lib, config, ... }@args:
+{
+  options.myEnv = (import ../../../environment.nix (args // { name = "dummy"; })).options.myEnv;
+  config = {
+    fileSystems."/".device = "/dev/disk/by-label/nixos";
+    boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "virtio_balloon" "virtio_blk" "virtio_pci" "virtio_ring" ];
+    boot.loader = {
+      grub = {
+        version = 2;
+        device = "/dev/vda";
+      };
+      timeout = 0;
+    };
+    services.openssh.enable = true;
+    networking.firewall.allowedTCPPorts = [ 22 ];
+    users = {
+      mutableUsers = false;
+      users.root.openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.immae_dilion ];
+    };
+  };
+}
diff --git a/modules/private/system/dilion/vms/base_image.nix b/modules/private/system/dilion/vms/base_image.nix
new file mode 100644
index 0000000..8de8560
--- /dev/null
+++ b/modules/private/system/dilion/vms/base_image.nix
@@ -0,0 +1,94 @@
+configuration_file: { pkgs ? import <nixpkgs> {}, system ? builtins.currentSystem, myEnv, ... }:
+let
+  config = (import <nixpkgs/nixos/lib/eval-config.nix> {
+    inherit system;
+    modules = [ {
+      myEnv = myEnv;
+      imports = [ configuration_file ];
+
+      # We want our template image to be as small as possible, but the deployed image should be able to be
+      # of any size. Hence we resize on the first boot.
+      systemd.services.resize-main-fs = {
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig.Type = "oneshot";
+        script =
+          ''
+            # Resize main partition to fill whole disk
+            echo ", +" | ${pkgs.utillinux}/bin/sfdisk /dev/vda --no-reread -N 1
+            ${pkgs.parted}/bin/partprobe
+            # Resize filesystem
+            ${pkgs.e2fsprogs}/bin/resize2fs /dev/vda1
+          '';
+      };
+    } ];
+  }).config;
+in pkgs.vmTools.runInLinuxVM (
+  pkgs.runCommand "nixos-base-image"
+    {
+      memSize = 768;
+      preVM =
+        ''
+          mkdir $out
+          diskImage=image.qcow2
+          ${pkgs.vmTools.qemu}/bin/qemu-img create -f qcow2 $diskImage 2G
+          mv closure xchg/
+        '';
+      postVM =
+        ''
+          echo compressing VM image...
+          ${pkgs.vmTools.qemu}/bin/qemu-img convert -c $diskImage -O qcow2 $out/nixos.qcow2
+        '';
+      buildInputs = [ pkgs.utillinux pkgs.perl pkgs.parted pkgs.e2fsprogs ];
+      exportReferencesGraph =
+        [ "closure" config.system.build.toplevel ];
+    }
+    ''
+      # Create the partition
+      parted /dev/vda mklabel msdos
+      parted /dev/vda -- mkpart primary ext4 1M -1s
+
+      # Format the partition
+      mkfs.ext4 -L nixos /dev/vda1
+      mkdir /mnt
+      mount /dev/vda1 /mnt
+
+      for dir in dev proc sys; do
+        mkdir /mnt/$dir
+        mount --bind /$dir /mnt/$dir
+      done
+
+      storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure)
+      echo filling Nix store...
+      mkdir -p /mnt/nix/store
+      set -f
+      cp -prd $storePaths /mnt/nix/store
+      # The permissions will be set up incorrectly if the host machine is not running NixOS
+      chown -R 0:30000 /mnt/nix/store
+
+      mkdir -p /mnt/etc/nix
+      echo 'build-users-group = ' > /mnt/etc/nix/nix.conf
+
+      # Register the paths in the Nix database.
+      export USER=root
+      printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \
+          chroot /mnt ${config.nix.package.out}/bin/nix-store --load-db
+
+      # Create the system profile to allow nixos-rebuild to work.
+      chroot /mnt ${config.nix.package.out}/bin/nix-env \
+          -p /nix/var/nix/profiles/system --set ${config.system.build.toplevel}
+
+      # `nixos-rebuild' requires an /etc/NIXOS.
+      mkdir -p /mnt/etc/nixos
+      touch /mnt/etc/NIXOS
+
+      # `switch-to-configuration' requires a /bin/sh
+      mkdir -p /mnt/bin
+      ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh
+
+      # Generate the GRUB menu.
+      chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot
+
+      umount /mnt/{proc,dev,sys}
+      umount /mnt
+    ''
+)
diff --git a/modules/private/system/dilion/vms/buildbot_configuration.nix b/modules/private/system/dilion/vms/buildbot_configuration.nix
new file mode 100644
index 0000000..05b02d4
--- /dev/null
+++ b/modules/private/system/dilion/vms/buildbot_configuration.nix
@@ -0,0 +1,67 @@
+{ pkgs, config, lib, ... }:
+{
+  imports = [
+    <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
+    ./base_configuration.nix
+  ];
+  systemd.services.buildbot-worker.serviceConfig.ExecStartPre = let
+    cfg = config.services.buildbot-worker;
+    script = pkgs.writeScript "decode-dmi" ''
+      #!${pkgs.stdenv.shell}
+
+      mkdir -vp "${cfg.buildbotDir}"
+      varfile=${cfg.buildbotDir}/variables
+      rm $varfile || true
+      echo "[DEFAULT]" > $varfile
+      strings=$(${pkgs.dmidecode}/bin/dmidecode --oem-string count)
+      for i in $(seq 1 $strings); do
+        ${pkgs.dmidecode}/bin/dmidecode --oem-string $i >> $varfile
+      done
+      chown -R ${cfg.user}:${cfg.group} ${cfg.buildbotDir}
+    '';
+    in
+      lib.mkForce ["+${script}"];
+  systemd.services.buildbot-worker.serviceConfig.ExecStart = let
+      cfg = config.services.buildbot-worker;
+      tacFile = pkgs.writeText "buildbot-worker.tac" ''
+        import os
+        from io import open
+
+        from buildbot_worker.bot import Worker
+        from twisted.application import service
+
+        basedir = '${cfg.buildbotDir}'
+
+        # note: this line is matched against to check that this is a worker
+        # directory; do not edit it.
+        application = service.Application('buildbot-worker')
+
+        import configparser
+        config = config = configparser.ConfigParser()
+        config.read("${cfg.buildbotDir}/variables")
+        master_url_split = config["DEFAULT"]["buildbot_master_url"].split(':')
+        buildmaster_host = master_url_split[0]
+        port = int(master_url_split[1])
+        workername = config["DEFAULT"]["buildbot_worker_name"]
+
+        with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
+            passwd = passwd_file.read().strip('\r\n')
+        keepalive = ${toString cfg.keepalive}
+        umask = None
+        maxdelay = 300
+        numcpus = None
+        allow_shutdown = None
+
+        s = Worker(buildmaster_host, port, workername, passwd, basedir,
+                  keepalive, umask=umask, maxdelay=maxdelay,
+                  numcpus=numcpus, allow_shutdown=allow_shutdown)
+        s.setServiceParent(application)
+      '';
+    in
+      lib.mkForce "${cfg.package.pythonModule.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
+  services.buildbot-worker = {
+    enable = true;
+    workerPass = config.myEnv.buildbot.workerPassword;
+    packages = [ pkgs.git pkgs.gzip pkgs.openssh ];
+  };
+}
diff --git a/nixops/secrets b/nixops/secrets
-Subproject 9ac6c1459d2eeb24be0a991f745b567f0fcb0cc
+Subproject 932623303fd674faf8106e03ea0b89195edfdc0