First attempt at making declarative VMs

In order to make buildbot more secure, the builds need to happen inside
VMs so that they can be thrown out on demand when not needed.

This commit implements this facility on dilion, and also defines
declaratively some previous VMs which used to run on the machine.

3 files changed, 337 insertions, 6 deletions

diff --git a/modules/private/buildbot/common/libvirt.py b/modules/private/buildbot/common/libvirt.py
new file mode 100644
index 0000000..85fd908
--- /dev/null
+++ b/modules/private/buildbot/common/libvirt.py
@@ -0,0 +1,306 @@
+# This file was part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Portions Copyright Buildbot Team Members
+# Portions Copyright 2010 Isotoma Limited
+
+
+import os
+
+from twisted.internet import defer
+from twisted.internet import threads
+from twisted.internet import utils
+from twisted.python import failure
+from twisted.python import log
+
+from buildbot import config
+from buildbot.util.eventual import eventually
+from buildbot.worker import AbstractLatentWorker
+
+try:
+    import libvirt
+except ImportError:
+    libvirt = None
+
+import random
+import string
+
+def random_string_generator():
+    chars = string.ascii_letters
+    return ''.join(random.choice(chars) for x in range(6))
+
+class WorkQueue:
+
+    """
+    I am a class that turns parallel access into serial access.
+
+    I exist because we want to run libvirt access in threads as we don't
+    trust calls not to block, but under load libvirt doesn't seem to like
+    this kind of threaded use.
+    """
+
+    def __init__(self):
+        self.queue = []
+
+    def _process(self):
+        log.msg("Looking to start a piece of work now...")
+
+        # Is there anything to do?
+        if not self.queue:
+            log.msg("_process called when there is no work")
+            return
+
+        # Peek at the top of the stack - get a function to call and
+        # a deferred to fire when its all over
+        d, next_operation, args, kwargs = self.queue[0]
+
+        # Start doing some work - expects a deferred
+        try:
+            d2 = next_operation(*args, **kwargs)
+        except Exception:
+            d2 = defer.fail()
+
+        # Whenever a piece of work is done, whether it worked or not
+        # call this to schedule the next piece of work
+        @d2.addBoth
+        def _work_done(res):
+            log.msg("Completed a piece of work")
+            self.queue.pop(0)
+            if self.queue:
+                log.msg("Preparing next piece of work")
+                eventually(self._process)
+            return res
+
+        # When the work is done, trigger d
+        d2.chainDeferred(d)
+
+    def execute(self, cb, *args, **kwargs):
+        kickstart_processing = not self.queue
+        d = defer.Deferred()
+        self.queue.append((d, cb, args, kwargs))
+        if kickstart_processing:
+            self._process()
+        return d
+
+    def executeInThread(self, cb, *args, **kwargs):
+        return self.execute(threads.deferToThread, cb, *args, **kwargs)
+
+
+# A module is effectively a singleton class, so this is OK
+queue = WorkQueue()
+
+
+class Domain:
+
+    """
+    I am a wrapper around a libvirt Domain object
+    """
+
+    def __init__(self, connection, domain):
+        self.connection = connection
+        self.domain = domain
+
+    def name(self):
+        return queue.executeInThread(self.domain.name)
+
+    def create(self):
+        return queue.executeInThread(self.domain.create)
+
+    def shutdown(self):
+        return queue.executeInThread(self.domain.shutdown)
+
+    def destroy(self):
+        return queue.executeInThread(self.domain.destroy)
+
+class Volume:
+    def __init__(self, connection, volume):
+        self.connection = connection
+        self.volume = volume
+
+    @defer.inlineCallbacks
+    def destroy(self):
+        yield queue.executeInThread(self.volume.wipe)
+        yield queue.executeInThread(self.volume.delete)
+
+class Pool:
+    VolumeClass = Volume
+    def __init__(self, connection, pool):
+        self.connection = connection
+        self.pool = pool
+
+    @defer.inlineCallbacks
+    def create_volume(self, xml):
+        res = yield queue.executeInThread(self.pool.createXML, xml)
+        return self.VolumeClass(self.connection, res)
+
+class Connection:
+
+    """
+    I am a wrapper around a libvirt Connection object.
+    """
+
+    DomainClass = Domain
+    PoolClass = Pool
+
+    def __init__(self, uri):
+        self.uri = uri
+        self.connection = libvirt.open(uri)
+
+    @defer.inlineCallbacks
+    def create(self, xml):
+        """ I take libvirt XML and start a new VM """
+        res = yield queue.executeInThread(self.connection.createXML, xml, 0)
+        return self.DomainClass(self, res)
+
+    @defer.inlineCallbacks
+    def lookup_pool(self, name):
+        res = yield queue.executeInThread(self.connection.storagePoolLookupByName, name)
+        return self.PoolClass(self, res)
+
+class LibVirtWorker(AbstractLatentWorker):
+
+    def __init__(self, name, password, connection, master_url, base_image=None, **kwargs):
+        super().__init__(name, password, **kwargs)
+        if not libvirt:
+            config.error(
+                "The python module 'libvirt' is needed to use a LibVirtWorker")
+
+        self.master_url = master_url
+        self.random_name = random_string_generator()
+        self.connection = connection
+        self.base_image = base_image
+
+        self.domain = None
+        self.domain_name = "buildbot-" + self.workername + "-" + self.random_name
+        self.volume = None
+        self.volume_name = "buildbot-" + self.workername + "-" + self.random_name
+        self.pool_name = "buildbot-disks"
+
+    def reconfigService(self, *args, **kwargs):
+        if 'build_wait_timeout' not in kwargs:
+            kwargs['build_wait_timeout'] = 0
+        return super().reconfigService(*args, **kwargs)
+
+    def canStartBuild(self):
+        if self.domain and not self.isConnected():
+            log.msg(
+                "Not accepting builds as existing domain but worker not connected")
+            return False
+
+        return super().canStartBuild()
+
+    @defer.inlineCallbacks
+    def _prepare_image(self):
+        log.msg("Creating temporary image {}".format(self.volume_name))
+        pool = yield self.connection.lookup_pool(self.pool_name)
+        vol_xml = """
+            <volume type='file'>
+                <name>{vol_name}</name>
+                <capacity unit='G'>10</capacity>
+                <target>
+                    <format type='qcow2'/>
+                    <permissions>
+                        <mode>0600</mode>
+                        <owner>0</owner>
+                        <group>0</group>
+                    </permissions>
+                </target>
+                <backingStore>
+                    <path>/etc/libvirtd/base-images/buildbot.qcow2</path>
+                    <format type='qcow2'/>
+                </backingStore>
+            </volume>
+        """.format(vol_name = self.volume_name)
+        self.volume = yield pool.create_volume(vol_xml)
+
+    @defer.inlineCallbacks
+    def start_instance(self, build):
+        """
+        I start a new instance of a VM.
+
+        If a base_image is specified, I will make a clone of that otherwise i will
+        use image directly.
+
+        If i'm not given libvirt domain definition XML, I will look for my name
+        in the list of defined virtual machines and start that.
+        """
+        domain_xml = """
+            <domain type="kvm">
+                <name>{domain_name}</name>
+                <memory unit="GiB">2</memory>
+                <vcpu>1</vcpu>
+                <sysinfo type='smbios'>
+                    <oemStrings>
+                        <entry>buildbot_master_url={master_url}</entry>
+                        <entry>buildbot_worker_name={worker_name}</entry>
+                    </oemStrings>
+                </sysinfo>
+                <os>
+                    <type arch="x86_64">hvm</type>
+                    <smbios mode='sysinfo'/>
+                </os>
+                <devices>
+                    <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
+                    <disk type="volume" device="disk">
+                        <driver name='qemu' type='qcow2' />
+                        <source type="volume" pool="{pool_name}" volume="{volume_name}" />
+                        <backingStore type='volume'>
+                        <format type='qcow2'/>
+                        <source type="volume" pool="niximages" volume="buildbot.qcow2" />
+                        </backingStore>
+                        <target dev="vda" bus="virtio"/>
+                    </disk>
+                    <input type="keyboard" bus="usb"/>
+                    <graphics type="vnc" port="-1" autoport="yes"/>
+                    <interface type="network">
+                        <source network="immae" />
+                    </interface>
+                </devices>
+            </domain>
+        """.format(volume_name = self.volume_name, master_url = self.master_url, pool_name =
+                self.pool_name, domain_name = self.domain_name, worker_name = self.workername)
+
+        yield self._prepare_image()
+
+        try:
+            self.domain = yield self.connection.create(domain_xml)
+        except Exception:
+            log.err(failure.Failure(),
+                    ("Cannot start a VM ({}), failing gracefully and triggering"
+                     "a new build check").format(self.workername))
+            self.domain = None
+            return False
+
+        return [self.domain_name]
+
+    def stop_instance(self, fast=False):
+        """
+        I attempt to stop a running VM.
+        I make sure any connection to the worker is removed.
+        If the VM was using a cloned image, I remove the clone
+        When everything is tidied up, I ask that bbot looks for work to do
+        """
+
+        log.msg("Attempting to stop '{}'".format(self.workername))
+        if self.domain is None:
+            log.msg("I don't think that domain is even running, aborting")
+            return defer.succeed(None)
+
+        domain = self.domain
+        self.domain = None
+
+        d = domain.destroy()
+        if self.volume is not None:
+            self.volume.destroy()
+
+        return d
diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix
index d6753e5..ac34845 100644
--- a/modules/private/buildbot/default.nix
+++ b/modules/private/buildbot/default.nix
@@ -107,7 +107,12 @@ in
                 project_env = with lib.attrsets;
                   mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment //
                   mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) //
-                  { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; };
+                  {
+                    BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}";
+                    BUILDBOT_WORKER_PORT = builtins.toString project.workerPort;
+                    BUILDBOT_HOST = config.hostEnv.fqdn;
+                    BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system";
+                  };
                 in builtins.concatStringsSep "\n"
                   (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env);
             }
@@ -126,6 +131,13 @@ in
         permissions = "0600";
         user = "buildbot";
         group = "buildbot";
+        text = config.myEnv.buildbot.workerPassword;
+        dest = "buildbot/worker_password";
+      }
+      {
+        permissions = "0600";
+        user = "buildbot";
+        group = "buildbot";
         text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key";
         dest = "buildbot/ssh_key";
       }
@@ -135,6 +147,7 @@ in
       restart = true;
       paths = [
         "/var/secrets/buildbot/ldap"
+        "/var/secrets/buildbot/worker_password"
         "/var/secrets/buildbot/ssh_key"
         "/var/secrets/buildbot/${project.name}/environment_file"
       ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets;
@@ -144,6 +157,7 @@ in
       description = "buildbot slice";
     };
 
+    networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects;
     systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
       description = "Buildbot Continuous Integration Server ${project.name}.";
       after = [ "network-online.target" ];
@@ -196,6 +210,7 @@ in
       buildbot_secrets=${varDir}/${project.name}/secrets
       install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
       install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap
+      install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/worker_password $buildbot_secrets/worker_password
       ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
         (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets
       )}
@@ -213,6 +228,7 @@ in
         });
         HOME = "${varDir}/${project.name}";
         PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
+          pkgs.python3Packages.libvirt
           pkgs.python3Packages.wokkel
           pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
           pkgs.python3Packages.buildbot-worker
diff --git a/modules/private/buildbot/projects/test/__init__.py b/modules/private/buildbot/projects/test/__init__.py
index e6b8d51..e2f6f82 100644
--- a/modules/private/buildbot/projects/test/__init__.py
+++ b/modules/private/buildbot/projects/test/__init__.py
@@ -1,5 +1,6 @@
 from buildbot.plugins import *
 from buildbot_common.build_helpers import *
+import buildbot_common.libvirt as ilibvirt
 import os
 from buildbot.util import bytes2unicode
 import json
@@ -10,11 +11,13 @@ class E():
     PROJECT       = "test"
     BUILDBOT_URL  = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
     SOCKET        = "unix:/run/buildbot/{}.sock".format(PROJECT)
-    PB_SOCKET     = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT)
+    PB_SOCKET     = os.environ["BUILDBOT_WORKER_PORT"]
+    WORKER_HOST   = "{}:{}".format(os.environ["BUILDBOT_HOST"], PB_SOCKET)
     RELEASE_PATH  = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT)
     RELEASE_URL   = "https://release.immae.eu/{}".format(PROJECT)
     GIT_URL       = "https://git.immae.eu/perso/Immae/TestProject.git"
     SSH_KEY_PATH  = "/var/lib/buildbot/buildbot_key"
+    LIBVIRT_URL   = os.environ["BUILDBOT_VIRT_URL"] + "?keyfile=" + SSH_KEY_PATH
     PUPPET_HOST   = "root@backup-1.v.immae.eu"
     LDAP_HOST     = "ldap.immae.eu"
     LDAP_DN       = "cn=buildbot,ou=services,dc=immae,dc=eu"
@@ -70,8 +73,14 @@ def configure(c):
     c["www"]["change_hook_dialects"]["base"] = {
             "custom_class": CustomBase
             }
-    c['workers'].append(worker.LocalWorker("generic-worker-test"))
-    c['workers'].append(worker.LocalWorker("deploy-worker-test"))
+    c['workers'].append(ilibvirt.LibVirtWorker("test-build",
+                                             open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
+                                             ilibvirt.Connection(E.LIBVIRT_URL),
+                                             E.WORKER_HOST))
+    c['workers'].append(ilibvirt.LibVirtWorker("test-deploy",
+                                             open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
+                                             ilibvirt.Connection(E.LIBVIRT_URL),
+                                             E.WORKER_HOST))
 
     c['schedulers'].append(hook_scheduler("TestProject", timer=1))
     c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"]))
@@ -109,7 +118,7 @@ def factory():
         logEnviron=False, command=["echo", package]))
     factory.addSteps(package_and_upload(package, package_dest, package_url))
 
-    return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory)
+    return util.BuilderConfig(name="TestProject_build", workernames=["test-build"], factory=factory)
 
 
 def compute_build_infos():
@@ -143,7 +152,7 @@ def deploy_factory():
         ldap_password=util.Secret("ldap")))
     factory.addStep(steps.MasterShellCommand(command=[
         "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host]))
-    return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory)
+    return util.BuilderConfig(name="TestProject_deploy", workernames=["test-deploy"], factory=factory)
 
 from twisted.internet import defer
 from buildbot.process.buildstep import FAILURE