3 files changed, 0 insertions, 664 deletions

diff --git a/modules/private/buildbot/common/build_helpers.py b/modules/private/buildbot/common/build_helpers.py
deleted file mode 100644
index acea905..0000000
--- a/modules/private/buildbot/common/build_helpers.py
+++ /dev/null
@@ -1,277 +0,0 @@
-from buildbot.plugins import util, steps, schedulers
-from buildbot_buildslist import BuildsList
-from shutil import which
-
-__all__ = [
-        "force_scheduler", "deploy_scheduler", "hook_scheduler",
-        "clean_branch", "package_and_upload", "SlackStatusPush",
-        "XMPPStatusPush", "NixShellCommand"
-        ]
-
-# Small helpers"
-@util.renderer
-def clean_branch(props):
-    if props.hasProperty("branch") and len(props["branch"]) > 0:
-        return props["branch"].replace("/", "_")
-    else:
-        return "HEAD"
-
-def package_and_upload(package, package_dest, package_url):
-    return [
-            steps.ShellCommand(name="build package",
-                logEnviron=False, haltOnFailure=True, workdir="source",
-                command=["git", "archive", "HEAD", "-o", package]),
-
-            steps.FileUpload(name="upload package", workersrc=package,
-                workdir="source", masterdest=package_dest,
-                url=package_url, mode=0o644),
-
-            steps.ShellCommand(name="cleanup package", logEnviron=False,
-                haltOnFailure=True, workdir="source", alwaysRun=True,
-                command=["rm", "-f", package]),
-            ]
-
-# Steps
-class NixShellCommand(steps.ShellCommand):
-    def __init__(self, command=None, pure=True, nixfile=None, **kwargs):
-        assert(isinstance(command, str))
-        oldpath = kwargs.get("env", {}).get("PATH", None)
-        if which("nix-shell", path=oldpath) is None:
-            kwargs["env"] = kwargs.get("env", {})
-            if isinstance(oldpath, str):
-                kwargs["env"]["PATH"] = "/run/current-system/sw/bin:" + oldpath
-            elif isinstance(oldpath, list):
-                kwargs["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath
-        nixcommand = ["nix-shell"]
-        if pure:
-            nixcommand.append("--pure")
-        nixcommand.append("--run")
-        nixcommand.append(command)
-        if nixfile is not None:
-            nixcommand.append(nixfile)
-        super().__init__(command=nixcommand, **kwargs)
-
-# Schedulers
-def force_scheduler(name, builders):
-    return schedulers.ForceScheduler(name=name,
-        label="Force build", buttonName="Force build",
-        reason=util.StringParameter(name="reason", label="Reason", default="Force build"),
-        codebases=[
-            util.CodebaseParameter("",
-                branch=util.StringParameter(
-                    name="branch", label="Git reference (tag, branch)", required=True),
-                revision=util.FixedParameter(name="revision", default=""),
-                repository=util.FixedParameter(name="repository", default=""),
-                project=util.FixedParameter(name="project", default=""),
-                ),
-            ],
-        username=util.FixedParameter(name="username", default="Web button"),
-        builderNames=builders)
-
-def deploy_scheduler(name, builders):
-    return schedulers.ForceScheduler(name=name,
-        builderNames=builders,
-        label="Deploy built package", buttonName="Deploy",
-        username=util.FixedParameter(name="username", default="Web button"),
-        codebases=[
-            util.CodebaseParameter(codebase="",
-                branch=util.FixedParameter(name="branch", default=""),
-                revision=util.FixedParameter(name="revision", default=""),
-                repository=util.FixedParameter(name="repository", default=""),
-                project=util.FixedParameter(name="project", default=""))],
-        reason=util.FixedParameter(name="reason", default="Deploy"),
-        properties=[
-            util.ChoiceStringParameter(label="Environment",
-                name="environment", default="integration",
-                choices=["integration", "production"]),
-            BuildsList(label="Build to deploy", name="build"),
-            ]
-        )
-
-def hook_scheduler(project, timer=10):
-    return schedulers.AnyBranchScheduler(
-            change_filter=util.ChangeFilter(category="hooks", project=project),
-            name=project, treeStableTimer=timer, builderNames=["{}_build".format(project)])
-
-# Slack/XMPP status push
-from buildbot.reporters.http import HttpStatusPushBase
-from twisted.internet import defer
-from twisted.python import log
-from buildbot.util import httpclientservice
-from buildbot.reporters import utils
-from buildbot.process import results
-from twisted.words.protocols.jabber.jid import JID
-from wokkel import client, xmppim
-from functools import partial
-
-class SlackStatusPush(HttpStatusPushBase):
-    name = "SlackStatusPush"
-
-    @defer.inlineCallbacks
-    def reconfigService(self, serverUrl, **kwargs):
-        yield HttpStatusPushBase.reconfigService(self, **kwargs)
-        self._http = yield httpclientservice.HTTPClientService.getService(
-            self.master, serverUrl)
-
-    @defer.inlineCallbacks
-    def send(self, build):
-        yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
-        response = yield self._http.post("", json=self.format(build))
-        if response.code != 200:
-            log.msg("%s: unable to upload status: %s" %
-                    (response.code, response.content))
-
-    def format(self, build):
-        colors = [
-                "#36A64F", # success
-                "#F1E903", # warnings
-                "#DA0505", # failure
-                "#FFFFFF", # skipped
-                "#000000", # exception
-                "#FFFFFF", # retry
-                "#D02CA9", # cancelled
-                ]
-
-        if "environment" in build["properties"]:
-            msg = "{} environment".format(build["properties"]["environment"][0])
-            if "build" in build["properties"]:
-                msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
-        elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
-            msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
-        else:
-            msg = "build"
-
-        if build["complete"]:
-            timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
-            hours, rest = divmod(timedelta, 3600)
-            minutes, seconds = divmod(rest, 60)
-            if hours > 0:
-                duration = "{}h {}min {}s".format(hours, minutes, seconds)
-            elif minutes > 0:
-                duration = "{}min {}s".format(minutes, seconds)
-            else:
-                duration = "{}s".format(seconds)
-
-            text = "Build <{}|{}> of {}'s {} was {} in {}.".format(
-                    build["url"], build["buildid"],
-                    build["builder"]["name"],
-                    msg,
-                    results.Results[build["results"]],
-                    duration,
-                    )
-            fields = [
-                    {
-                        "title": "Build",
-                        "value": "<{}|{}>".format(build["url"], build["buildid"]),
-                        "short": True,
-                        },
-                    {
-                        "title": "Project",
-                        "value": build["builder"]["name"],
-                        "short": True,
-                        },
-                    {
-                        "title": "Build status",
-                        "value": results.Results[build["results"]],
-                        "short": True,
-                        },
-                    {
-                        "title": "Build duration",
-                        "value": duration,
-                        "short": True,
-                        },
-                    ]
-            if "environment" in build["properties"]:
-                fields.append({
-                        "title": "Environment",
-                        "value": build["properties"]["environment"][0],
-                        "short": True,
-                    })
-            if "build" in build["properties"]:
-                fields.append({
-                        "title": "Archive",
-                        "value": build["properties"]["build"][0],
-                        "short": True,
-                    })
-            attachments = [{
-                    "fallback": "",
-                    "color": colors[build["results"]],
-                    "fields": fields
-                    }]
-        else:
-            text = "Build <{}|{}> of {}'s {} started.".format(
-                    build["url"], build["buildid"],
-                    build["builder"]["name"],
-                    msg,
-                    )
-            attachments = []
-
-        return {
-                "username": "Buildbot",
-                "icon_url": "http://docs.buildbot.net/current/_static/icon.png",
-                "text": text,
-                "attachments": attachments,
-                }
-
-class XMPPStatusPush(HttpStatusPushBase):
-    name = "XMPPStatusPush"
-
-    @defer.inlineCallbacks
-    def reconfigService(self, password, recipients, **kwargs):
-        yield HttpStatusPushBase.reconfigService(self, **kwargs)
-        self.password = password
-        self.recipients = recipients
-
-    @defer.inlineCallbacks
-    def send(self, build):
-        yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
-        body = self.format(build)
-        factory = client.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self.password)
-        d = client.clientCreator(factory)
-        def send_message(recipient, stream):
-            message = xmppim.Message(recipient=JID(recipient), body=body)
-            message.stanzaType = 'chat'
-            stream.send(message.toElement())
-            # To allow chaining
-            return stream
-        for recipient in self.recipients:
-            d.addCallback(partial(send_message, recipient))
-        d.addCallback(lambda _: factory.streamManager.xmlstream.sendFooter())
-        d.addErrback(log.err)
-
-    def format(self, build):
-        if "environment" in build["properties"]:
-            msg = "{} environment".format(build["properties"]["environment"][0])
-            if "build" in build["properties"]:
-                msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
-        elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
-            msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
-        else:
-            msg = "build"
-
-        if build["complete"]:
-            timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
-            hours, rest = divmod(timedelta, 3600)
-            minutes, seconds = divmod(rest, 60)
-            if hours > 0:
-                duration = "{}h {}min {}s".format(hours, minutes, seconds)
-            elif minutes > 0:
-                duration = "{}min {}s".format(minutes, seconds)
-            else:
-                duration = "{}s".format(seconds)
-
-            text = "Build {} ( {} ) of {}'s {} was {} in {}.".format(
-                    build["buildid"], build["url"],
-                    build["builder"]["name"],
-                    msg,
-                    results.Results[build["results"]],
-                    duration,
-                    )
-        else:
-            text = "Build {} ( {} ) of {}'s {} started.".format(
-                    build["buildid"], build["url"],
-                    build["builder"]["name"],
-                    msg,
-                    )
-
-        return text
diff --git a/modules/private/buildbot/common/libvirt.py b/modules/private/buildbot/common/libvirt.py
deleted file mode 100644
index e250627..0000000
--- a/modules/private/buildbot/common/libvirt.py
+++ /dev/null
@@ -1,318 +0,0 @@
-# This file was part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Portions Copyright Buildbot Team Members
-# Portions Copyright 2010 Isotoma Limited
-
-
-import os
-
-from twisted.internet import defer
-from twisted.internet import threads
-from twisted.internet import utils
-from twisted.python import failure
-from twisted.python import log
-
-from buildbot import config
-from buildbot.util.eventual import eventually
-from buildbot.worker import AbstractLatentWorker
-
-try:
-    import libvirt
-except ImportError:
-    libvirt = None
-
-import random
-import string
-
-def random_string_generator():
-    chars = string.ascii_letters
-    return ''.join(random.choice(chars) for x in range(6))
-
-class WorkQueue:
-
-    """
-    I am a class that turns parallel access into serial access.
-
-    I exist because we want to run libvirt access in threads as we don't
-    trust calls not to block, but under load libvirt doesn't seem to like
-    this kind of threaded use.
-    """
-
-    def __init__(self):
-        self.queue = []
-
-    def _process(self):
-        log.msg("Looking to start a piece of work now...")
-
-        # Is there anything to do?
-        if not self.queue:
-            log.msg("_process called when there is no work")
-            return
-
-        # Peek at the top of the stack - get a function to call and
-        # a deferred to fire when its all over
-        d, next_operation, args, kwargs = self.queue[0]
-
-        # Start doing some work - expects a deferred
-        try:
-            d2 = next_operation(*args, **kwargs)
-        except Exception:
-            d2 = defer.fail()
-
-        # Whenever a piece of work is done, whether it worked or not
-        # call this to schedule the next piece of work
-        @d2.addBoth
-        def _work_done(res):
-            log.msg("Completed a piece of work")
-            self.queue.pop(0)
-            if self.queue:
-                log.msg("Preparing next piece of work")
-                eventually(self._process)
-            return res
-
-        # When the work is done, trigger d
-        d2.chainDeferred(d)
-
-    def execute(self, cb, *args, **kwargs):
-        kickstart_processing = not self.queue
-        d = defer.Deferred()
-        self.queue.append((d, cb, args, kwargs))
-        if kickstart_processing:
-            self._process()
-        return d
-
-    def executeInThread(self, cb, *args, **kwargs):
-        return self.execute(threads.deferToThread, cb, *args, **kwargs)
-
-
-# A module is effectively a singleton class, so this is OK
-queue = WorkQueue()
-
-
-class Domain:
-
-    """
-    I am a wrapper around a libvirt Domain object
-    """
-
-    def __init__(self, connection, domain):
-        self.connection = connection
-        self.domain = domain
-
-    def name(self):
-        return queue.executeInThread(self.domain.name)
-
-    def create(self):
-        return queue.executeInThread(self.domain.create)
-
-    def shutdown(self):
-        return queue.executeInThread(self.domain.shutdown)
-
-    def destroy(self):
-        return queue.executeInThread(self.domain.destroy)
-
-class Volume:
-    def __init__(self, connection, volume):
-        self.connection = connection
-        self.volume = volume
-
-    @defer.inlineCallbacks
-    def destroy(self):
-        yield queue.executeInThread(self.volume.wipe)
-        yield queue.executeInThread(self.volume.delete)
-
-class Pool:
-    VolumeClass = Volume
-    def __init__(self, connection, pool):
-        self.connection = connection
-        self.pool = pool
-
-    @defer.inlineCallbacks
-    def create_volume(self, xml):
-        res = yield queue.executeInThread(self.pool.createXML, xml)
-        return self.VolumeClass(self.connection, res)
-
-class Connection:
-
-    """
-    I am a wrapper around a libvirt Connection object.
-    """
-
-    DomainClass = Domain
-    PoolClass = Pool
-
-    def __init__(self, uri):
-        self.uri = uri
-        self._connection = None
-
-    @property
-    def connection(self):
-        if self._connection is not None:
-            try:
-                if not self._connection.isAlive():
-                    self._connection = None
-            except:
-                self._connection = None
-        if self._connection is None:
-            self._connection = libvirt.open(self.uri)
-        return self._connection
-
-    @defer.inlineCallbacks
-    def create(self, xml):
-        """ I take libvirt XML and start a new VM """
-        res = yield queue.executeInThread(self.connection.createXML, xml, 0)
-        return self.DomainClass(self, res)
-
-    @defer.inlineCallbacks
-    def lookup_pool(self, name):
-        res = yield queue.executeInThread(self.connection.storagePoolLookupByName, name)
-        return self.PoolClass(self, res)
-
-class LibVirtWorker(AbstractLatentWorker):
-
-    def __init__(self, name, password, connection, master_url, base_image=None, **kwargs):
-        super().__init__(name, password, **kwargs)
-        if not libvirt:
-            config.error(
-                "The python module 'libvirt' is needed to use a LibVirtWorker")
-
-        self.master_url = master_url
-        self.random_name = random_string_generator()
-        self.connection = connection
-        self.base_image = base_image
-
-        self.domain = None
-        self.domain_name = "buildbot-" + self.workername + "-" + self.random_name
-        self.volume = None
-        self.volume_name = "buildbot-" + self.workername + "-" + self.random_name
-        self.pool_name = "buildbot-disks"
-
-    def reconfigService(self, *args, **kwargs):
-        if 'build_wait_timeout' not in kwargs:
-            kwargs['build_wait_timeout'] = 0
-        return super().reconfigService(*args, **kwargs)
-
-    def canStartBuild(self):
-        if self.domain and not self.isConnected():
-            log.msg(
-                "Not accepting builds as existing domain but worker not connected")
-            return False
-
-        return super().canStartBuild()
-
-    @defer.inlineCallbacks
-    def _prepare_image(self):
-        log.msg("Creating temporary image {}".format(self.volume_name))
-        pool = yield self.connection.lookup_pool(self.pool_name)
-        vol_xml = """
-            <volume type='file'>
-                <name>{vol_name}</name>
-                <capacity unit='G'>10</capacity>
-                <target>
-                    <format type='qcow2'/>
-                    <permissions>
-                        <mode>0600</mode>
-                        <owner>0</owner>
-                        <group>0</group>
-                    </permissions>
-                </target>
-                <backingStore>
-                    <path>/etc/libvirtd/base-images/buildbot.qcow2</path>
-                    <format type='qcow2'/>
-                </backingStore>
-            </volume>
-        """.format(vol_name = self.volume_name)
-        self.volume = yield pool.create_volume(vol_xml)
-
-    @defer.inlineCallbacks
-    def start_instance(self, build):
-        """
-        I start a new instance of a VM.
-
-        If a base_image is specified, I will make a clone of that otherwise i will
-        use image directly.
-
-        If i'm not given libvirt domain definition XML, I will look for my name
-        in the list of defined virtual machines and start that.
-        """
-        domain_xml = """
-            <domain type="kvm">
-                <name>{domain_name}</name>
-                <memory unit="GiB">2</memory>
-                <vcpu>1</vcpu>
-                <sysinfo type='smbios'>
-                    <oemStrings>
-                        <entry>buildbot_master_url={master_url}</entry>
-                        <entry>buildbot_worker_name={worker_name}</entry>
-                    </oemStrings>
-                </sysinfo>
-                <os>
-                    <type arch="x86_64">hvm</type>
-                    <smbios mode='sysinfo'/>
-                </os>
-                <devices>
-                    <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
-                    <disk type="volume" device="disk">
-                        <driver name='qemu' type='qcow2' />
-                        <source type="volume" pool="{pool_name}" volume="{volume_name}" />
-                        <backingStore type='volume'>
-                        <format type='qcow2'/>
-                        <source type="volume" pool="niximages" volume="buildbot.qcow2" />
-                        </backingStore>
-                        <target dev="vda" bus="virtio"/>
-                    </disk>
-                    <input type="keyboard" bus="usb"/>
-                    <graphics type="vnc" port="-1" autoport="yes"/>
-                    <interface type="network">
-                        <source network="immae" />
-                    </interface>
-                </devices>
-            </domain>
-        """.format(volume_name = self.volume_name, master_url = self.master_url, pool_name =
-                self.pool_name, domain_name = self.domain_name, worker_name = self.workername)
-
-        yield self._prepare_image()
-
-        try:
-            self.domain = yield self.connection.create(domain_xml)
-        except Exception:
-            log.err(failure.Failure(),
-                    ("Cannot start a VM ({}), failing gracefully and triggering"
-                     "a new build check").format(self.workername))
-            self.domain = None
-            return False
-
-        return [self.domain_name]
-
-    def stop_instance(self, fast=False):
-        """
-        I attempt to stop a running VM.
-        I make sure any connection to the worker is removed.
-        If the VM was using a cloned image, I remove the clone
-        When everything is tidied up, I ask that bbot looks for work to do
-        """
-
-        log.msg("Attempting to stop '{}'".format(self.workername))
-        if self.domain is None:
-            log.msg("I don't think that domain is even running, aborting")
-            return defer.succeed(None)
-
-        domain = self.domain
-        self.domain = None
-
-        d = domain.destroy()
-        if self.volume is not None:
-            self.volume.destroy()
-
-        return d
diff --git a/modules/private/buildbot/common/master.cfg b/modules/private/buildbot/common/master.cfg
deleted file mode 100644
index abe08e0..0000000
--- a/modules/private/buildbot/common/master.cfg
+++ /dev/null
@@ -1,69 +0,0 @@
-# -*- python -*-
-# ex: set filetype=python:
-
-from buildbot.plugins import secrets, util, webhooks
-from buildbot.util import bytes2unicode
-import re
-import os
-from buildbot_config import E, configure
-import json
-
-class CustomBase(webhooks.base):
-    def getChanges(self, request):
-        try:
-            content = request.content.read()
-            args = json.loads(bytes2unicode(content))
-        except Exception as e:
-            raise ValueError("Error loading JSON: " + str(e))
-
-        args.setdefault("comments", "")
-        args.setdefault("repository", "")
-        args.setdefault("author", args.get("who"))
-
-        return ([args], None)
-
-userInfoProvider = util.LdapUserInfo(
-        uri=E.LDAP_URL,
-        bindUser=E.LDAP_ADMIN_USER,
-        bindPw=open(E.SECRETS_FILE + "/ldap", "r").read().rstrip(),
-        accountBase=E.LDAP_BASE,
-        accountPattern=E.LDAP_PATTERN,
-        accountFullName='cn',
-        accountEmail='mail',
-        avatarData="jpegPhoto",
-        groupBase=E.LDAP_BASE,
-        groupName="cn",
-        groupMemberPattern=E.LDAP_GROUP_PATTERN,
-        )
-
-c = BuildmasterConfig = {
-        "title": E.TITLE,
-        "titleURL": E.TITLE_URL,
-        "db": {
-            "db_url": "sqlite:///state.sqlite"
-            },
-        "protocols": { "pb": { "port": E.PB_SOCKET } },
-        "workers": [],
-        "change_source": [],
-        "schedulers": [],
-        "builders": [],
-        "services": [],
-        "secretsProviders": [
-            secrets.SecretInAFile(E.SECRETS_FILE),
-            ],
-        "www": {
-            "change_hook_dialects": { "base": { "custom_class": CustomBase } },
-            "plugins": {
-                "waterfall_view": {},
-                "console_view": {},
-                "grid_view": {},
-                "buildslist": {},
-                },
-            "auth": util.RemoteUserAuth(
-                header=b"X-Remote-User",
-                userInfoProvider=userInfoProvider,
-                headerRegex=re.compile(br"(?P<username>[^ @]+)")),
-            }
-        }
-
-configure(c)