aboutsummaryrefslogtreecommitdiff
path: root/modules/private/buildbot
diff options
context:
space:
mode:
authorIsmaël Bouya <ismael.bouya@normalesup.org>2021-06-24 22:24:15 +0200
committerIsmaël Bouya <ismael.bouya@normalesup.org>2021-06-24 22:24:15 +0200
commit200690c9aecec1f38c1a62a65916df2950e1afe7 (patch)
tree6aa365dd4c7164016837ac1e728d7bb25a7ce2be /modules/private/buildbot
parent6689bca19502aa8823dfc0fd3948e8e0a7cb9976 (diff)
downloadNix-200690c9aecec1f38c1a62a65916df2950e1afe7.tar.gz
Nix-200690c9aecec1f38c1a62a65916df2950e1afe7.tar.zst
Nix-200690c9aecec1f38c1a62a65916df2950e1afe7.zip
First attempt at making declarative VMs
In order to make buildbot more secure, the builds need to happen inside VMs so that they can be thrown out on demand when not needed. This commit implements this facility on dilion, and also defines declaratively some previous VMs which used to run on the machine.
Diffstat (limited to 'modules/private/buildbot')
-rw-r--r--modules/private/buildbot/common/libvirt.py306
-rw-r--r--modules/private/buildbot/default.nix18
-rw-r--r--modules/private/buildbot/projects/test/__init__.py19
3 files changed, 337 insertions, 6 deletions
diff --git a/modules/private/buildbot/common/libvirt.py b/modules/private/buildbot/common/libvirt.py
new file mode 100644
index 0000000..85fd908
--- /dev/null
+++ b/modules/private/buildbot/common/libvirt.py
@@ -0,0 +1,306 @@
1# This file was part of Buildbot. Buildbot is free software: you can
2# redistribute it and/or modify it under the terms of the GNU General Public
3# License as published by the Free Software Foundation, version 2.
4#
5# This program is distributed in the hope that it will be useful, but WITHOUT
6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
8# details.
9#
10# You should have received a copy of the GNU General Public License along with
11# this program; if not, write to the Free Software Foundation, Inc., 51
12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13#
14# Portions Copyright Buildbot Team Members
15# Portions Copyright 2010 Isotoma Limited
16
17
18import os
19
20from twisted.internet import defer
21from twisted.internet import threads
22from twisted.internet import utils
23from twisted.python import failure
24from twisted.python import log
25
26from buildbot import config
27from buildbot.util.eventual import eventually
28from buildbot.worker import AbstractLatentWorker
29
30try:
31 import libvirt
32except ImportError:
33 libvirt = None
34
35import random
36import string
37
38def random_string_generator():
39 chars = string.ascii_letters
40 return ''.join(random.choice(chars) for x in range(6))
41
42class WorkQueue:
43
44 """
45 I am a class that turns parallel access into serial access.
46
47 I exist because we want to run libvirt access in threads as we don't
48 trust calls not to block, but under load libvirt doesn't seem to like
49 this kind of threaded use.
50 """
51
52 def __init__(self):
53 self.queue = []
54
55 def _process(self):
56 log.msg("Looking to start a piece of work now...")
57
58 # Is there anything to do?
59 if not self.queue:
60 log.msg("_process called when there is no work")
61 return
62
63 # Peek at the top of the stack - get a function to call and
64 # a deferred to fire when its all over
65 d, next_operation, args, kwargs = self.queue[0]
66
67 # Start doing some work - expects a deferred
68 try:
69 d2 = next_operation(*args, **kwargs)
70 except Exception:
71 d2 = defer.fail()
72
73 # Whenever a piece of work is done, whether it worked or not
74 # call this to schedule the next piece of work
75 @d2.addBoth
76 def _work_done(res):
77 log.msg("Completed a piece of work")
78 self.queue.pop(0)
79 if self.queue:
80 log.msg("Preparing next piece of work")
81 eventually(self._process)
82 return res
83
84 # When the work is done, trigger d
85 d2.chainDeferred(d)
86
87 def execute(self, cb, *args, **kwargs):
88 kickstart_processing = not self.queue
89 d = defer.Deferred()
90 self.queue.append((d, cb, args, kwargs))
91 if kickstart_processing:
92 self._process()
93 return d
94
95 def executeInThread(self, cb, *args, **kwargs):
96 return self.execute(threads.deferToThread, cb, *args, **kwargs)
97
98
99# A module is effectively a singleton class, so this is OK
100queue = WorkQueue()
101
102
103class Domain:
104
105 """
106 I am a wrapper around a libvirt Domain object
107 """
108
109 def __init__(self, connection, domain):
110 self.connection = connection
111 self.domain = domain
112
113 def name(self):
114 return queue.executeInThread(self.domain.name)
115
116 def create(self):
117 return queue.executeInThread(self.domain.create)
118
119 def shutdown(self):
120 return queue.executeInThread(self.domain.shutdown)
121
122 def destroy(self):
123 return queue.executeInThread(self.domain.destroy)
124
125class Volume:
126 def __init__(self, connection, volume):
127 self.connection = connection
128 self.volume = volume
129
130 @defer.inlineCallbacks
131 def destroy(self):
132 yield queue.executeInThread(self.volume.wipe)
133 yield queue.executeInThread(self.volume.delete)
134
135class Pool:
136 VolumeClass = Volume
137 def __init__(self, connection, pool):
138 self.connection = connection
139 self.pool = pool
140
141 @defer.inlineCallbacks
142 def create_volume(self, xml):
143 res = yield queue.executeInThread(self.pool.createXML, xml)
144 return self.VolumeClass(self.connection, res)
145
146class Connection:
147
148 """
149 I am a wrapper around a libvirt Connection object.
150 """
151
152 DomainClass = Domain
153 PoolClass = Pool
154
155 def __init__(self, uri):
156 self.uri = uri
157 self.connection = libvirt.open(uri)
158
159 @defer.inlineCallbacks
160 def create(self, xml):
161 """ I take libvirt XML and start a new VM """
162 res = yield queue.executeInThread(self.connection.createXML, xml, 0)
163 return self.DomainClass(self, res)
164
165 @defer.inlineCallbacks
166 def lookup_pool(self, name):
167 res = yield queue.executeInThread(self.connection.storagePoolLookupByName, name)
168 return self.PoolClass(self, res)
169
170class LibVirtWorker(AbstractLatentWorker):
171
172 def __init__(self, name, password, connection, master_url, base_image=None, **kwargs):
173 super().__init__(name, password, **kwargs)
174 if not libvirt:
175 config.error(
176 "The python module 'libvirt' is needed to use a LibVirtWorker")
177
178 self.master_url = master_url
179 self.random_name = random_string_generator()
180 self.connection = connection
181 self.base_image = base_image
182
183 self.domain = None
184 self.domain_name = "buildbot-" + self.workername + "-" + self.random_name
185 self.volume = None
186 self.volume_name = "buildbot-" + self.workername + "-" + self.random_name
187 self.pool_name = "buildbot-disks"
188
189 def reconfigService(self, *args, **kwargs):
190 if 'build_wait_timeout' not in kwargs:
191 kwargs['build_wait_timeout'] = 0
192 return super().reconfigService(*args, **kwargs)
193
194 def canStartBuild(self):
195 if self.domain and not self.isConnected():
196 log.msg(
197 "Not accepting builds as existing domain but worker not connected")
198 return False
199
200 return super().canStartBuild()
201
202 @defer.inlineCallbacks
203 def _prepare_image(self):
204 log.msg("Creating temporary image {}".format(self.volume_name))
205 pool = yield self.connection.lookup_pool(self.pool_name)
206 vol_xml = """
207 <volume type='file'>
208 <name>{vol_name}</name>
209 <capacity unit='G'>10</capacity>
210 <target>
211 <format type='qcow2'/>
212 <permissions>
213 <mode>0600</mode>
214 <owner>0</owner>
215 <group>0</group>
216 </permissions>
217 </target>
218 <backingStore>
219 <path>/etc/libvirtd/base-images/buildbot.qcow2</path>
220 <format type='qcow2'/>
221 </backingStore>
222 </volume>
223 """.format(vol_name = self.volume_name)
224 self.volume = yield pool.create_volume(vol_xml)
225
226 @defer.inlineCallbacks
227 def start_instance(self, build):
228 """
229 I start a new instance of a VM.
230
231 If a base_image is specified, I will make a clone of that otherwise i will
232 use image directly.
233
234 If i'm not given libvirt domain definition XML, I will look for my name
235 in the list of defined virtual machines and start that.
236 """
237 domain_xml = """
238 <domain type="kvm">
239 <name>{domain_name}</name>
240 <memory unit="GiB">2</memory>
241 <vcpu>1</vcpu>
242 <sysinfo type='smbios'>
243 <oemStrings>
244 <entry>buildbot_master_url={master_url}</entry>
245 <entry>buildbot_worker_name={worker_name}</entry>
246 </oemStrings>
247 </sysinfo>
248 <os>
249 <type arch="x86_64">hvm</type>
250 <smbios mode='sysinfo'/>
251 </os>
252 <devices>
253 <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
254 <disk type="volume" device="disk">
255 <driver name='qemu' type='qcow2' />
256 <source type="volume" pool="{pool_name}" volume="{volume_name}" />
257 <backingStore type='volume'>
258 <format type='qcow2'/>
259 <source type="volume" pool="niximages" volume="buildbot.qcow2" />
260 </backingStore>
261 <target dev="vda" bus="virtio"/>
262 </disk>
263 <input type="keyboard" bus="usb"/>
264 <graphics type="vnc" port="-1" autoport="yes"/>
265 <interface type="network">
266 <source network="immae" />
267 </interface>
268 </devices>
269 </domain>
270 """.format(volume_name = self.volume_name, master_url = self.master_url, pool_name =
271 self.pool_name, domain_name = self.domain_name, worker_name = self.workername)
272
273 yield self._prepare_image()
274
275 try:
276 self.domain = yield self.connection.create(domain_xml)
277 except Exception:
278 log.err(failure.Failure(),
279 ("Cannot start a VM ({}), failing gracefully and triggering"
280 "a new build check").format(self.workername))
281 self.domain = None
282 return False
283
284 return [self.domain_name]
285
286 def stop_instance(self, fast=False):
287 """
288 I attempt to stop a running VM.
289 I make sure any connection to the worker is removed.
290 If the VM was using a cloned image, I remove the clone
291 When everything is tidied up, I ask that bbot looks for work to do
292 """
293
294 log.msg("Attempting to stop '{}'".format(self.workername))
295 if self.domain is None:
296 log.msg("I don't think that domain is even running, aborting")
297 return defer.succeed(None)
298
299 domain = self.domain
300 self.domain = None
301
302 d = domain.destroy()
303 if self.volume is not None:
304 self.volume.destroy()
305
306 return d
diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix
index d6753e5..ac34845 100644
--- a/modules/private/buildbot/default.nix
+++ b/modules/private/buildbot/default.nix
@@ -107,7 +107,12 @@ in
107 project_env = with lib.attrsets; 107 project_env = with lib.attrsets;
108 mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment // 108 mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment //
109 mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) // 109 mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) //
110 { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; }; 110 {
111 BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}";
112 BUILDBOT_WORKER_PORT = builtins.toString project.workerPort;
113 BUILDBOT_HOST = config.hostEnv.fqdn;
114 BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system";
115 };
111 in builtins.concatStringsSep "\n" 116 in builtins.concatStringsSep "\n"
112 (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env); 117 (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env);
113 } 118 }
@@ -126,6 +131,13 @@ in
126 permissions = "0600"; 131 permissions = "0600";
127 user = "buildbot"; 132 user = "buildbot";
128 group = "buildbot"; 133 group = "buildbot";
134 text = config.myEnv.buildbot.workerPassword;
135 dest = "buildbot/worker_password";
136 }
137 {
138 permissions = "0600";
139 user = "buildbot";
140 group = "buildbot";
129 text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key"; 141 text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key";
130 dest = "buildbot/ssh_key"; 142 dest = "buildbot/ssh_key";
131 } 143 }
@@ -135,6 +147,7 @@ in
135 restart = true; 147 restart = true;
136 paths = [ 148 paths = [
137 "/var/secrets/buildbot/ldap" 149 "/var/secrets/buildbot/ldap"
150 "/var/secrets/buildbot/worker_password"
138 "/var/secrets/buildbot/ssh_key" 151 "/var/secrets/buildbot/ssh_key"
139 "/var/secrets/buildbot/${project.name}/environment_file" 152 "/var/secrets/buildbot/${project.name}/environment_file"
140 ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets; 153 ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets;
@@ -144,6 +157,7 @@ in
144 description = "buildbot slice"; 157 description = "buildbot slice";
145 }; 158 };
146 159
160 networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects;
147 systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { 161 systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
148 description = "Buildbot Continuous Integration Server ${project.name}."; 162 description = "Buildbot Continuous Integration Server ${project.name}.";
149 after = [ "network-online.target" ]; 163 after = [ "network-online.target" ];
@@ -196,6 +210,7 @@ in
196 buildbot_secrets=${varDir}/${project.name}/secrets 210 buildbot_secrets=${varDir}/${project.name}/secrets
197 install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets 211 install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
198 install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap 212 install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap
213 install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/worker_password $buildbot_secrets/worker_password
199 ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList 214 ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
200 (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets 215 (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets
201 )} 216 )}
@@ -213,6 +228,7 @@ in
213 }); 228 });
214 HOME = "${varDir}/${project.name}"; 229 HOME = "${varDir}/${project.name}";
215 PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [ 230 PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
231 pkgs.python3Packages.libvirt
216 pkgs.python3Packages.wokkel 232 pkgs.python3Packages.wokkel
217 pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot 233 pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
218 pkgs.python3Packages.buildbot-worker 234 pkgs.python3Packages.buildbot-worker
diff --git a/modules/private/buildbot/projects/test/__init__.py b/modules/private/buildbot/projects/test/__init__.py
index e6b8d51..e2f6f82 100644
--- a/modules/private/buildbot/projects/test/__init__.py
+++ b/modules/private/buildbot/projects/test/__init__.py
@@ -1,5 +1,6 @@
1from buildbot.plugins import * 1from buildbot.plugins import *
2from buildbot_common.build_helpers import * 2from buildbot_common.build_helpers import *
3import buildbot_common.libvirt as ilibvirt
3import os 4import os
4from buildbot.util import bytes2unicode 5from buildbot.util import bytes2unicode
5import json 6import json
@@ -10,11 +11,13 @@ class E():
10 PROJECT = "test" 11 PROJECT = "test"
11 BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) 12 BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
12 SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) 13 SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT)
13 PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) 14 PB_SOCKET = os.environ["BUILDBOT_WORKER_PORT"]
15 WORKER_HOST = "{}:{}".format(os.environ["BUILDBOT_HOST"], PB_SOCKET)
14 RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT) 16 RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT)
15 RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT) 17 RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT)
16 GIT_URL = "https://git.immae.eu/perso/Immae/TestProject.git" 18 GIT_URL = "https://git.immae.eu/perso/Immae/TestProject.git"
17 SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" 19 SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key"
20 LIBVIRT_URL = os.environ["BUILDBOT_VIRT_URL"] + "?keyfile=" + SSH_KEY_PATH
18 PUPPET_HOST = "root@backup-1.v.immae.eu" 21 PUPPET_HOST = "root@backup-1.v.immae.eu"
19 LDAP_HOST = "ldap.immae.eu" 22 LDAP_HOST = "ldap.immae.eu"
20 LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" 23 LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu"
@@ -70,8 +73,14 @@ def configure(c):
70 c["www"]["change_hook_dialects"]["base"] = { 73 c["www"]["change_hook_dialects"]["base"] = {
71 "custom_class": CustomBase 74 "custom_class": CustomBase
72 } 75 }
73 c['workers'].append(worker.LocalWorker("generic-worker-test")) 76 c['workers'].append(ilibvirt.LibVirtWorker("test-build",
74 c['workers'].append(worker.LocalWorker("deploy-worker-test")) 77 open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
78 ilibvirt.Connection(E.LIBVIRT_URL),
79 E.WORKER_HOST))
80 c['workers'].append(ilibvirt.LibVirtWorker("test-deploy",
81 open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
82 ilibvirt.Connection(E.LIBVIRT_URL),
83 E.WORKER_HOST))
75 84
76 c['schedulers'].append(hook_scheduler("TestProject", timer=1)) 85 c['schedulers'].append(hook_scheduler("TestProject", timer=1))
77 c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"])) 86 c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"]))
@@ -109,7 +118,7 @@ def factory():
109 logEnviron=False, command=["echo", package])) 118 logEnviron=False, command=["echo", package]))
110 factory.addSteps(package_and_upload(package, package_dest, package_url)) 119 factory.addSteps(package_and_upload(package, package_dest, package_url))
111 120
112 return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory) 121 return util.BuilderConfig(name="TestProject_build", workernames=["test-build"], factory=factory)
113 122
114 123
115def compute_build_infos(): 124def compute_build_infos():
@@ -143,7 +152,7 @@ def deploy_factory():
143 ldap_password=util.Secret("ldap"))) 152 ldap_password=util.Secret("ldap")))
144 factory.addStep(steps.MasterShellCommand(command=[ 153 factory.addStep(steps.MasterShellCommand(command=[
145 "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host])) 154 "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host]))
146 return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory) 155 return util.BuilderConfig(name="TestProject_deploy", workernames=["test-deploy"], factory=factory)
147 156
148from twisted.internet import defer 157from twisted.internet import defer
149from buildbot.process.buildstep import FAILURE 158from buildbot.process.buildstep import FAILURE