aboutsummaryrefslogtreecommitdiff
path: root/modules/private
diff options
context:
space:
mode:
authorIsmaël Bouya <ismael.bouya@normalesup.org>2021-06-24 22:24:15 +0200
committerIsmaël Bouya <ismael.bouya@normalesup.org>2021-06-24 22:24:15 +0200
commit200690c9aecec1f38c1a62a65916df2950e1afe7 (patch)
tree6aa365dd4c7164016837ac1e728d7bb25a7ce2be /modules/private
parent6689bca19502aa8823dfc0fd3948e8e0a7cb9976 (diff)
downloadNix-200690c9aecec1f38c1a62a65916df2950e1afe7.tar.gz
Nix-200690c9aecec1f38c1a62a65916df2950e1afe7.tar.zst
Nix-200690c9aecec1f38c1a62a65916df2950e1afe7.zip
First attempt at making declarative VMs
In order to make buildbot more secure, the builds need to happen inside VMs so that they can be thrown out on demand when not needed. This commit implements this facility on dilion, and also defines declaratively some previous VMs which used to run on the machine.
Diffstat (limited to 'modules/private')
-rw-r--r--modules/private/buildbot/common/libvirt.py306
-rw-r--r--modules/private/buildbot/default.nix18
-rw-r--r--modules/private/buildbot/projects/test/__init__.py19
-rw-r--r--modules/private/environment.nix3
-rw-r--r--modules/private/gitolite/default.nix2
-rw-r--r--modules/private/system/dilion.nix18
-rw-r--r--modules/private/system/dilion/vms.nix146
-rw-r--r--modules/private/system/dilion/vms/base_configuration.nix21
-rw-r--r--modules/private/system/dilion/vms/base_image.nix94
-rw-r--r--modules/private/system/dilion/vms/buildbot_configuration.nix67
10 files changed, 684 insertions, 10 deletions
diff --git a/modules/private/buildbot/common/libvirt.py b/modules/private/buildbot/common/libvirt.py
new file mode 100644
index 0000000..85fd908
--- /dev/null
+++ b/modules/private/buildbot/common/libvirt.py
@@ -0,0 +1,306 @@
1# This file was part of Buildbot. Buildbot is free software: you can
2# redistribute it and/or modify it under the terms of the GNU General Public
3# License as published by the Free Software Foundation, version 2.
4#
5# This program is distributed in the hope that it will be useful, but WITHOUT
6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
8# details.
9#
10# You should have received a copy of the GNU General Public License along with
11# this program; if not, write to the Free Software Foundation, Inc., 51
12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13#
14# Portions Copyright Buildbot Team Members
15# Portions Copyright 2010 Isotoma Limited
16
17
18import os
19
20from twisted.internet import defer
21from twisted.internet import threads
22from twisted.internet import utils
23from twisted.python import failure
24from twisted.python import log
25
26from buildbot import config
27from buildbot.util.eventual import eventually
28from buildbot.worker import AbstractLatentWorker
29
30try:
31 import libvirt
32except ImportError:
33 libvirt = None
34
35import random
36import string
37
38def random_string_generator():
39 chars = string.ascii_letters
40 return ''.join(random.choice(chars) for x in range(6))
41
42class WorkQueue:
43
44 """
45 I am a class that turns parallel access into serial access.
46
47 I exist because we want to run libvirt access in threads as we don't
48 trust calls not to block, but under load libvirt doesn't seem to like
49 this kind of threaded use.
50 """
51
52 def __init__(self):
53 self.queue = []
54
55 def _process(self):
56 log.msg("Looking to start a piece of work now...")
57
58 # Is there anything to do?
59 if not self.queue:
60 log.msg("_process called when there is no work")
61 return
62
63 # Peek at the top of the stack - get a function to call and
64 # a deferred to fire when its all over
65 d, next_operation, args, kwargs = self.queue[0]
66
67 # Start doing some work - expects a deferred
68 try:
69 d2 = next_operation(*args, **kwargs)
70 except Exception:
71 d2 = defer.fail()
72
73 # Whenever a piece of work is done, whether it worked or not
74 # call this to schedule the next piece of work
75 @d2.addBoth
76 def _work_done(res):
77 log.msg("Completed a piece of work")
78 self.queue.pop(0)
79 if self.queue:
80 log.msg("Preparing next piece of work")
81 eventually(self._process)
82 return res
83
84 # When the work is done, trigger d
85 d2.chainDeferred(d)
86
87 def execute(self, cb, *args, **kwargs):
88 kickstart_processing = not self.queue
89 d = defer.Deferred()
90 self.queue.append((d, cb, args, kwargs))
91 if kickstart_processing:
92 self._process()
93 return d
94
95 def executeInThread(self, cb, *args, **kwargs):
96 return self.execute(threads.deferToThread, cb, *args, **kwargs)
97
98
99# A module is effectively a singleton class, so this is OK
100queue = WorkQueue()
101
102
103class Domain:
104
105 """
106 I am a wrapper around a libvirt Domain object
107 """
108
109 def __init__(self, connection, domain):
110 self.connection = connection
111 self.domain = domain
112
113 def name(self):
114 return queue.executeInThread(self.domain.name)
115
116 def create(self):
117 return queue.executeInThread(self.domain.create)
118
119 def shutdown(self):
120 return queue.executeInThread(self.domain.shutdown)
121
122 def destroy(self):
123 return queue.executeInThread(self.domain.destroy)
124
125class Volume:
126 def __init__(self, connection, volume):
127 self.connection = connection
128 self.volume = volume
129
130 @defer.inlineCallbacks
131 def destroy(self):
132 yield queue.executeInThread(self.volume.wipe)
133 yield queue.executeInThread(self.volume.delete)
134
135class Pool:
136 VolumeClass = Volume
137 def __init__(self, connection, pool):
138 self.connection = connection
139 self.pool = pool
140
141 @defer.inlineCallbacks
142 def create_volume(self, xml):
143 res = yield queue.executeInThread(self.pool.createXML, xml)
144 return self.VolumeClass(self.connection, res)
145
146class Connection:
147
148 """
149 I am a wrapper around a libvirt Connection object.
150 """
151
152 DomainClass = Domain
153 PoolClass = Pool
154
155 def __init__(self, uri):
156 self.uri = uri
157 self.connection = libvirt.open(uri)
158
159 @defer.inlineCallbacks
160 def create(self, xml):
161 """ I take libvirt XML and start a new VM """
162 res = yield queue.executeInThread(self.connection.createXML, xml, 0)
163 return self.DomainClass(self, res)
164
165 @defer.inlineCallbacks
166 def lookup_pool(self, name):
167 res = yield queue.executeInThread(self.connection.storagePoolLookupByName, name)
168 return self.PoolClass(self, res)
169
170class LibVirtWorker(AbstractLatentWorker):
171
172 def __init__(self, name, password, connection, master_url, base_image=None, **kwargs):
173 super().__init__(name, password, **kwargs)
174 if not libvirt:
175 config.error(
176 "The python module 'libvirt' is needed to use a LibVirtWorker")
177
178 self.master_url = master_url
179 self.random_name = random_string_generator()
180 self.connection = connection
181 self.base_image = base_image
182
183 self.domain = None
184 self.domain_name = "buildbot-" + self.workername + "-" + self.random_name
185 self.volume = None
186 self.volume_name = "buildbot-" + self.workername + "-" + self.random_name
187 self.pool_name = "buildbot-disks"
188
189 def reconfigService(self, *args, **kwargs):
190 if 'build_wait_timeout' not in kwargs:
191 kwargs['build_wait_timeout'] = 0
192 return super().reconfigService(*args, **kwargs)
193
194 def canStartBuild(self):
195 if self.domain and not self.isConnected():
196 log.msg(
197 "Not accepting builds as existing domain but worker not connected")
198 return False
199
200 return super().canStartBuild()
201
202 @defer.inlineCallbacks
203 def _prepare_image(self):
204 log.msg("Creating temporary image {}".format(self.volume_name))
205 pool = yield self.connection.lookup_pool(self.pool_name)
206 vol_xml = """
207 <volume type='file'>
208 <name>{vol_name}</name>
209 <capacity unit='G'>10</capacity>
210 <target>
211 <format type='qcow2'/>
212 <permissions>
213 <mode>0600</mode>
214 <owner>0</owner>
215 <group>0</group>
216 </permissions>
217 </target>
218 <backingStore>
219 <path>/etc/libvirtd/base-images/buildbot.qcow2</path>
220 <format type='qcow2'/>
221 </backingStore>
222 </volume>
223 """.format(vol_name = self.volume_name)
224 self.volume = yield pool.create_volume(vol_xml)
225
226 @defer.inlineCallbacks
227 def start_instance(self, build):
228 """
229 I start a new instance of a VM.
230
231 If a base_image is specified, I will make a clone of that otherwise i will
232 use image directly.
233
234 If i'm not given libvirt domain definition XML, I will look for my name
235 in the list of defined virtual machines and start that.
236 """
237 domain_xml = """
238 <domain type="kvm">
239 <name>{domain_name}</name>
240 <memory unit="GiB">2</memory>
241 <vcpu>1</vcpu>
242 <sysinfo type='smbios'>
243 <oemStrings>
244 <entry>buildbot_master_url={master_url}</entry>
245 <entry>buildbot_worker_name={worker_name}</entry>
246 </oemStrings>
247 </sysinfo>
248 <os>
249 <type arch="x86_64">hvm</type>
250 <smbios mode='sysinfo'/>
251 </os>
252 <devices>
253 <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
254 <disk type="volume" device="disk">
255 <driver name='qemu' type='qcow2' />
256 <source type="volume" pool="{pool_name}" volume="{volume_name}" />
257 <backingStore type='volume'>
258 <format type='qcow2'/>
259 <source type="volume" pool="niximages" volume="buildbot.qcow2" />
260 </backingStore>
261 <target dev="vda" bus="virtio"/>
262 </disk>
263 <input type="keyboard" bus="usb"/>
264 <graphics type="vnc" port="-1" autoport="yes"/>
265 <interface type="network">
266 <source network="immae" />
267 </interface>
268 </devices>
269 </domain>
270 """.format(volume_name = self.volume_name, master_url = self.master_url, pool_name =
271 self.pool_name, domain_name = self.domain_name, worker_name = self.workername)
272
273 yield self._prepare_image()
274
275 try:
276 self.domain = yield self.connection.create(domain_xml)
277 except Exception:
278 log.err(failure.Failure(),
279 ("Cannot start a VM ({}), failing gracefully and triggering"
280 "a new build check").format(self.workername))
281 self.domain = None
282 return False
283
284 return [self.domain_name]
285
286 def stop_instance(self, fast=False):
287 """
288 I attempt to stop a running VM.
289 I make sure any connection to the worker is removed.
290 If the VM was using a cloned image, I remove the clone
291 When everything is tidied up, I ask that bbot looks for work to do
292 """
293
294 log.msg("Attempting to stop '{}'".format(self.workername))
295 if self.domain is None:
296 log.msg("I don't think that domain is even running, aborting")
297 return defer.succeed(None)
298
299 domain = self.domain
300 self.domain = None
301
302 d = domain.destroy()
303 if self.volume is not None:
304 self.volume.destroy()
305
306 return d
diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix
index d6753e5..ac34845 100644
--- a/modules/private/buildbot/default.nix
+++ b/modules/private/buildbot/default.nix
@@ -107,7 +107,12 @@ in
107 project_env = with lib.attrsets; 107 project_env = with lib.attrsets;
108 mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment // 108 mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment //
109 mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) // 109 mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) //
110 { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; }; 110 {
111 BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}";
112 BUILDBOT_WORKER_PORT = builtins.toString project.workerPort;
113 BUILDBOT_HOST = config.hostEnv.fqdn;
114 BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system";
115 };
111 in builtins.concatStringsSep "\n" 116 in builtins.concatStringsSep "\n"
112 (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env); 117 (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env);
113 } 118 }
@@ -126,6 +131,13 @@ in
126 permissions = "0600"; 131 permissions = "0600";
127 user = "buildbot"; 132 user = "buildbot";
128 group = "buildbot"; 133 group = "buildbot";
134 text = config.myEnv.buildbot.workerPassword;
135 dest = "buildbot/worker_password";
136 }
137 {
138 permissions = "0600";
139 user = "buildbot";
140 group = "buildbot";
129 text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key"; 141 text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key";
130 dest = "buildbot/ssh_key"; 142 dest = "buildbot/ssh_key";
131 } 143 }
@@ -135,6 +147,7 @@ in
135 restart = true; 147 restart = true;
136 paths = [ 148 paths = [
137 "/var/secrets/buildbot/ldap" 149 "/var/secrets/buildbot/ldap"
150 "/var/secrets/buildbot/worker_password"
138 "/var/secrets/buildbot/ssh_key" 151 "/var/secrets/buildbot/ssh_key"
139 "/var/secrets/buildbot/${project.name}/environment_file" 152 "/var/secrets/buildbot/${project.name}/environment_file"
140 ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets; 153 ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets;
@@ -144,6 +157,7 @@ in
144 description = "buildbot slice"; 157 description = "buildbot slice";
145 }; 158 };
146 159
160 networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects;
147 systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { 161 systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
148 description = "Buildbot Continuous Integration Server ${project.name}."; 162 description = "Buildbot Continuous Integration Server ${project.name}.";
149 after = [ "network-online.target" ]; 163 after = [ "network-online.target" ];
@@ -196,6 +210,7 @@ in
196 buildbot_secrets=${varDir}/${project.name}/secrets 210 buildbot_secrets=${varDir}/${project.name}/secrets
197 install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets 211 install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
198 install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap 212 install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap
213 install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/worker_password $buildbot_secrets/worker_password
199 ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList 214 ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
200 (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets 215 (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets
201 )} 216 )}
@@ -213,6 +228,7 @@ in
213 }); 228 });
214 HOME = "${varDir}/${project.name}"; 229 HOME = "${varDir}/${project.name}";
215 PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [ 230 PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
231 pkgs.python3Packages.libvirt
216 pkgs.python3Packages.wokkel 232 pkgs.python3Packages.wokkel
217 pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot 233 pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
218 pkgs.python3Packages.buildbot-worker 234 pkgs.python3Packages.buildbot-worker
diff --git a/modules/private/buildbot/projects/test/__init__.py b/modules/private/buildbot/projects/test/__init__.py
index e6b8d51..e2f6f82 100644
--- a/modules/private/buildbot/projects/test/__init__.py
+++ b/modules/private/buildbot/projects/test/__init__.py
@@ -1,5 +1,6 @@
1from buildbot.plugins import * 1from buildbot.plugins import *
2from buildbot_common.build_helpers import * 2from buildbot_common.build_helpers import *
3import buildbot_common.libvirt as ilibvirt
3import os 4import os
4from buildbot.util import bytes2unicode 5from buildbot.util import bytes2unicode
5import json 6import json
@@ -10,11 +11,13 @@ class E():
10 PROJECT = "test" 11 PROJECT = "test"
11 BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) 12 BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
12 SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) 13 SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT)
13 PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) 14 PB_SOCKET = os.environ["BUILDBOT_WORKER_PORT"]
15 WORKER_HOST = "{}:{}".format(os.environ["BUILDBOT_HOST"], PB_SOCKET)
14 RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT) 16 RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT)
15 RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT) 17 RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT)
16 GIT_URL = "https://git.immae.eu/perso/Immae/TestProject.git" 18 GIT_URL = "https://git.immae.eu/perso/Immae/TestProject.git"
17 SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" 19 SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key"
20 LIBVIRT_URL = os.environ["BUILDBOT_VIRT_URL"] + "?keyfile=" + SSH_KEY_PATH
18 PUPPET_HOST = "root@backup-1.v.immae.eu" 21 PUPPET_HOST = "root@backup-1.v.immae.eu"
19 LDAP_HOST = "ldap.immae.eu" 22 LDAP_HOST = "ldap.immae.eu"
20 LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" 23 LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu"
@@ -70,8 +73,14 @@ def configure(c):
70 c["www"]["change_hook_dialects"]["base"] = { 73 c["www"]["change_hook_dialects"]["base"] = {
71 "custom_class": CustomBase 74 "custom_class": CustomBase
72 } 75 }
73 c['workers'].append(worker.LocalWorker("generic-worker-test")) 76 c['workers'].append(ilibvirt.LibVirtWorker("test-build",
74 c['workers'].append(worker.LocalWorker("deploy-worker-test")) 77 open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
78 ilibvirt.Connection(E.LIBVIRT_URL),
79 E.WORKER_HOST))
80 c['workers'].append(ilibvirt.LibVirtWorker("test-deploy",
81 open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
82 ilibvirt.Connection(E.LIBVIRT_URL),
83 E.WORKER_HOST))
75 84
76 c['schedulers'].append(hook_scheduler("TestProject", timer=1)) 85 c['schedulers'].append(hook_scheduler("TestProject", timer=1))
77 c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"])) 86 c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"]))
@@ -109,7 +118,7 @@ def factory():
109 logEnviron=False, command=["echo", package])) 118 logEnviron=False, command=["echo", package]))
110 factory.addSteps(package_and_upload(package, package_dest, package_url)) 119 factory.addSteps(package_and_upload(package, package_dest, package_url))
111 120
112 return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory) 121 return util.BuilderConfig(name="TestProject_build", workernames=["test-build"], factory=factory)
113 122
114 123
115def compute_build_infos(): 124def compute_build_infos():
@@ -143,7 +152,7 @@ def deploy_factory():
143 ldap_password=util.Secret("ldap"))) 152 ldap_password=util.Secret("ldap")))
144 factory.addStep(steps.MasterShellCommand(command=[ 153 factory.addStep(steps.MasterShellCommand(command=[
145 "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host])) 154 "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host]))
146 return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory) 155 return util.BuilderConfig(name="TestProject_deploy", workernames=["test-deploy"], factory=factory)
147 156
148from twisted.internet import defer 157from twisted.internet import defer
149from buildbot.process.buildstep import FAILURE 158from buildbot.process.buildstep import FAILURE
diff --git a/modules/private/environment.nix b/modules/private/environment.nix
index 719bf8f..f0af572 100644
--- a/modules/private/environment.nix
+++ b/modules/private/environment.nix
@@ -228,6 +228,7 @@ in
228 ''; 228 '';
229 type = submodule { 229 type = submodule {
230 options = { 230 options = {
231 rootKeys = mkOption { type = attrsOf str; description = "Keys of root users"; };
231 ldap = mkOption { 232 ldap = mkOption {
232 description = '' 233 description = ''
233 LDAP credentials for cn=ssh,ou=services,dc=immae,dc=eu dn 234 LDAP credentials for cn=ssh,ou=services,dc=immae,dc=eu dn
@@ -804,6 +805,7 @@ in
804 description = "Buildbot configuration"; 805 description = "Buildbot configuration";
805 type = submodule { 806 type = submodule {
806 options = { 807 options = {
808 workerPassword = mkOption { description = "Buildbot worker password"; type = str; };
807 user = mkOption { 809 user = mkOption {
808 description = "Buildbot user"; 810 description = "Buildbot user";
809 type = submodule { 811 type = submodule {
@@ -855,6 +857,7 @@ in
855 ''; 857 '';
856 }; 858 };
857 pythonPathHome = mkOption { type = bool; description = "Whether to add project’s python home to python path"; }; 859 pythonPathHome = mkOption { type = bool; description = "Whether to add project’s python home to python path"; };
860 workerPort = mkOption { type = port; description = "Port for the worker"; };
858 secrets = mkOption { 861 secrets = mkOption {
859 type = attrsOf str; 862 type = attrsOf str;
860 description = "Secrets for the project to dump as files"; 863 description = "Secrets for the project to dump as files";
diff --git a/modules/private/gitolite/default.nix b/modules/private/gitolite/default.nix
index 6b573e3..e54ee8a 100644
--- a/modules/private/gitolite/default.nix
+++ b/modules/private/gitolite/default.nix
@@ -74,7 +74,7 @@ in {
74 # Installation: https://git.immae.eu/mantisbt/view.php?id=93 74 # Installation: https://git.immae.eu/mantisbt/view.php?id=93
75 services.gitolite = { 75 services.gitolite = {
76 enable = true; 76 enable = true;
77 adminPubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXqRbiHw7QoHADNIEuo4nUT9fSOIEBMdJZH0bkQAxXyJFyCM1IMz0pxsHV0wu9tdkkr36bPEUj2aV5bkYLBN6nxcV2Y49X8bjOSCPfx3n6Own1h+NeZVBj4ZByrFmqCbTxUJIZ2bZKcWOFncML39VmWdsVhNjg0X4NBBehqXRIKr2gt3E/ESAxTYJFm0BnU0baciw9cN0bsRGqvFgf5h2P48CIAfwhVcGmPQnnAwabnosYQzRWxR0OygH5Kd8mePh6FheIRIigfXsDO8f/jdxwut8buvNIf3m5EBr3tUbTsvM+eV3M5vKGt7sk8T64DVtepTSdOOWtp+47ktsnHOMh immae@immae.eu"; 77 adminPubkey = config.myEnv.sshd.rootKeys.immae_dilion;
78 }; 78 };
79 }; 79 };
80} 80}
diff --git a/modules/private/system/dilion.nix b/modules/private/system/dilion.nix
index be8269e..a59d607 100644
--- a/modules/private/system/dilion.nix
+++ b/modules/private/system/dilion.nix
@@ -76,12 +76,24 @@
76 }; 76 };
77 77
78 myServices.ssh.modules = [ config.myServices.ssh.predefinedModules.regular ]; 78 myServices.ssh.modules = [ config.myServices.ssh.predefinedModules.regular ];
79 imports = builtins.attrValues (import ../..); 79 imports = builtins.attrValues (import ../..) ++ [ ./dilion/vms.nix ];
80 80
81 system.nssModules = [ pkgs.libvirt ]; 81 system.nssModules = [ pkgs.libvirt ];
82 system.nssDatabases.hosts = lib.mkForce [ "files" "libvirt_guest" "mymachines" "dns" "myhostname" ]; 82 system.nssDatabases.hosts = lib.mkForce [ "files" "libvirt_guest" "mymachines" "dns" "myhostname" ];
83 programs.zsh.enable = true; 83 programs.zsh.enable = true;
84 84
85 users.users.libvirt = {
86 hashedPassword = "!";
87 shell = pkgs.bashInteractive;
88 isSystemUser = true;
89 group = "libvirtd";
90 packages = [ pkgs.netcat-openbsd ];
91 openssh.authorizedKeys.keyFiles = [
92 "${privateFiles}/buildbot_ssh_key.pub"
93 ];
94 openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.ismael_flony ];
95 };
96
85 users.users.backup = { 97 users.users.backup = {
86 hashedPassword = "!"; 98 hashedPassword = "!";
87 isSystemUser = true; 99 isSystemUser = true;
@@ -118,7 +130,7 @@
118 after = [ "network.target" ]; 130 after = [ "network.target" ];
119 131
120 serviceConfig = { 132 serviceConfig = {
121 ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:22"; 133 ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:caldance:22";
122 }; 134 };
123 }; 135 };
124 136
@@ -170,7 +182,7 @@
170 recommendedGzipSettings = true; 182 recommendedGzipSettings = true;
171 recommendedProxySettings = true; 183 recommendedProxySettings = true;
172 upstreams = { 184 upstreams = {
173 caldance.servers."nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:3031" = {}; 185 caldance.servers."caldance:3031" = {};
174 }; 186 };
175 virtualHosts = { 187 virtualHosts = {
176 "dev.immae.eu" = { 188 "dev.immae.eu" = {
diff --git a/modules/private/system/dilion/vms.nix b/modules/private/system/dilion/vms.nix
new file mode 100644
index 0000000..8d5a57b
--- /dev/null
+++ b/modules/private/system/dilion/vms.nix
@@ -0,0 +1,146 @@
1# inspired from https://nixos.wiki/wiki/Virtualization_in_NixOS
2{ config, pkgs, lib, ... }@args:
3let
4 networks = {
5 immae = {
6 bridgeNumber = "1";
7 ipRange = "192.168.100";
8 };
9 };
10 guests = {
11 caldance = {
12 pool = "zfspool";
13 cpus = "1";
14 memory = "2";
15 network = "immae";
16 diskSize = "10GiB";
17 extraDevicesXML = ''
18 <filesystem type="mount">
19 <source dir="/var/lib/caldance"/>
20 <target dir="home"/>
21 </filesystem>
22 '';
23 };
24 buildbot = {
25 pool = "zfspool";
26 cpus = "1";
27 memory = "3";
28 network = "immae";
29 diskSize = "10GiB";
30 destroyVolumeOnExit = true;
31 preStart = ''
32 if ! ${pkgs.libvirt}/bin/virsh pool-info --pool niximages &> /dev/null; then
33 pool-create-as --name niximages --type dir --target /etc/libvirtd/base-images/
34 fi
35 if ! ${pkgs.libvirt}/bin/virsh pool-info --pool buildbot-disks &> /dev/null; then
36 mkdir -p /var/lib/libvirt/images/buildbot-disks
37 pool-create-as --name buildbot-disks --type dir --target /var/lib/libvirt/images/buildbot-disks
38 fi
39 '';
40 };
41 };
42 toImage = f: "${import ./vms/base_image.nix f (args // { myEnv = config.myEnv; })}/nixos.qcow2";
43in
44{
45 environment.etc."libvirtd/base-images/nixos.qcow2".source = toImage ./vms/base_configuration.nix;
46 environment.etc."libvirtd/base-images/buildbot.qcow2".source = toImage ./vms/buildbot_configuration.nix;
47 systemd.services = lib.mapAttrs' (name: guest: lib.nameValuePair "libvirtd-guest-${name}" {
48 after = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
49 requires = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
50 wantedBy = [ "multi-user.target" ];
51 serviceConfig = {
52 Type = "oneshot";
53 RemainAfterExit = "yes";
54 };
55 script =
56 let
57 xml = pkgs.writeText "libvirt-guest-${name}.xml"
58 ''
59 <domain type="kvm">
60 <name>${name}</name>
61 <uuid>UUID</uuid>
62 <memory unit="GiB">${guest.memory}</memory>
63 <vcpu>${guest.cpus}</vcpu>
64 <os>
65 <type arch="x86_64">hvm</type>
66 </os>
67 <devices>
68 <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
69 <disk type="volume">
70 <source pool="${guest.pool}" volume="guest-${name}" />
71 <target dev="vda" bus="virtio"/>
72 </disk>
73 ${guest.extraDevicesXML or ""}
74 <input type="keyboard" bus="usb"/>
75 <graphics type="vnc" port="-1" autoport="yes"/>
76 <interface type="network">
77 <source network="${guest.network}" />
78 </interface>
79 </devices>
80 <features>
81 <acpi/>
82 </features>
83 </domain>
84 '';
85 in
86 guest.preStart or "" + ''
87 if ! ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
88 ${pkgs.libvirt}/bin/virsh vol-create-as --pool ${guest.pool} --name 'guest-${name}' --capacity '${guest.diskSize}'
89 volume_path=$(${pkgs.libvirt}/bin/virsh vol-path --pool ${guest.pool} --vol 'guest-${name}')
90 ${pkgs.qemu}/bin/qemu-img convert /etc/libvirtd/base-images/nixos.qcow2 $volume_path
91 fi
92 uuid="$(${pkgs.libvirt}/bin/virsh domuuid '${name}' || true)"
93 ${pkgs.libvirt}/bin/virsh define <(sed "s/UUID/$uuid/" '${xml}')
94 ${pkgs.libvirt}/bin/virsh start '${name}'
95 '';
96 preStop = ''
97 ${pkgs.libvirt}/bin/virsh shutdown '${name}'
98 let "timeout = $(date +%s) + 10"
99 while [ "$(${pkgs.libvirt}/bin/virsh list --name | grep --count '^${name}$')" -gt 0 ]; do
100 if [ "$(date +%s)" -ge "$timeout" ]; then
101 # Meh, we warned it...
102 ${pkgs.libvirt}/bin/virsh destroy '${name}'
103 else
104 # The machine is still running, let's give it some time to shut down
105 sleep 0.5
106 fi
107 done
108 '' + lib.optionalString (guest.destroyVolumeOnExit or false) ''
109 if ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
110 ${pkgs.libvirt}/bin/virsh vol-wipe --pool ${guest.pool} --vol 'guest-${name}' || true
111 ${pkgs.libvirt}/bin/virsh vol-delete --pool ${guest.pool} --vol 'guest-${name}'
112 fi
113 '';
114 }) guests // (lib.mapAttrs' (name: network: lib.nameValuePair "libvirtd-network-${name}" {
115 after = [ "libvirtd.service" ];
116 requires = [ "libvirtd.service" ];
117 wantedBy = [ "multi-user.target" ];
118 serviceConfig = {
119 Type = "oneshot";
120 RemainAfterExit = "yes";
121 };
122 script = let
123 xml = pkgs.writeText "libvirt-network-${name}.xml" ''
124 <network>
125 <name>${name}</name>
126 <uuid>UUID</uuid>
127 <forward mode='nat' />
128 <bridge name='virbr${network.bridgeNumber}' />
129 <domain name='${name}' localOnly='yes'/>
130 <ip address='${network.ipRange}.1' netmask='255.255.255.0'>
131 <dhcp>
132 <range start='${network.ipRange}.2' end='${network.ipRange}.254'/>
133 </dhcp>
134 </ip>
135 </network>
136 '';
137 in ''
138 uuid="$(${pkgs.libvirt}/bin/virsh net-uuid '${name}' || true)"
139 ${pkgs.libvirt}/bin/virsh net-define <(sed "s/UUID/$uuid/" '${xml}')
140 ${pkgs.libvirt}/bin/virsh net-start '${name}'
141 '';
142 preStop = ''
143 ${pkgs.libvirt}/bin/virsh net-destroy '${name}'
144 '';
145 }) networks);
146}
diff --git a/modules/private/system/dilion/vms/base_configuration.nix b/modules/private/system/dilion/vms/base_configuration.nix
new file mode 100644
index 0000000..e2caba2
--- /dev/null
+++ b/modules/private/system/dilion/vms/base_configuration.nix
@@ -0,0 +1,21 @@
1{ lib, config, ... }@args:
2{
3 options.myEnv = (import ../../../environment.nix (args // { name = "dummy"; })).options.myEnv;
4 config = {
5 fileSystems."/".device = "/dev/disk/by-label/nixos";
6 boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "virtio_balloon" "virtio_blk" "virtio_pci" "virtio_ring" ];
7 boot.loader = {
8 grub = {
9 version = 2;
10 device = "/dev/vda";
11 };
12 timeout = 0;
13 };
14 services.openssh.enable = true;
15 networking.firewall.allowedTCPPorts = [ 22 ];
16 users = {
17 mutableUsers = false;
18 users.root.openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.immae_dilion ];
19 };
20 };
21}
diff --git a/modules/private/system/dilion/vms/base_image.nix b/modules/private/system/dilion/vms/base_image.nix
new file mode 100644
index 0000000..8de8560
--- /dev/null
+++ b/modules/private/system/dilion/vms/base_image.nix
@@ -0,0 +1,94 @@
1configuration_file: { pkgs ? import <nixpkgs> {}, system ? builtins.currentSystem, myEnv, ... }:
2let
3 config = (import <nixpkgs/nixos/lib/eval-config.nix> {
4 inherit system;
5 modules = [ {
6 myEnv = myEnv;
7 imports = [ configuration_file ];
8
9 # We want our template image to be as small as possible, but the deployed image should be able to be
10 # of any size. Hence we resize on the first boot.
11 systemd.services.resize-main-fs = {
12 wantedBy = [ "multi-user.target" ];
13 serviceConfig.Type = "oneshot";
14 script =
15 ''
16 # Resize main partition to fill whole disk
17 echo ", +" | ${pkgs.utillinux}/bin/sfdisk /dev/vda --no-reread -N 1
18 ${pkgs.parted}/bin/partprobe
19 # Resize filesystem
20 ${pkgs.e2fsprogs}/bin/resize2fs /dev/vda1
21 '';
22 };
23 } ];
24 }).config;
25in pkgs.vmTools.runInLinuxVM (
26 pkgs.runCommand "nixos-base-image"
27 {
28 memSize = 768;
29 preVM =
30 ''
31 mkdir $out
32 diskImage=image.qcow2
33 ${pkgs.vmTools.qemu}/bin/qemu-img create -f qcow2 $diskImage 2G
34 mv closure xchg/
35 '';
36 postVM =
37 ''
38 echo compressing VM image...
39 ${pkgs.vmTools.qemu}/bin/qemu-img convert -c $diskImage -O qcow2 $out/nixos.qcow2
40 '';
41 buildInputs = [ pkgs.utillinux pkgs.perl pkgs.parted pkgs.e2fsprogs ];
42 exportReferencesGraph =
43 [ "closure" config.system.build.toplevel ];
44 }
45 ''
46 # Create the partition
47 parted /dev/vda mklabel msdos
48 parted /dev/vda -- mkpart primary ext4 1M -1s
49
50 # Format the partition
51 mkfs.ext4 -L nixos /dev/vda1
52 mkdir /mnt
53 mount /dev/vda1 /mnt
54
55 for dir in dev proc sys; do
56 mkdir /mnt/$dir
57 mount --bind /$dir /mnt/$dir
58 done
59
60 storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure)
61 echo filling Nix store...
62 mkdir -p /mnt/nix/store
63 set -f
64 cp -prd $storePaths /mnt/nix/store
65 # The permissions will be set up incorrectly if the host machine is not running NixOS
66 chown -R 0:30000 /mnt/nix/store
67
68 mkdir -p /mnt/etc/nix
69 echo 'build-users-group = ' > /mnt/etc/nix/nix.conf
70
71 # Register the paths in the Nix database.
72 export USER=root
73 printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \
74 chroot /mnt ${config.nix.package.out}/bin/nix-store --load-db
75
76 # Create the system profile to allow nixos-rebuild to work.
77 chroot /mnt ${config.nix.package.out}/bin/nix-env \
78 -p /nix/var/nix/profiles/system --set ${config.system.build.toplevel}
79
80 # `nixos-rebuild' requires an /etc/NIXOS.
81 mkdir -p /mnt/etc/nixos
82 touch /mnt/etc/NIXOS
83
84 # `switch-to-configuration' requires a /bin/sh
85 mkdir -p /mnt/bin
86 ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh
87
88 # Generate the GRUB menu.
89 chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot
90
91 umount /mnt/{proc,dev,sys}
92 umount /mnt
93 ''
94)
diff --git a/modules/private/system/dilion/vms/buildbot_configuration.nix b/modules/private/system/dilion/vms/buildbot_configuration.nix
new file mode 100644
index 0000000..05b02d4
--- /dev/null
+++ b/modules/private/system/dilion/vms/buildbot_configuration.nix
@@ -0,0 +1,67 @@
1{ pkgs, config, lib, ... }:
2{
3 imports = [
4 <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
5 ./base_configuration.nix
6 ];
7 systemd.services.buildbot-worker.serviceConfig.ExecStartPre = let
8 cfg = config.services.buildbot-worker;
9 script = pkgs.writeScript "decode-dmi" ''
10 #!${pkgs.stdenv.shell}
11
12 mkdir -vp "${cfg.buildbotDir}"
13 varfile=${cfg.buildbotDir}/variables
14 rm $varfile || true
15 echo "[DEFAULT]" > $varfile
16 strings=$(${pkgs.dmidecode}/bin/dmidecode --oem-string count)
17 for i in $(seq 1 $strings); do
18 ${pkgs.dmidecode}/bin/dmidecode --oem-string $i >> $varfile
19 done
20 chown -R ${cfg.user}:${cfg.group} ${cfg.buildbotDir}
21 '';
22 in
23 lib.mkForce ["+${script}"];
24 systemd.services.buildbot-worker.serviceConfig.ExecStart = let
25 cfg = config.services.buildbot-worker;
26 tacFile = pkgs.writeText "buildbot-worker.tac" ''
27 import os
28 from io import open
29
30 from buildbot_worker.bot import Worker
31 from twisted.application import service
32
33 basedir = '${cfg.buildbotDir}'
34
35 # note: this line is matched against to check that this is a worker
36 # directory; do not edit it.
37 application = service.Application('buildbot-worker')
38
39 import configparser
40 config = config = configparser.ConfigParser()
41 config.read("${cfg.buildbotDir}/variables")
42 master_url_split = config["DEFAULT"]["buildbot_master_url"].split(':')
43 buildmaster_host = master_url_split[0]
44 port = int(master_url_split[1])
45 workername = config["DEFAULT"]["buildbot_worker_name"]
46
47 with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
48 passwd = passwd_file.read().strip('\r\n')
49 keepalive = ${toString cfg.keepalive}
50 umask = None
51 maxdelay = 300
52 numcpus = None
53 allow_shutdown = None
54
55 s = Worker(buildmaster_host, port, workername, passwd, basedir,
56 keepalive, umask=umask, maxdelay=maxdelay,
57 numcpus=numcpus, allow_shutdown=allow_shutdown)
58 s.setServiceParent(application)
59 '';
60 in
61 lib.mkForce "${cfg.package.pythonModule.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
62 services.buildbot-worker = {
63 enable = true;
64 workerPass = config.myEnv.buildbot.workerPassword;
65 packages = [ pkgs.git pkgs.gzip pkgs.openssh ];
66 };
67}