From 200690c9aecec1f38c1a62a65916df2950e1afe7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Thu, 24 Jun 2021 22:24:15 +0200 Subject: [PATCH] First attempt at making declarative VMs In order to make buildbot more secure, the builds need to happen inside VMs so that they can be thrown out on demand when not needed. This commit implements this facility on dilion, and also defines declaratively some previous VMs which used to run on the machine. --- DOCUMENTATION.md | 2 +- environments/immae-eu.nix | 2 +- modules/private/buildbot/common/libvirt.py | 306 ++++++++++++++++++ modules/private/buildbot/default.nix | 18 +- .../buildbot/projects/test/__init__.py | 19 +- modules/private/environment.nix | 3 + modules/private/gitolite/default.nix | 2 +- modules/private/system/dilion.nix | 18 +- modules/private/system/dilion/vms.nix | 146 +++++++++ .../system/dilion/vms/base_configuration.nix | 21 ++ .../private/system/dilion/vms/base_image.nix | 94 ++++++ .../dilion/vms/buildbot_configuration.nix | 67 ++++ nixops/secrets | 2 +- 13 files changed, 687 insertions(+), 13 deletions(-) create mode 100644 modules/private/buildbot/common/libvirt.py create mode 100644 modules/private/system/dilion/vms.nix create mode 100644 modules/private/system/dilion/vms/base_configuration.nix create mode 100644 modules/private/system/dilion/vms/base_image.nix create mode 100644 modules/private/system/dilion/vms/buildbot_configuration.nix diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 1697299..50eeca4 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -180,7 +180,7 @@ Things to look at during upgrades: Upgrade to latest unstable ------------------- -- Nothing in particular yet +- Weechat: https://specs.weechat.org/specs/001285-follow-xdg-base-dir-spec.html Etherpad-lite ------------- diff --git a/environments/immae-eu.nix b/environments/immae-eu.nix index ca9e5b5..df57e55 100644 --- a/environments/immae-eu.nix +++ b/environments/immae-eu.nix @@ -123,7 +123,7 @@ let sshfs ncdu procps-ng # other tools - pgloader s3cmd lftp jq cpulimit libxslt gandi-cli + pgloader s3cmd lftp jq cpulimit libxslt gandi-cli bubblewrap # Terraform + AWS terraform_0_12 awscli diff --git a/modules/private/buildbot/common/libvirt.py b/modules/private/buildbot/common/libvirt.py new file mode 100644 index 0000000..85fd908 --- /dev/null +++ b/modules/private/buildbot/common/libvirt.py @@ -0,0 +1,306 @@ +# This file was part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Portions Copyright Buildbot Team Members +# Portions Copyright 2010 Isotoma Limited + + +import os + +from twisted.internet import defer +from twisted.internet import threads +from twisted.internet import utils +from twisted.python import failure +from twisted.python import log + +from buildbot import config +from buildbot.util.eventual import eventually +from buildbot.worker import AbstractLatentWorker + +try: + import libvirt +except ImportError: + libvirt = None + +import random +import string + +def random_string_generator(): + chars = string.ascii_letters + return ''.join(random.choice(chars) for x in range(6)) + +class WorkQueue: + + """ + I am a class that turns parallel access into serial access. + + I exist because we want to run libvirt access in threads as we don't + trust calls not to block, but under load libvirt doesn't seem to like + this kind of threaded use. + """ + + def __init__(self): + self.queue = [] + + def _process(self): + log.msg("Looking to start a piece of work now...") + + # Is there anything to do? + if not self.queue: + log.msg("_process called when there is no work") + return + + # Peek at the top of the stack - get a function to call and + # a deferred to fire when its all over + d, next_operation, args, kwargs = self.queue[0] + + # Start doing some work - expects a deferred + try: + d2 = next_operation(*args, **kwargs) + except Exception: + d2 = defer.fail() + + # Whenever a piece of work is done, whether it worked or not + # call this to schedule the next piece of work + @d2.addBoth + def _work_done(res): + log.msg("Completed a piece of work") + self.queue.pop(0) + if self.queue: + log.msg("Preparing next piece of work") + eventually(self._process) + return res + + # When the work is done, trigger d + d2.chainDeferred(d) + + def execute(self, cb, *args, **kwargs): + kickstart_processing = not self.queue + d = defer.Deferred() + self.queue.append((d, cb, args, kwargs)) + if kickstart_processing: + self._process() + return d + + def executeInThread(self, cb, *args, **kwargs): + return self.execute(threads.deferToThread, cb, *args, **kwargs) + + +# A module is effectively a singleton class, so this is OK +queue = WorkQueue() + + +class Domain: + + """ + I am a wrapper around a libvirt Domain object + """ + + def __init__(self, connection, domain): + self.connection = connection + self.domain = domain + + def name(self): + return queue.executeInThread(self.domain.name) + + def create(self): + return queue.executeInThread(self.domain.create) + + def shutdown(self): + return queue.executeInThread(self.domain.shutdown) + + def destroy(self): + return queue.executeInThread(self.domain.destroy) + +class Volume: + def __init__(self, connection, volume): + self.connection = connection + self.volume = volume + + @defer.inlineCallbacks + def destroy(self): + yield queue.executeInThread(self.volume.wipe) + yield queue.executeInThread(self.volume.delete) + +class Pool: + VolumeClass = Volume + def __init__(self, connection, pool): + self.connection = connection + self.pool = pool + + @defer.inlineCallbacks + def create_volume(self, xml): + res = yield queue.executeInThread(self.pool.createXML, xml) + return self.VolumeClass(self.connection, res) + +class Connection: + + """ + I am a wrapper around a libvirt Connection object. + """ + + DomainClass = Domain + PoolClass = Pool + + def __init__(self, uri): + self.uri = uri + self.connection = libvirt.open(uri) + + @defer.inlineCallbacks + def create(self, xml): + """ I take libvirt XML and start a new VM """ + res = yield queue.executeInThread(self.connection.createXML, xml, 0) + return self.DomainClass(self, res) + + @defer.inlineCallbacks + def lookup_pool(self, name): + res = yield queue.executeInThread(self.connection.storagePoolLookupByName, name) + return self.PoolClass(self, res) + +class LibVirtWorker(AbstractLatentWorker): + + def __init__(self, name, password, connection, master_url, base_image=None, **kwargs): + super().__init__(name, password, **kwargs) + if not libvirt: + config.error( + "The python module 'libvirt' is needed to use a LibVirtWorker") + + self.master_url = master_url + self.random_name = random_string_generator() + self.connection = connection + self.base_image = base_image + + self.domain = None + self.domain_name = "buildbot-" + self.workername + "-" + self.random_name + self.volume = None + self.volume_name = "buildbot-" + self.workername + "-" + self.random_name + self.pool_name = "buildbot-disks" + + def reconfigService(self, *args, **kwargs): + if 'build_wait_timeout' not in kwargs: + kwargs['build_wait_timeout'] = 0 + return super().reconfigService(*args, **kwargs) + + def canStartBuild(self): + if self.domain and not self.isConnected(): + log.msg( + "Not accepting builds as existing domain but worker not connected") + return False + + return super().canStartBuild() + + @defer.inlineCallbacks + def _prepare_image(self): + log.msg("Creating temporary image {}".format(self.volume_name)) + pool = yield self.connection.lookup_pool(self.pool_name) + vol_xml = """ + + {vol_name} + 10 + + + + 0600 + 0 + 0 + + + + /etc/libvirtd/base-images/buildbot.qcow2 + + + + """.format(vol_name = self.volume_name) + self.volume = yield pool.create_volume(vol_xml) + + @defer.inlineCallbacks + def start_instance(self, build): + """ + I start a new instance of a VM. + + If a base_image is specified, I will make a clone of that otherwise i will + use image directly. + + If i'm not given libvirt domain definition XML, I will look for my name + in the list of defined virtual machines and start that. + """ + domain_xml = """ + + {domain_name} + 2 + 1 + + + buildbot_master_url={master_url} + buildbot_worker_name={worker_name} + + + + hvm + + + + /run/current-system/sw/bin/qemu-system-x86_64 + + + + + + + + + + + + + + + + + """.format(volume_name = self.volume_name, master_url = self.master_url, pool_name = + self.pool_name, domain_name = self.domain_name, worker_name = self.workername) + + yield self._prepare_image() + + try: + self.domain = yield self.connection.create(domain_xml) + except Exception: + log.err(failure.Failure(), + ("Cannot start a VM ({}), failing gracefully and triggering" + "a new build check").format(self.workername)) + self.domain = None + return False + + return [self.domain_name] + + def stop_instance(self, fast=False): + """ + I attempt to stop a running VM. + I make sure any connection to the worker is removed. + If the VM was using a cloned image, I remove the clone + When everything is tidied up, I ask that bbot looks for work to do + """ + + log.msg("Attempting to stop '{}'".format(self.workername)) + if self.domain is None: + log.msg("I don't think that domain is even running, aborting") + return defer.succeed(None) + + domain = self.domain + self.domain = None + + d = domain.destroy() + if self.volume is not None: + self.volume.destroy() + + return d diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix index d6753e5..ac34845 100644 --- a/modules/private/buildbot/default.nix +++ b/modules/private/buildbot/default.nix @@ -107,7 +107,12 @@ in project_env = with lib.attrsets; mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment // mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) // - { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; }; + { + BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; + BUILDBOT_WORKER_PORT = builtins.toString project.workerPort; + BUILDBOT_HOST = config.hostEnv.fqdn; + BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system"; + }; in builtins.concatStringsSep "\n" (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env); } @@ -122,6 +127,13 @@ in text = config.myEnv.buildbot.ldap.password; dest = "buildbot/ldap"; } + { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = config.myEnv.buildbot.workerPassword; + dest = "buildbot/worker_password"; + } { permissions = "0600"; user = "buildbot"; @@ -135,6 +147,7 @@ in restart = true; paths = [ "/var/secrets/buildbot/ldap" + "/var/secrets/buildbot/worker_password" "/var/secrets/buildbot/ssh_key" "/var/secrets/buildbot/${project.name}/environment_file" ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets; @@ -144,6 +157,7 @@ in description = "buildbot slice"; }; + networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects; systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { description = "Buildbot Continuous Integration Server ${project.name}."; after = [ "network-online.target" ]; @@ -196,6 +210,7 @@ in buildbot_secrets=${varDir}/${project.name}/secrets install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap + install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/worker_password $buildbot_secrets/worker_password ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets )} @@ -213,6 +228,7 @@ in }); HOME = "${varDir}/${project.name}"; PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [ + pkgs.python3Packages.libvirt pkgs.python3Packages.wokkel pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot pkgs.python3Packages.buildbot-worker diff --git a/modules/private/buildbot/projects/test/__init__.py b/modules/private/buildbot/projects/test/__init__.py index e6b8d51..e2f6f82 100644 --- a/modules/private/buildbot/projects/test/__init__.py +++ b/modules/private/buildbot/projects/test/__init__.py @@ -1,5 +1,6 @@ from buildbot.plugins import * from buildbot_common.build_helpers import * +import buildbot_common.libvirt as ilibvirt import os from buildbot.util import bytes2unicode import json @@ -10,11 +11,13 @@ class E(): PROJECT = "test" BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) - PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) + PB_SOCKET = os.environ["BUILDBOT_WORKER_PORT"] + WORKER_HOST = "{}:{}".format(os.environ["BUILDBOT_HOST"], PB_SOCKET) RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT) RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT) GIT_URL = "https://git.immae.eu/perso/Immae/TestProject.git" SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" + LIBVIRT_URL = os.environ["BUILDBOT_VIRT_URL"] + "?keyfile=" + SSH_KEY_PATH PUPPET_HOST = "root@backup-1.v.immae.eu" LDAP_HOST = "ldap.immae.eu" LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" @@ -70,8 +73,14 @@ def configure(c): c["www"]["change_hook_dialects"]["base"] = { "custom_class": CustomBase } - c['workers'].append(worker.LocalWorker("generic-worker-test")) - c['workers'].append(worker.LocalWorker("deploy-worker-test")) + c['workers'].append(ilibvirt.LibVirtWorker("test-build", + open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(), + ilibvirt.Connection(E.LIBVIRT_URL), + E.WORKER_HOST)) + c['workers'].append(ilibvirt.LibVirtWorker("test-deploy", + open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(), + ilibvirt.Connection(E.LIBVIRT_URL), + E.WORKER_HOST)) c['schedulers'].append(hook_scheduler("TestProject", timer=1)) c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"])) @@ -109,7 +118,7 @@ def factory(): logEnviron=False, command=["echo", package])) factory.addSteps(package_and_upload(package, package_dest, package_url)) - return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory) + return util.BuilderConfig(name="TestProject_build", workernames=["test-build"], factory=factory) def compute_build_infos(): @@ -143,7 +152,7 @@ def deploy_factory(): ldap_password=util.Secret("ldap"))) factory.addStep(steps.MasterShellCommand(command=[ "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host])) - return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory) + return util.BuilderConfig(name="TestProject_deploy", workernames=["test-deploy"], factory=factory) from twisted.internet import defer from buildbot.process.buildstep import FAILURE diff --git a/modules/private/environment.nix b/modules/private/environment.nix index 719bf8f..f0af572 100644 --- a/modules/private/environment.nix +++ b/modules/private/environment.nix @@ -228,6 +228,7 @@ in ''; type = submodule { options = { + rootKeys = mkOption { type = attrsOf str; description = "Keys of root users"; }; ldap = mkOption { description = '' LDAP credentials for cn=ssh,ou=services,dc=immae,dc=eu dn @@ -804,6 +805,7 @@ in description = "Buildbot configuration"; type = submodule { options = { + workerPassword = mkOption { description = "Buildbot worker password"; type = str; }; user = mkOption { description = "Buildbot user"; type = submodule { @@ -855,6 +857,7 @@ in ''; }; pythonPathHome = mkOption { type = bool; description = "Whether to add project’s python home to python path"; }; + workerPort = mkOption { type = port; description = "Port for the worker"; }; secrets = mkOption { type = attrsOf str; description = "Secrets for the project to dump as files"; diff --git a/modules/private/gitolite/default.nix b/modules/private/gitolite/default.nix index 6b573e3..e54ee8a 100644 --- a/modules/private/gitolite/default.nix +++ b/modules/private/gitolite/default.nix @@ -74,7 +74,7 @@ in { # Installation: https://git.immae.eu/mantisbt/view.php?id=93 services.gitolite = { enable = true; - adminPubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXqRbiHw7QoHADNIEuo4nUT9fSOIEBMdJZH0bkQAxXyJFyCM1IMz0pxsHV0wu9tdkkr36bPEUj2aV5bkYLBN6nxcV2Y49X8bjOSCPfx3n6Own1h+NeZVBj4ZByrFmqCbTxUJIZ2bZKcWOFncML39VmWdsVhNjg0X4NBBehqXRIKr2gt3E/ESAxTYJFm0BnU0baciw9cN0bsRGqvFgf5h2P48CIAfwhVcGmPQnnAwabnosYQzRWxR0OygH5Kd8mePh6FheIRIigfXsDO8f/jdxwut8buvNIf3m5EBr3tUbTsvM+eV3M5vKGt7sk8T64DVtepTSdOOWtp+47ktsnHOMh immae@immae.eu"; + adminPubkey = config.myEnv.sshd.rootKeys.immae_dilion; }; }; } diff --git a/modules/private/system/dilion.nix b/modules/private/system/dilion.nix index be8269e..a59d607 100644 --- a/modules/private/system/dilion.nix +++ b/modules/private/system/dilion.nix @@ -76,12 +76,24 @@ }; myServices.ssh.modules = [ config.myServices.ssh.predefinedModules.regular ]; - imports = builtins.attrValues (import ../..); + imports = builtins.attrValues (import ../..) ++ [ ./dilion/vms.nix ]; system.nssModules = [ pkgs.libvirt ]; system.nssDatabases.hosts = lib.mkForce [ "files" "libvirt_guest" "mymachines" "dns" "myhostname" ]; programs.zsh.enable = true; + users.users.libvirt = { + hashedPassword = "!"; + shell = pkgs.bashInteractive; + isSystemUser = true; + group = "libvirtd"; + packages = [ pkgs.netcat-openbsd ]; + openssh.authorizedKeys.keyFiles = [ + "${privateFiles}/buildbot_ssh_key.pub" + ]; + openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.ismael_flony ]; + }; + users.users.backup = { hashedPassword = "!"; isSystemUser = true; @@ -118,7 +130,7 @@ after = [ "network.target" ]; serviceConfig = { - ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:22"; + ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:caldance:22"; }; }; @@ -170,7 +182,7 @@ recommendedGzipSettings = true; recommendedProxySettings = true; upstreams = { - caldance.servers."nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:3031" = {}; + caldance.servers."caldance:3031" = {}; }; virtualHosts = { "dev.immae.eu" = { diff --git a/modules/private/system/dilion/vms.nix b/modules/private/system/dilion/vms.nix new file mode 100644 index 0000000..8d5a57b --- /dev/null +++ b/modules/private/system/dilion/vms.nix @@ -0,0 +1,146 @@ +# inspired from https://nixos.wiki/wiki/Virtualization_in_NixOS +{ config, pkgs, lib, ... }@args: +let + networks = { + immae = { + bridgeNumber = "1"; + ipRange = "192.168.100"; + }; + }; + guests = { + caldance = { + pool = "zfspool"; + cpus = "1"; + memory = "2"; + network = "immae"; + diskSize = "10GiB"; + extraDevicesXML = '' + + + + + ''; + }; + buildbot = { + pool = "zfspool"; + cpus = "1"; + memory = "3"; + network = "immae"; + diskSize = "10GiB"; + destroyVolumeOnExit = true; + preStart = '' + if ! ${pkgs.libvirt}/bin/virsh pool-info --pool niximages &> /dev/null; then + pool-create-as --name niximages --type dir --target /etc/libvirtd/base-images/ + fi + if ! ${pkgs.libvirt}/bin/virsh pool-info --pool buildbot-disks &> /dev/null; then + mkdir -p /var/lib/libvirt/images/buildbot-disks + pool-create-as --name buildbot-disks --type dir --target /var/lib/libvirt/images/buildbot-disks + fi + ''; + }; + }; + toImage = f: "${import ./vms/base_image.nix f (args // { myEnv = config.myEnv; })}/nixos.qcow2"; +in +{ + environment.etc."libvirtd/base-images/nixos.qcow2".source = toImage ./vms/base_configuration.nix; + environment.etc."libvirtd/base-images/buildbot.qcow2".source = toImage ./vms/buildbot_configuration.nix; + systemd.services = lib.mapAttrs' (name: guest: lib.nameValuePair "libvirtd-guest-${name}" { + after = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ]; + requires = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = "yes"; + }; + script = + let + xml = pkgs.writeText "libvirt-guest-${name}.xml" + '' + + ${name} + UUID + ${guest.memory} + ${guest.cpus} + + hvm + + + /run/current-system/sw/bin/qemu-system-x86_64 + + + + + ${guest.extraDevicesXML or ""} + + + + + + + + + + + ''; + in + guest.preStart or "" + '' + if ! ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then + ${pkgs.libvirt}/bin/virsh vol-create-as --pool ${guest.pool} --name 'guest-${name}' --capacity '${guest.diskSize}' + volume_path=$(${pkgs.libvirt}/bin/virsh vol-path --pool ${guest.pool} --vol 'guest-${name}') + ${pkgs.qemu}/bin/qemu-img convert /etc/libvirtd/base-images/nixos.qcow2 $volume_path + fi + uuid="$(${pkgs.libvirt}/bin/virsh domuuid '${name}' || true)" + ${pkgs.libvirt}/bin/virsh define <(sed "s/UUID/$uuid/" '${xml}') + ${pkgs.libvirt}/bin/virsh start '${name}' + ''; + preStop = '' + ${pkgs.libvirt}/bin/virsh shutdown '${name}' + let "timeout = $(date +%s) + 10" + while [ "$(${pkgs.libvirt}/bin/virsh list --name | grep --count '^${name}$')" -gt 0 ]; do + if [ "$(date +%s)" -ge "$timeout" ]; then + # Meh, we warned it... + ${pkgs.libvirt}/bin/virsh destroy '${name}' + else + # The machine is still running, let's give it some time to shut down + sleep 0.5 + fi + done + '' + lib.optionalString (guest.destroyVolumeOnExit or false) '' + if ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then + ${pkgs.libvirt}/bin/virsh vol-wipe --pool ${guest.pool} --vol 'guest-${name}' || true + ${pkgs.libvirt}/bin/virsh vol-delete --pool ${guest.pool} --vol 'guest-${name}' + fi + ''; + }) guests // (lib.mapAttrs' (name: network: lib.nameValuePair "libvirtd-network-${name}" { + after = [ "libvirtd.service" ]; + requires = [ "libvirtd.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = "yes"; + }; + script = let + xml = pkgs.writeText "libvirt-network-${name}.xml" '' + + ${name} + UUID + + + + + + + + + + ''; + in '' + uuid="$(${pkgs.libvirt}/bin/virsh net-uuid '${name}' || true)" + ${pkgs.libvirt}/bin/virsh net-define <(sed "s/UUID/$uuid/" '${xml}') + ${pkgs.libvirt}/bin/virsh net-start '${name}' + ''; + preStop = '' + ${pkgs.libvirt}/bin/virsh net-destroy '${name}' + ''; + }) networks); +} diff --git a/modules/private/system/dilion/vms/base_configuration.nix b/modules/private/system/dilion/vms/base_configuration.nix new file mode 100644 index 0000000..e2caba2 --- /dev/null +++ b/modules/private/system/dilion/vms/base_configuration.nix @@ -0,0 +1,21 @@ +{ lib, config, ... }@args: +{ + options.myEnv = (import ../../../environment.nix (args // { name = "dummy"; })).options.myEnv; + config = { + fileSystems."/".device = "/dev/disk/by-label/nixos"; + boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "virtio_balloon" "virtio_blk" "virtio_pci" "virtio_ring" ]; + boot.loader = { + grub = { + version = 2; + device = "/dev/vda"; + }; + timeout = 0; + }; + services.openssh.enable = true; + networking.firewall.allowedTCPPorts = [ 22 ]; + users = { + mutableUsers = false; + users.root.openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.immae_dilion ]; + }; + }; +} diff --git a/modules/private/system/dilion/vms/base_image.nix b/modules/private/system/dilion/vms/base_image.nix new file mode 100644 index 0000000..8de8560 --- /dev/null +++ b/modules/private/system/dilion/vms/base_image.nix @@ -0,0 +1,94 @@ +configuration_file: { pkgs ? import {}, system ? builtins.currentSystem, myEnv, ... }: +let + config = (import { + inherit system; + modules = [ { + myEnv = myEnv; + imports = [ configuration_file ]; + + # We want our template image to be as small as possible, but the deployed image should be able to be + # of any size. Hence we resize on the first boot. + systemd.services.resize-main-fs = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.Type = "oneshot"; + script = + '' + # Resize main partition to fill whole disk + echo ", +" | ${pkgs.utillinux}/bin/sfdisk /dev/vda --no-reread -N 1 + ${pkgs.parted}/bin/partprobe + # Resize filesystem + ${pkgs.e2fsprogs}/bin/resize2fs /dev/vda1 + ''; + }; + } ]; + }).config; +in pkgs.vmTools.runInLinuxVM ( + pkgs.runCommand "nixos-base-image" + { + memSize = 768; + preVM = + '' + mkdir $out + diskImage=image.qcow2 + ${pkgs.vmTools.qemu}/bin/qemu-img create -f qcow2 $diskImage 2G + mv closure xchg/ + ''; + postVM = + '' + echo compressing VM image... + ${pkgs.vmTools.qemu}/bin/qemu-img convert -c $diskImage -O qcow2 $out/nixos.qcow2 + ''; + buildInputs = [ pkgs.utillinux pkgs.perl pkgs.parted pkgs.e2fsprogs ]; + exportReferencesGraph = + [ "closure" config.system.build.toplevel ]; + } + '' + # Create the partition + parted /dev/vda mklabel msdos + parted /dev/vda -- mkpart primary ext4 1M -1s + + # Format the partition + mkfs.ext4 -L nixos /dev/vda1 + mkdir /mnt + mount /dev/vda1 /mnt + + for dir in dev proc sys; do + mkdir /mnt/$dir + mount --bind /$dir /mnt/$dir + done + + storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure) + echo filling Nix store... + mkdir -p /mnt/nix/store + set -f + cp -prd $storePaths /mnt/nix/store + # The permissions will be set up incorrectly if the host machine is not running NixOS + chown -R 0:30000 /mnt/nix/store + + mkdir -p /mnt/etc/nix + echo 'build-users-group = ' > /mnt/etc/nix/nix.conf + + # Register the paths in the Nix database. + export USER=root + printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \ + chroot /mnt ${config.nix.package.out}/bin/nix-store --load-db + + # Create the system profile to allow nixos-rebuild to work. + chroot /mnt ${config.nix.package.out}/bin/nix-env \ + -p /nix/var/nix/profiles/system --set ${config.system.build.toplevel} + + # `nixos-rebuild' requires an /etc/NIXOS. + mkdir -p /mnt/etc/nixos + touch /mnt/etc/NIXOS + + # `switch-to-configuration' requires a /bin/sh + mkdir -p /mnt/bin + ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh + + # Generate the GRUB menu. + chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot + + umount /mnt/{proc,dev,sys} + umount /mnt + '' +) diff --git a/modules/private/system/dilion/vms/buildbot_configuration.nix b/modules/private/system/dilion/vms/buildbot_configuration.nix new file mode 100644 index 0000000..05b02d4 --- /dev/null +++ b/modules/private/system/dilion/vms/buildbot_configuration.nix @@ -0,0 +1,67 @@ +{ pkgs, config, lib, ... }: +{ + imports = [ + + ./base_configuration.nix + ]; + systemd.services.buildbot-worker.serviceConfig.ExecStartPre = let + cfg = config.services.buildbot-worker; + script = pkgs.writeScript "decode-dmi" '' + #!${pkgs.stdenv.shell} + + mkdir -vp "${cfg.buildbotDir}" + varfile=${cfg.buildbotDir}/variables + rm $varfile || true + echo "[DEFAULT]" > $varfile + strings=$(${pkgs.dmidecode}/bin/dmidecode --oem-string count) + for i in $(seq 1 $strings); do + ${pkgs.dmidecode}/bin/dmidecode --oem-string $i >> $varfile + done + chown -R ${cfg.user}:${cfg.group} ${cfg.buildbotDir} + ''; + in + lib.mkForce ["+${script}"]; + systemd.services.buildbot-worker.serviceConfig.ExecStart = let + cfg = config.services.buildbot-worker; + tacFile = pkgs.writeText "buildbot-worker.tac" '' + import os + from io import open + + from buildbot_worker.bot import Worker + from twisted.application import service + + basedir = '${cfg.buildbotDir}' + + # note: this line is matched against to check that this is a worker + # directory; do not edit it. + application = service.Application('buildbot-worker') + + import configparser + config = config = configparser.ConfigParser() + config.read("${cfg.buildbotDir}/variables") + master_url_split = config["DEFAULT"]["buildbot_master_url"].split(':') + buildmaster_host = master_url_split[0] + port = int(master_url_split[1]) + workername = config["DEFAULT"]["buildbot_worker_name"] + + with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file: + passwd = passwd_file.read().strip('\r\n') + keepalive = ${toString cfg.keepalive} + umask = None + maxdelay = 300 + numcpus = None + allow_shutdown = None + + s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + numcpus=numcpus, allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; + in + lib.mkForce "${cfg.package.pythonModule.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}"; + services.buildbot-worker = { + enable = true; + workerPass = config.myEnv.buildbot.workerPassword; + packages = [ pkgs.git pkgs.gzip pkgs.openssh ]; + }; +} diff --git a/nixops/secrets b/nixops/secrets index 9ac6c14..9326233 160000 --- a/nixops/secrets +++ b/nixops/secrets @@ -1 +1 @@ -Subproject commit 9ac6c1459d2eeb24be0a991f745b567f0fcb0cca +Subproject commit 932623303fd674faf8106e03ea0b89195edfdc02 -- 2.41.0