From 200690c9aecec1f38c1a62a65916df2950e1afe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Thu, 24 Jun 2021 22:24:15 +0200 Subject: First attempt at making declarative VMs In order to make buildbot more secure, the builds need to happen inside VMs so that they can be thrown out on demand when not needed. This commit implements this facility on dilion, and also defines declaratively some previous VMs which used to run on the machine. --- modules/private/system/dilion.nix | 18 ++- modules/private/system/dilion/vms.nix | 146 +++++++++++++++++++++ .../system/dilion/vms/base_configuration.nix | 21 +++ modules/private/system/dilion/vms/base_image.nix | 94 +++++++++++++ .../system/dilion/vms/buildbot_configuration.nix | 67 ++++++++++ 5 files changed, 343 insertions(+), 3 deletions(-) create mode 100644 modules/private/system/dilion/vms.nix create mode 100644 modules/private/system/dilion/vms/base_configuration.nix create mode 100644 modules/private/system/dilion/vms/base_image.nix create mode 100644 modules/private/system/dilion/vms/buildbot_configuration.nix (limited to 'modules/private/system') diff --git a/modules/private/system/dilion.nix b/modules/private/system/dilion.nix index be8269e..a59d607 100644 --- a/modules/private/system/dilion.nix +++ b/modules/private/system/dilion.nix @@ -76,12 +76,24 @@ }; myServices.ssh.modules = [ config.myServices.ssh.predefinedModules.regular ]; - imports = builtins.attrValues (import ../..); + imports = builtins.attrValues (import ../..) ++ [ ./dilion/vms.nix ]; system.nssModules = [ pkgs.libvirt ]; system.nssDatabases.hosts = lib.mkForce [ "files" "libvirt_guest" "mymachines" "dns" "myhostname" ]; programs.zsh.enable = true; + users.users.libvirt = { + hashedPassword = "!"; + shell = pkgs.bashInteractive; + isSystemUser = true; + group = "libvirtd"; + packages = [ pkgs.netcat-openbsd ]; + openssh.authorizedKeys.keyFiles = [ + "${privateFiles}/buildbot_ssh_key.pub" + ]; + openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.ismael_flony ]; + }; + users.users.backup = { hashedPassword = "!"; isSystemUser = true; @@ -118,7 +130,7 @@ after = [ "network.target" ]; serviceConfig = { - ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:22"; + ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:caldance:22"; }; }; @@ -170,7 +182,7 @@ recommendedGzipSettings = true; recommendedProxySettings = true; upstreams = { - caldance.servers."nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:3031" = {}; + caldance.servers."caldance:3031" = {}; }; virtualHosts = { "dev.immae.eu" = { diff --git a/modules/private/system/dilion/vms.nix b/modules/private/system/dilion/vms.nix new file mode 100644 index 0000000..8d5a57b --- /dev/null +++ b/modules/private/system/dilion/vms.nix @@ -0,0 +1,146 @@ +# inspired from https://nixos.wiki/wiki/Virtualization_in_NixOS +{ config, pkgs, lib, ... }@args: +let + networks = { + immae = { + bridgeNumber = "1"; + ipRange = "192.168.100"; + }; + }; + guests = { + caldance = { + pool = "zfspool"; + cpus = "1"; + memory = "2"; + network = "immae"; + diskSize = "10GiB"; + extraDevicesXML = '' + + + + + ''; + }; + buildbot = { + pool = "zfspool"; + cpus = "1"; + memory = "3"; + network = "immae"; + diskSize = "10GiB"; + destroyVolumeOnExit = true; + preStart = '' + if ! ${pkgs.libvirt}/bin/virsh pool-info --pool niximages &> /dev/null; then + pool-create-as --name niximages --type dir --target /etc/libvirtd/base-images/ + fi + if ! ${pkgs.libvirt}/bin/virsh pool-info --pool buildbot-disks &> /dev/null; then + mkdir -p /var/lib/libvirt/images/buildbot-disks + pool-create-as --name buildbot-disks --type dir --target /var/lib/libvirt/images/buildbot-disks + fi + ''; + }; + }; + toImage = f: "${import ./vms/base_image.nix f (args // { myEnv = config.myEnv; })}/nixos.qcow2"; +in +{ + environment.etc."libvirtd/base-images/nixos.qcow2".source = toImage ./vms/base_configuration.nix; + environment.etc."libvirtd/base-images/buildbot.qcow2".source = toImage ./vms/buildbot_configuration.nix; + systemd.services = lib.mapAttrs' (name: guest: lib.nameValuePair "libvirtd-guest-${name}" { + after = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ]; + requires = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = "yes"; + }; + script = + let + xml = pkgs.writeText "libvirt-guest-${name}.xml" + '' + + ${name} + UUID + ${guest.memory} + ${guest.cpus} + + hvm + + + /run/current-system/sw/bin/qemu-system-x86_64 + + + + + ${guest.extraDevicesXML or ""} + + + + + + + + + + + ''; + in + guest.preStart or "" + '' + if ! ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then + ${pkgs.libvirt}/bin/virsh vol-create-as --pool ${guest.pool} --name 'guest-${name}' --capacity '${guest.diskSize}' + volume_path=$(${pkgs.libvirt}/bin/virsh vol-path --pool ${guest.pool} --vol 'guest-${name}') + ${pkgs.qemu}/bin/qemu-img convert /etc/libvirtd/base-images/nixos.qcow2 $volume_path + fi + uuid="$(${pkgs.libvirt}/bin/virsh domuuid '${name}' || true)" + ${pkgs.libvirt}/bin/virsh define <(sed "s/UUID/$uuid/" '${xml}') + ${pkgs.libvirt}/bin/virsh start '${name}' + ''; + preStop = '' + ${pkgs.libvirt}/bin/virsh shutdown '${name}' + let "timeout = $(date +%s) + 10" + while [ "$(${pkgs.libvirt}/bin/virsh list --name | grep --count '^${name}$')" -gt 0 ]; do + if [ "$(date +%s)" -ge "$timeout" ]; then + # Meh, we warned it... + ${pkgs.libvirt}/bin/virsh destroy '${name}' + else + # The machine is still running, let's give it some time to shut down + sleep 0.5 + fi + done + '' + lib.optionalString (guest.destroyVolumeOnExit or false) '' + if ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then + ${pkgs.libvirt}/bin/virsh vol-wipe --pool ${guest.pool} --vol 'guest-${name}' || true + ${pkgs.libvirt}/bin/virsh vol-delete --pool ${guest.pool} --vol 'guest-${name}' + fi + ''; + }) guests // (lib.mapAttrs' (name: network: lib.nameValuePair "libvirtd-network-${name}" { + after = [ "libvirtd.service" ]; + requires = [ "libvirtd.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = "yes"; + }; + script = let + xml = pkgs.writeText "libvirt-network-${name}.xml" '' + + ${name} + UUID + + + + + + + + + + ''; + in '' + uuid="$(${pkgs.libvirt}/bin/virsh net-uuid '${name}' || true)" + ${pkgs.libvirt}/bin/virsh net-define <(sed "s/UUID/$uuid/" '${xml}') + ${pkgs.libvirt}/bin/virsh net-start '${name}' + ''; + preStop = '' + ${pkgs.libvirt}/bin/virsh net-destroy '${name}' + ''; + }) networks); +} diff --git a/modules/private/system/dilion/vms/base_configuration.nix b/modules/private/system/dilion/vms/base_configuration.nix new file mode 100644 index 0000000..e2caba2 --- /dev/null +++ b/modules/private/system/dilion/vms/base_configuration.nix @@ -0,0 +1,21 @@ +{ lib, config, ... }@args: +{ + options.myEnv = (import ../../../environment.nix (args // { name = "dummy"; })).options.myEnv; + config = { + fileSystems."/".device = "/dev/disk/by-label/nixos"; + boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "virtio_balloon" "virtio_blk" "virtio_pci" "virtio_ring" ]; + boot.loader = { + grub = { + version = 2; + device = "/dev/vda"; + }; + timeout = 0; + }; + services.openssh.enable = true; + networking.firewall.allowedTCPPorts = [ 22 ]; + users = { + mutableUsers = false; + users.root.openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.immae_dilion ]; + }; + }; +} diff --git a/modules/private/system/dilion/vms/base_image.nix b/modules/private/system/dilion/vms/base_image.nix new file mode 100644 index 0000000..8de8560 --- /dev/null +++ b/modules/private/system/dilion/vms/base_image.nix @@ -0,0 +1,94 @@ +configuration_file: { pkgs ? import {}, system ? builtins.currentSystem, myEnv, ... }: +let + config = (import { + inherit system; + modules = [ { + myEnv = myEnv; + imports = [ configuration_file ]; + + # We want our template image to be as small as possible, but the deployed image should be able to be + # of any size. Hence we resize on the first boot. + systemd.services.resize-main-fs = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.Type = "oneshot"; + script = + '' + # Resize main partition to fill whole disk + echo ", +" | ${pkgs.utillinux}/bin/sfdisk /dev/vda --no-reread -N 1 + ${pkgs.parted}/bin/partprobe + # Resize filesystem + ${pkgs.e2fsprogs}/bin/resize2fs /dev/vda1 + ''; + }; + } ]; + }).config; +in pkgs.vmTools.runInLinuxVM ( + pkgs.runCommand "nixos-base-image" + { + memSize = 768; + preVM = + '' + mkdir $out + diskImage=image.qcow2 + ${pkgs.vmTools.qemu}/bin/qemu-img create -f qcow2 $diskImage 2G + mv closure xchg/ + ''; + postVM = + '' + echo compressing VM image... + ${pkgs.vmTools.qemu}/bin/qemu-img convert -c $diskImage -O qcow2 $out/nixos.qcow2 + ''; + buildInputs = [ pkgs.utillinux pkgs.perl pkgs.parted pkgs.e2fsprogs ]; + exportReferencesGraph = + [ "closure" config.system.build.toplevel ]; + } + '' + # Create the partition + parted /dev/vda mklabel msdos + parted /dev/vda -- mkpart primary ext4 1M -1s + + # Format the partition + mkfs.ext4 -L nixos /dev/vda1 + mkdir /mnt + mount /dev/vda1 /mnt + + for dir in dev proc sys; do + mkdir /mnt/$dir + mount --bind /$dir /mnt/$dir + done + + storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure) + echo filling Nix store... + mkdir -p /mnt/nix/store + set -f + cp -prd $storePaths /mnt/nix/store + # The permissions will be set up incorrectly if the host machine is not running NixOS + chown -R 0:30000 /mnt/nix/store + + mkdir -p /mnt/etc/nix + echo 'build-users-group = ' > /mnt/etc/nix/nix.conf + + # Register the paths in the Nix database. + export USER=root + printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \ + chroot /mnt ${config.nix.package.out}/bin/nix-store --load-db + + # Create the system profile to allow nixos-rebuild to work. + chroot /mnt ${config.nix.package.out}/bin/nix-env \ + -p /nix/var/nix/profiles/system --set ${config.system.build.toplevel} + + # `nixos-rebuild' requires an /etc/NIXOS. + mkdir -p /mnt/etc/nixos + touch /mnt/etc/NIXOS + + # `switch-to-configuration' requires a /bin/sh + mkdir -p /mnt/bin + ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh + + # Generate the GRUB menu. + chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot + + umount /mnt/{proc,dev,sys} + umount /mnt + '' +) diff --git a/modules/private/system/dilion/vms/buildbot_configuration.nix b/modules/private/system/dilion/vms/buildbot_configuration.nix new file mode 100644 index 0000000..05b02d4 --- /dev/null +++ b/modules/private/system/dilion/vms/buildbot_configuration.nix @@ -0,0 +1,67 @@ +{ pkgs, config, lib, ... }: +{ + imports = [ + + ./base_configuration.nix + ]; + systemd.services.buildbot-worker.serviceConfig.ExecStartPre = let + cfg = config.services.buildbot-worker; + script = pkgs.writeScript "decode-dmi" '' + #!${pkgs.stdenv.shell} + + mkdir -vp "${cfg.buildbotDir}" + varfile=${cfg.buildbotDir}/variables + rm $varfile || true + echo "[DEFAULT]" > $varfile + strings=$(${pkgs.dmidecode}/bin/dmidecode --oem-string count) + for i in $(seq 1 $strings); do + ${pkgs.dmidecode}/bin/dmidecode --oem-string $i >> $varfile + done + chown -R ${cfg.user}:${cfg.group} ${cfg.buildbotDir} + ''; + in + lib.mkForce ["+${script}"]; + systemd.services.buildbot-worker.serviceConfig.ExecStart = let + cfg = config.services.buildbot-worker; + tacFile = pkgs.writeText "buildbot-worker.tac" '' + import os + from io import open + + from buildbot_worker.bot import Worker + from twisted.application import service + + basedir = '${cfg.buildbotDir}' + + # note: this line is matched against to check that this is a worker + # directory; do not edit it. + application = service.Application('buildbot-worker') + + import configparser + config = config = configparser.ConfigParser() + config.read("${cfg.buildbotDir}/variables") + master_url_split = config["DEFAULT"]["buildbot_master_url"].split(':') + buildmaster_host = master_url_split[0] + port = int(master_url_split[1]) + workername = config["DEFAULT"]["buildbot_worker_name"] + + with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file: + passwd = passwd_file.read().strip('\r\n') + keepalive = ${toString cfg.keepalive} + umask = None + maxdelay = 300 + numcpus = None + allow_shutdown = None + + s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + numcpus=numcpus, allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; + in + lib.mkForce "${cfg.package.pythonModule.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}"; + services.buildbot-worker = { + enable = true; + workerPass = config.myEnv.buildbot.workerPassword; + packages = [ pkgs.git pkgs.gzip pkgs.openssh ]; + }; +} -- cgit v1.2.3