Upgrade to latest unstable
-------------------
-- Nothing in particular yet
+- Weechat: https://specs.weechat.org/specs/001285-follow-xdg-base-dir-spec.html
Etherpad-lite
-------------
sshfs ncdu procps-ng
# other tools
- pgloader s3cmd lftp jq cpulimit libxslt gandi-cli
+ pgloader s3cmd lftp jq cpulimit libxslt gandi-cli bubblewrap
# Terraform + AWS
terraform_0_12 awscli
--- /dev/null
+# This file was part of Buildbot. Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Portions Copyright Buildbot Team Members
+# Portions Copyright 2010 Isotoma Limited
+
+
+import os
+
+from twisted.internet import defer
+from twisted.internet import threads
+from twisted.internet import utils
+from twisted.python import failure
+from twisted.python import log
+
+from buildbot import config
+from buildbot.util.eventual import eventually
+from buildbot.worker import AbstractLatentWorker
+
+try:
+ import libvirt
+except ImportError:
+ libvirt = None
+
+import random
+import string
+
+def random_string_generator():
+ chars = string.ascii_letters
+ return ''.join(random.choice(chars) for x in range(6))
+
+class WorkQueue:
+
+ """
+ I am a class that turns parallel access into serial access.
+
+ I exist because we want to run libvirt access in threads as we don't
+ trust calls not to block, but under load libvirt doesn't seem to like
+ this kind of threaded use.
+ """
+
+ def __init__(self):
+ self.queue = []
+
+ def _process(self):
+ log.msg("Looking to start a piece of work now...")
+
+ # Is there anything to do?
+ if not self.queue:
+ log.msg("_process called when there is no work")
+ return
+
+ # Peek at the top of the stack - get a function to call and
+ # a deferred to fire when its all over
+ d, next_operation, args, kwargs = self.queue[0]
+
+ # Start doing some work - expects a deferred
+ try:
+ d2 = next_operation(*args, **kwargs)
+ except Exception:
+ d2 = defer.fail()
+
+ # Whenever a piece of work is done, whether it worked or not
+ # call this to schedule the next piece of work
+ @d2.addBoth
+ def _work_done(res):
+ log.msg("Completed a piece of work")
+ self.queue.pop(0)
+ if self.queue:
+ log.msg("Preparing next piece of work")
+ eventually(self._process)
+ return res
+
+ # When the work is done, trigger d
+ d2.chainDeferred(d)
+
+ def execute(self, cb, *args, **kwargs):
+ kickstart_processing = not self.queue
+ d = defer.Deferred()
+ self.queue.append((d, cb, args, kwargs))
+ if kickstart_processing:
+ self._process()
+ return d
+
+ def executeInThread(self, cb, *args, **kwargs):
+ return self.execute(threads.deferToThread, cb, *args, **kwargs)
+
+
+# A module is effectively a singleton class, so this is OK
+queue = WorkQueue()
+
+
+class Domain:
+
+ """
+ I am a wrapper around a libvirt Domain object
+ """
+
+ def __init__(self, connection, domain):
+ self.connection = connection
+ self.domain = domain
+
+ def name(self):
+ return queue.executeInThread(self.domain.name)
+
+ def create(self):
+ return queue.executeInThread(self.domain.create)
+
+ def shutdown(self):
+ return queue.executeInThread(self.domain.shutdown)
+
+ def destroy(self):
+ return queue.executeInThread(self.domain.destroy)
+
+class Volume:
+ def __init__(self, connection, volume):
+ self.connection = connection
+ self.volume = volume
+
+ @defer.inlineCallbacks
+ def destroy(self):
+ yield queue.executeInThread(self.volume.wipe)
+ yield queue.executeInThread(self.volume.delete)
+
+class Pool:
+ VolumeClass = Volume
+ def __init__(self, connection, pool):
+ self.connection = connection
+ self.pool = pool
+
+ @defer.inlineCallbacks
+ def create_volume(self, xml):
+ res = yield queue.executeInThread(self.pool.createXML, xml)
+ return self.VolumeClass(self.connection, res)
+
+class Connection:
+
+ """
+ I am a wrapper around a libvirt Connection object.
+ """
+
+ DomainClass = Domain
+ PoolClass = Pool
+
+ def __init__(self, uri):
+ self.uri = uri
+ self.connection = libvirt.open(uri)
+
+ @defer.inlineCallbacks
+ def create(self, xml):
+ """ I take libvirt XML and start a new VM """
+ res = yield queue.executeInThread(self.connection.createXML, xml, 0)
+ return self.DomainClass(self, res)
+
+ @defer.inlineCallbacks
+ def lookup_pool(self, name):
+ res = yield queue.executeInThread(self.connection.storagePoolLookupByName, name)
+ return self.PoolClass(self, res)
+
+class LibVirtWorker(AbstractLatentWorker):
+
+ def __init__(self, name, password, connection, master_url, base_image=None, **kwargs):
+ super().__init__(name, password, **kwargs)
+ if not libvirt:
+ config.error(
+ "The python module 'libvirt' is needed to use a LibVirtWorker")
+
+ self.master_url = master_url
+ self.random_name = random_string_generator()
+ self.connection = connection
+ self.base_image = base_image
+
+ self.domain = None
+ self.domain_name = "buildbot-" + self.workername + "-" + self.random_name
+ self.volume = None
+ self.volume_name = "buildbot-" + self.workername + "-" + self.random_name
+ self.pool_name = "buildbot-disks"
+
+ def reconfigService(self, *args, **kwargs):
+ if 'build_wait_timeout' not in kwargs:
+ kwargs['build_wait_timeout'] = 0
+ return super().reconfigService(*args, **kwargs)
+
+ def canStartBuild(self):
+ if self.domain and not self.isConnected():
+ log.msg(
+ "Not accepting builds as existing domain but worker not connected")
+ return False
+
+ return super().canStartBuild()
+
+ @defer.inlineCallbacks
+ def _prepare_image(self):
+ log.msg("Creating temporary image {}".format(self.volume_name))
+ pool = yield self.connection.lookup_pool(self.pool_name)
+ vol_xml = """
+ <volume type='file'>
+ <name>{vol_name}</name>
+ <capacity unit='G'>10</capacity>
+ <target>
+ <format type='qcow2'/>
+ <permissions>
+ <mode>0600</mode>
+ <owner>0</owner>
+ <group>0</group>
+ </permissions>
+ </target>
+ <backingStore>
+ <path>/etc/libvirtd/base-images/buildbot.qcow2</path>
+ <format type='qcow2'/>
+ </backingStore>
+ </volume>
+ """.format(vol_name = self.volume_name)
+ self.volume = yield pool.create_volume(vol_xml)
+
+ @defer.inlineCallbacks
+ def start_instance(self, build):
+ """
+ I start a new instance of a VM.
+
+ If a base_image is specified, I will make a clone of that otherwise i will
+ use image directly.
+
+ If i'm not given libvirt domain definition XML, I will look for my name
+ in the list of defined virtual machines and start that.
+ """
+ domain_xml = """
+ <domain type="kvm">
+ <name>{domain_name}</name>
+ <memory unit="GiB">2</memory>
+ <vcpu>1</vcpu>
+ <sysinfo type='smbios'>
+ <oemStrings>
+ <entry>buildbot_master_url={master_url}</entry>
+ <entry>buildbot_worker_name={worker_name}</entry>
+ </oemStrings>
+ </sysinfo>
+ <os>
+ <type arch="x86_64">hvm</type>
+ <smbios mode='sysinfo'/>
+ </os>
+ <devices>
+ <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
+ <disk type="volume" device="disk">
+ <driver name='qemu' type='qcow2' />
+ <source type="volume" pool="{pool_name}" volume="{volume_name}" />
+ <backingStore type='volume'>
+ <format type='qcow2'/>
+ <source type="volume" pool="niximages" volume="buildbot.qcow2" />
+ </backingStore>
+ <target dev="vda" bus="virtio"/>
+ </disk>
+ <input type="keyboard" bus="usb"/>
+ <graphics type="vnc" port="-1" autoport="yes"/>
+ <interface type="network">
+ <source network="immae" />
+ </interface>
+ </devices>
+ </domain>
+ """.format(volume_name = self.volume_name, master_url = self.master_url, pool_name =
+ self.pool_name, domain_name = self.domain_name, worker_name = self.workername)
+
+ yield self._prepare_image()
+
+ try:
+ self.domain = yield self.connection.create(domain_xml)
+ except Exception:
+ log.err(failure.Failure(),
+ ("Cannot start a VM ({}), failing gracefully and triggering"
+ "a new build check").format(self.workername))
+ self.domain = None
+ return False
+
+ return [self.domain_name]
+
+ def stop_instance(self, fast=False):
+ """
+ I attempt to stop a running VM.
+ I make sure any connection to the worker is removed.
+ If the VM was using a cloned image, I remove the clone
+ When everything is tidied up, I ask that bbot looks for work to do
+ """
+
+ log.msg("Attempting to stop '{}'".format(self.workername))
+ if self.domain is None:
+ log.msg("I don't think that domain is even running, aborting")
+ return defer.succeed(None)
+
+ domain = self.domain
+ self.domain = None
+
+ d = domain.destroy()
+ if self.volume is not None:
+ self.volume.destroy()
+
+ return d
project_env = with lib.attrsets;
mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment //
mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) //
- { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; };
+ {
+ BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}";
+ BUILDBOT_WORKER_PORT = builtins.toString project.workerPort;
+ BUILDBOT_HOST = config.hostEnv.fqdn;
+ BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system";
+ };
in builtins.concatStringsSep "\n"
(lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env);
}
text = config.myEnv.buildbot.ldap.password;
dest = "buildbot/ldap";
}
+ {
+ permissions = "0600";
+ user = "buildbot";
+ group = "buildbot";
+ text = config.myEnv.buildbot.workerPassword;
+ dest = "buildbot/worker_password";
+ }
{
permissions = "0600";
user = "buildbot";
restart = true;
paths = [
"/var/secrets/buildbot/ldap"
+ "/var/secrets/buildbot/worker_password"
"/var/secrets/buildbot/ssh_key"
"/var/secrets/buildbot/${project.name}/environment_file"
] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets;
description = "buildbot slice";
};
+ networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects;
systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
description = "Buildbot Continuous Integration Server ${project.name}.";
after = [ "network-online.target" ];
buildbot_secrets=${varDir}/${project.name}/secrets
install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap
+ install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/worker_password $buildbot_secrets/worker_password
${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
(k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets
)}
});
HOME = "${varDir}/${project.name}";
PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
+ pkgs.python3Packages.libvirt
pkgs.python3Packages.wokkel
pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
pkgs.python3Packages.buildbot-worker
from buildbot.plugins import *
from buildbot_common.build_helpers import *
+import buildbot_common.libvirt as ilibvirt
import os
from buildbot.util import bytes2unicode
import json
PROJECT = "test"
BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT)
- PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT)
+ PB_SOCKET = os.environ["BUILDBOT_WORKER_PORT"]
+ WORKER_HOST = "{}:{}".format(os.environ["BUILDBOT_HOST"], PB_SOCKET)
RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT)
RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT)
GIT_URL = "https://git.immae.eu/perso/Immae/TestProject.git"
SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key"
+ LIBVIRT_URL = os.environ["BUILDBOT_VIRT_URL"] + "?keyfile=" + SSH_KEY_PATH
PUPPET_HOST = "root@backup-1.v.immae.eu"
LDAP_HOST = "ldap.immae.eu"
LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu"
c["www"]["change_hook_dialects"]["base"] = {
"custom_class": CustomBase
}
- c['workers'].append(worker.LocalWorker("generic-worker-test"))
- c['workers'].append(worker.LocalWorker("deploy-worker-test"))
+ c['workers'].append(ilibvirt.LibVirtWorker("test-build",
+ open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
+ ilibvirt.Connection(E.LIBVIRT_URL),
+ E.WORKER_HOST))
+ c['workers'].append(ilibvirt.LibVirtWorker("test-deploy",
+ open(E.SECRETS_FILE + "/worker_password", "r").read().rstrip(),
+ ilibvirt.Connection(E.LIBVIRT_URL),
+ E.WORKER_HOST))
c['schedulers'].append(hook_scheduler("TestProject", timer=1))
c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"]))
logEnviron=False, command=["echo", package]))
factory.addSteps(package_and_upload(package, package_dest, package_url))
- return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory)
+ return util.BuilderConfig(name="TestProject_build", workernames=["test-build"], factory=factory)
def compute_build_infos():
ldap_password=util.Secret("ldap")))
factory.addStep(steps.MasterShellCommand(command=[
"ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host]))
- return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory)
+ return util.BuilderConfig(name="TestProject_deploy", workernames=["test-deploy"], factory=factory)
from twisted.internet import defer
from buildbot.process.buildstep import FAILURE
'';
type = submodule {
options = {
+ rootKeys = mkOption { type = attrsOf str; description = "Keys of root users"; };
ldap = mkOption {
description = ''
LDAP credentials for cn=ssh,ou=services,dc=immae,dc=eu dn
description = "Buildbot configuration";
type = submodule {
options = {
+ workerPassword = mkOption { description = "Buildbot worker password"; type = str; };
user = mkOption {
description = "Buildbot user";
type = submodule {
'';
};
pythonPathHome = mkOption { type = bool; description = "Whether to add project’s python home to python path"; };
+ workerPort = mkOption { type = port; description = "Port for the worker"; };
secrets = mkOption {
type = attrsOf str;
description = "Secrets for the project to dump as files";
# Installation: https://git.immae.eu/mantisbt/view.php?id=93
services.gitolite = {
enable = true;
- adminPubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXqRbiHw7QoHADNIEuo4nUT9fSOIEBMdJZH0bkQAxXyJFyCM1IMz0pxsHV0wu9tdkkr36bPEUj2aV5bkYLBN6nxcV2Y49X8bjOSCPfx3n6Own1h+NeZVBj4ZByrFmqCbTxUJIZ2bZKcWOFncML39VmWdsVhNjg0X4NBBehqXRIKr2gt3E/ESAxTYJFm0BnU0baciw9cN0bsRGqvFgf5h2P48CIAfwhVcGmPQnnAwabnosYQzRWxR0OygH5Kd8mePh6FheIRIigfXsDO8f/jdxwut8buvNIf3m5EBr3tUbTsvM+eV3M5vKGt7sk8T64DVtepTSdOOWtp+47ktsnHOMh immae@immae.eu";
+ adminPubkey = config.myEnv.sshd.rootKeys.immae_dilion;
};
};
}
};
myServices.ssh.modules = [ config.myServices.ssh.predefinedModules.regular ];
- imports = builtins.attrValues (import ../..);
+ imports = builtins.attrValues (import ../..) ++ [ ./dilion/vms.nix ];
system.nssModules = [ pkgs.libvirt ];
system.nssDatabases.hosts = lib.mkForce [ "files" "libvirt_guest" "mymachines" "dns" "myhostname" ];
programs.zsh.enable = true;
+ users.users.libvirt = {
+ hashedPassword = "!";
+ shell = pkgs.bashInteractive;
+ isSystemUser = true;
+ group = "libvirtd";
+ packages = [ pkgs.netcat-openbsd ];
+ openssh.authorizedKeys.keyFiles = [
+ "${privateFiles}/buildbot_ssh_key.pub"
+ ];
+ openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.ismael_flony ];
+ };
+
users.users.backup = {
hashedPassword = "!";
isSystemUser = true;
after = [ "network.target" ];
serviceConfig = {
- ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:22";
+ ExecStart = "${pkgs.socat}/bin/socat TCP-LISTEN:8022,fork TCP:caldance:22";
};
};
recommendedGzipSettings = true;
recommendedProxySettings = true;
upstreams = {
- caldance.servers."nixops-99a7e1ba-54dc-11ea-a965-10bf487fe63b-caldance:3031" = {};
+ caldance.servers."caldance:3031" = {};
};
virtualHosts = {
"dev.immae.eu" = {
--- /dev/null
+# inspired from https://nixos.wiki/wiki/Virtualization_in_NixOS
+{ config, pkgs, lib, ... }@args:
+let
+ networks = {
+ immae = {
+ bridgeNumber = "1";
+ ipRange = "192.168.100";
+ };
+ };
+ guests = {
+ caldance = {
+ pool = "zfspool";
+ cpus = "1";
+ memory = "2";
+ network = "immae";
+ diskSize = "10GiB";
+ extraDevicesXML = ''
+ <filesystem type="mount">
+ <source dir="/var/lib/caldance"/>
+ <target dir="home"/>
+ </filesystem>
+ '';
+ };
+ buildbot = {
+ pool = "zfspool";
+ cpus = "1";
+ memory = "3";
+ network = "immae";
+ diskSize = "10GiB";
+ destroyVolumeOnExit = true;
+ preStart = ''
+ if ! ${pkgs.libvirt}/bin/virsh pool-info --pool niximages &> /dev/null; then
+ pool-create-as --name niximages --type dir --target /etc/libvirtd/base-images/
+ fi
+ if ! ${pkgs.libvirt}/bin/virsh pool-info --pool buildbot-disks &> /dev/null; then
+ mkdir -p /var/lib/libvirt/images/buildbot-disks
+ pool-create-as --name buildbot-disks --type dir --target /var/lib/libvirt/images/buildbot-disks
+ fi
+ '';
+ };
+ };
+ toImage = f: "${import ./vms/base_image.nix f (args // { myEnv = config.myEnv; })}/nixos.qcow2";
+in
+{
+ environment.etc."libvirtd/base-images/nixos.qcow2".source = toImage ./vms/base_configuration.nix;
+ environment.etc."libvirtd/base-images/buildbot.qcow2".source = toImage ./vms/buildbot_configuration.nix;
+ systemd.services = lib.mapAttrs' (name: guest: lib.nameValuePair "libvirtd-guest-${name}" {
+ after = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
+ requires = [ "libvirtd.service" "libvirtd-network-${guest.network}.service" ];
+ wantedBy = [ "multi-user.target" ];
+ serviceConfig = {
+ Type = "oneshot";
+ RemainAfterExit = "yes";
+ };
+ script =
+ let
+ xml = pkgs.writeText "libvirt-guest-${name}.xml"
+ ''
+ <domain type="kvm">
+ <name>${name}</name>
+ <uuid>UUID</uuid>
+ <memory unit="GiB">${guest.memory}</memory>
+ <vcpu>${guest.cpus}</vcpu>
+ <os>
+ <type arch="x86_64">hvm</type>
+ </os>
+ <devices>
+ <emulator>/run/current-system/sw/bin/qemu-system-x86_64</emulator>
+ <disk type="volume">
+ <source pool="${guest.pool}" volume="guest-${name}" />
+ <target dev="vda" bus="virtio"/>
+ </disk>
+ ${guest.extraDevicesXML or ""}
+ <input type="keyboard" bus="usb"/>
+ <graphics type="vnc" port="-1" autoport="yes"/>
+ <interface type="network">
+ <source network="${guest.network}" />
+ </interface>
+ </devices>
+ <features>
+ <acpi/>
+ </features>
+ </domain>
+ '';
+ in
+ guest.preStart or "" + ''
+ if ! ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
+ ${pkgs.libvirt}/bin/virsh vol-create-as --pool ${guest.pool} --name 'guest-${name}' --capacity '${guest.diskSize}'
+ volume_path=$(${pkgs.libvirt}/bin/virsh vol-path --pool ${guest.pool} --vol 'guest-${name}')
+ ${pkgs.qemu}/bin/qemu-img convert /etc/libvirtd/base-images/nixos.qcow2 $volume_path
+ fi
+ uuid="$(${pkgs.libvirt}/bin/virsh domuuid '${name}' || true)"
+ ${pkgs.libvirt}/bin/virsh define <(sed "s/UUID/$uuid/" '${xml}')
+ ${pkgs.libvirt}/bin/virsh start '${name}'
+ '';
+ preStop = ''
+ ${pkgs.libvirt}/bin/virsh shutdown '${name}'
+ let "timeout = $(date +%s) + 10"
+ while [ "$(${pkgs.libvirt}/bin/virsh list --name | grep --count '^${name}$')" -gt 0 ]; do
+ if [ "$(date +%s)" -ge "$timeout" ]; then
+ # Meh, we warned it...
+ ${pkgs.libvirt}/bin/virsh destroy '${name}'
+ else
+ # The machine is still running, let's give it some time to shut down
+ sleep 0.5
+ fi
+ done
+ '' + lib.optionalString (guest.destroyVolumeOnExit or false) ''
+ if ${pkgs.libvirt}/bin/virsh vol-key 'guest-${name}' --pool ${guest.pool} &> /dev/null; then
+ ${pkgs.libvirt}/bin/virsh vol-wipe --pool ${guest.pool} --vol 'guest-${name}' || true
+ ${pkgs.libvirt}/bin/virsh vol-delete --pool ${guest.pool} --vol 'guest-${name}'
+ fi
+ '';
+ }) guests // (lib.mapAttrs' (name: network: lib.nameValuePair "libvirtd-network-${name}" {
+ after = [ "libvirtd.service" ];
+ requires = [ "libvirtd.service" ];
+ wantedBy = [ "multi-user.target" ];
+ serviceConfig = {
+ Type = "oneshot";
+ RemainAfterExit = "yes";
+ };
+ script = let
+ xml = pkgs.writeText "libvirt-network-${name}.xml" ''
+ <network>
+ <name>${name}</name>
+ <uuid>UUID</uuid>
+ <forward mode='nat' />
+ <bridge name='virbr${network.bridgeNumber}' />
+ <domain name='${name}' localOnly='yes'/>
+ <ip address='${network.ipRange}.1' netmask='255.255.255.0'>
+ <dhcp>
+ <range start='${network.ipRange}.2' end='${network.ipRange}.254'/>
+ </dhcp>
+ </ip>
+ </network>
+ '';
+ in ''
+ uuid="$(${pkgs.libvirt}/bin/virsh net-uuid '${name}' || true)"
+ ${pkgs.libvirt}/bin/virsh net-define <(sed "s/UUID/$uuid/" '${xml}')
+ ${pkgs.libvirt}/bin/virsh net-start '${name}'
+ '';
+ preStop = ''
+ ${pkgs.libvirt}/bin/virsh net-destroy '${name}'
+ '';
+ }) networks);
+}
--- /dev/null
+{ lib, config, ... }@args:
+{
+ options.myEnv = (import ../../../environment.nix (args // { name = "dummy"; })).options.myEnv;
+ config = {
+ fileSystems."/".device = "/dev/disk/by-label/nixos";
+ boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "virtio_balloon" "virtio_blk" "virtio_pci" "virtio_ring" ];
+ boot.loader = {
+ grub = {
+ version = 2;
+ device = "/dev/vda";
+ };
+ timeout = 0;
+ };
+ services.openssh.enable = true;
+ networking.firewall.allowedTCPPorts = [ 22 ];
+ users = {
+ mutableUsers = false;
+ users.root.openssh.authorizedKeys.keys = [ config.myEnv.sshd.rootKeys.immae_dilion ];
+ };
+ };
+}
--- /dev/null
+configuration_file: { pkgs ? import <nixpkgs> {}, system ? builtins.currentSystem, myEnv, ... }:
+let
+ config = (import <nixpkgs/nixos/lib/eval-config.nix> {
+ inherit system;
+ modules = [ {
+ myEnv = myEnv;
+ imports = [ configuration_file ];
+
+ # We want our template image to be as small as possible, but the deployed image should be able to be
+ # of any size. Hence we resize on the first boot.
+ systemd.services.resize-main-fs = {
+ wantedBy = [ "multi-user.target" ];
+ serviceConfig.Type = "oneshot";
+ script =
+ ''
+ # Resize main partition to fill whole disk
+ echo ", +" | ${pkgs.utillinux}/bin/sfdisk /dev/vda --no-reread -N 1
+ ${pkgs.parted}/bin/partprobe
+ # Resize filesystem
+ ${pkgs.e2fsprogs}/bin/resize2fs /dev/vda1
+ '';
+ };
+ } ];
+ }).config;
+in pkgs.vmTools.runInLinuxVM (
+ pkgs.runCommand "nixos-base-image"
+ {
+ memSize = 768;
+ preVM =
+ ''
+ mkdir $out
+ diskImage=image.qcow2
+ ${pkgs.vmTools.qemu}/bin/qemu-img create -f qcow2 $diskImage 2G
+ mv closure xchg/
+ '';
+ postVM =
+ ''
+ echo compressing VM image...
+ ${pkgs.vmTools.qemu}/bin/qemu-img convert -c $diskImage -O qcow2 $out/nixos.qcow2
+ '';
+ buildInputs = [ pkgs.utillinux pkgs.perl pkgs.parted pkgs.e2fsprogs ];
+ exportReferencesGraph =
+ [ "closure" config.system.build.toplevel ];
+ }
+ ''
+ # Create the partition
+ parted /dev/vda mklabel msdos
+ parted /dev/vda -- mkpart primary ext4 1M -1s
+
+ # Format the partition
+ mkfs.ext4 -L nixos /dev/vda1
+ mkdir /mnt
+ mount /dev/vda1 /mnt
+
+ for dir in dev proc sys; do
+ mkdir /mnt/$dir
+ mount --bind /$dir /mnt/$dir
+ done
+
+ storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure)
+ echo filling Nix store...
+ mkdir -p /mnt/nix/store
+ set -f
+ cp -prd $storePaths /mnt/nix/store
+ # The permissions will be set up incorrectly if the host machine is not running NixOS
+ chown -R 0:30000 /mnt/nix/store
+
+ mkdir -p /mnt/etc/nix
+ echo 'build-users-group = ' > /mnt/etc/nix/nix.conf
+
+ # Register the paths in the Nix database.
+ export USER=root
+ printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \
+ chroot /mnt ${config.nix.package.out}/bin/nix-store --load-db
+
+ # Create the system profile to allow nixos-rebuild to work.
+ chroot /mnt ${config.nix.package.out}/bin/nix-env \
+ -p /nix/var/nix/profiles/system --set ${config.system.build.toplevel}
+
+ # `nixos-rebuild' requires an /etc/NIXOS.
+ mkdir -p /mnt/etc/nixos
+ touch /mnt/etc/NIXOS
+
+ # `switch-to-configuration' requires a /bin/sh
+ mkdir -p /mnt/bin
+ ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh
+
+ # Generate the GRUB menu.
+ chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot
+
+ umount /mnt/{proc,dev,sys}
+ umount /mnt
+ ''
+)
--- /dev/null
+{ pkgs, config, lib, ... }:
+{
+ imports = [
+ <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
+ ./base_configuration.nix
+ ];
+ systemd.services.buildbot-worker.serviceConfig.ExecStartPre = let
+ cfg = config.services.buildbot-worker;
+ script = pkgs.writeScript "decode-dmi" ''
+ #!${pkgs.stdenv.shell}
+
+ mkdir -vp "${cfg.buildbotDir}"
+ varfile=${cfg.buildbotDir}/variables
+ rm $varfile || true
+ echo "[DEFAULT]" > $varfile
+ strings=$(${pkgs.dmidecode}/bin/dmidecode --oem-string count)
+ for i in $(seq 1 $strings); do
+ ${pkgs.dmidecode}/bin/dmidecode --oem-string $i >> $varfile
+ done
+ chown -R ${cfg.user}:${cfg.group} ${cfg.buildbotDir}
+ '';
+ in
+ lib.mkForce ["+${script}"];
+ systemd.services.buildbot-worker.serviceConfig.ExecStart = let
+ cfg = config.services.buildbot-worker;
+ tacFile = pkgs.writeText "buildbot-worker.tac" ''
+ import os
+ from io import open
+
+ from buildbot_worker.bot import Worker
+ from twisted.application import service
+
+ basedir = '${cfg.buildbotDir}'
+
+ # note: this line is matched against to check that this is a worker
+ # directory; do not edit it.
+ application = service.Application('buildbot-worker')
+
+ import configparser
+ config = config = configparser.ConfigParser()
+ config.read("${cfg.buildbotDir}/variables")
+ master_url_split = config["DEFAULT"]["buildbot_master_url"].split(':')
+ buildmaster_host = master_url_split[0]
+ port = int(master_url_split[1])
+ workername = config["DEFAULT"]["buildbot_worker_name"]
+
+ with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
+ passwd = passwd_file.read().strip('\r\n')
+ keepalive = ${toString cfg.keepalive}
+ umask = None
+ maxdelay = 300
+ numcpus = None
+ allow_shutdown = None
+
+ s = Worker(buildmaster_host, port, workername, passwd, basedir,
+ keepalive, umask=umask, maxdelay=maxdelay,
+ numcpus=numcpus, allow_shutdown=allow_shutdown)
+ s.setServiceParent(application)
+ '';
+ in
+ lib.mkForce "${cfg.package.pythonModule.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
+ services.buildbot-worker = {
+ enable = true;
+ workerPass = config.myEnv.buildbot.workerPassword;
+ packages = [ pkgs.git pkgs.gzip pkgs.openssh ];
+ };
+}
-Subproject commit 9ac6c1459d2eeb24be0a991f745b567f0fcb0cca
+Subproject commit 932623303fd674faf8106e03ea0b89195edfdc02
projects / perso / Immae / Config / Nix.git / commitdiff
