X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=modules%2Fprivate%2Fbuildbot%2Fdefault.nix;h=d9e5966edc82d525eb15ae9285fd0e6836ad5ffe;hb=c7b16397101fd534c41ede5ae7c55e7e120c06f7;hp=c8ee48e980c37add026f9f4e4ebd5be72a4b30a9;hpb=5400b9b6f65451d41a9106fae6fc00f97d83f4ef;p=perso%2FImmae%2FConfig%2FNix.git diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix index c8ee48e..d9e5966 100644 --- a/modules/private/buildbot/default.nix +++ b/modules/private/buildbot/default.nix @@ -1,16 +1,8 @@ { lib, pkgs, config, ... }: let varDir = "/var/lib/buildbot"; - buildbot_common = pkgs.python3Packages.buildPythonPackage rec { - name = "buildbot_common"; - src = ./common; - format = "other"; - installPhase = '' - mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages} - cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common - ''; - }; - buildbot = pkgs.python3Packages.buildbot-full; + bb-python = buildbot.pythonModule; + buildbot = pkgs.immae-buildbot; in { options = { @@ -24,9 +16,60 @@ in }; config = lib.mkIf config.myServices.buildbot.enable { - services.duplyBackup.profiles.buildbot = { - rootDir = varDir; + myServices.chatonsProperties.hostings.buildbot = { + file.datetime = "2022-08-21T10:37:00"; + hosting = { + name = "Buildbot"; + description = "Python-based continuous integration testing framework"; + type = "INSTANCE"; + website = "https://git.immae.eu"; + logo = "https://www.buildbot.net/img/icon.png"; + status.level = "OK"; + status.description = "OK"; + registration.load = "OPEN"; + install.type = "PACKAGE"; + guide.user = "https://www.immae.eu/docs/forge-logicielle.html"; + }; + software = { + name = "Buildbot"; + website = "https://www.buildbot.net/"; + license.url = "https://github.com/buildbot/buildbot/blob/master/LICENSE"; + license.name = "GNU General Public License v2.0"; + version = pkgs.buildbot.version; + source.url = "https://github.com/buildbot/buildbot"; + }; }; + nixpkgs.overlays = [ + (self: super: { + follow-systemd-unit = self.writeScriptBin "follow-systemd-unit" '' + #!${self.stdenv.shell} + + set -euo pipefail + + service=$1 + before_invocation_id=$2 + + get_id() { + systemctl show -p InvocationID --value "$service" + } + + while [ "$(get_id)" = "$before_invocation_id" ]; do sleep 1; done + + invocation_id="$(get_id)" + cursor="$(mktemp)" + trap "rm -f $cursor" EXIT + + get_logs() { + journalctl --quiet --cursor-file=$cursor INVOCATION_ID=$invocation_id + _SYSTEMD_INVOCATION_ID=$invocation_id + } + + while [ -n "$(systemctl show -p Job --value "$service")" ]; do + get_logs + done + get_logs + ''; + }) + ]; ids.uids.buildbot = config.myEnv.buildbot.user.uid; ids.gids.buildbot = config.myEnv.buildbot.user.gid; @@ -37,11 +80,13 @@ in group = "buildbot"; description = "Buildbot user"; home = varDir; - extraGroups = [ "keys" ]; + extraGroups = [ "keys" "systemd-journal" ]; + useDefaultShell = true; + openssh.authorizedKeys.keys = [ config.myEnv.buildbot.ssh_key.public ]; }; services.websites.env.tools.watchPaths = lib.attrsets.mapAttrsToList - (k: project: "/var/secrets/buildbot/${project.name}/webhook-httpd-include") + (k: project: config.secrets.fullPaths."buildbot/${project.name}/webhook-httpd-include") config.myEnv.buildbot.projects; services.websites.env.tools.vhostConfs.git.extraConfig = lib.attrsets.mapAttrsToList (k: project: '' @@ -61,7 +106,7 @@ in Require local Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu - Include /var/secrets/buildbot/${project.name}/webhook-httpd-include + Include ${config.secrets.fullPaths."buildbot/${project.name}/webhook-httpd-include"} '') config.myEnv.buildbot.projects; @@ -75,63 +120,98 @@ in ''; }) config.myEnv.buildbot.projects; - secrets.keys = ( + secrets.keys = lib.listToAttrs ( lib.lists.flatten ( lib.attrsets.mapAttrsToList (k: project: lib.attrsets.mapAttrsToList (k: v: - { + (lib.nameValuePair "buildbot/${project.name}/${k}" { permissions = "0600"; user = "buildbot"; group = "buildbot"; - text = v; - dest = "buildbot/${project.name}/${k}"; - } + text = if builtins.isFunction v then v pkgs config else v; + }) ) project.secrets ++ [ - { + (lib.nameValuePair "buildbot/${project.name}/webhook-httpd-include" { permissions = "0600"; user = "wwwrun"; group = "wwwrun"; text = lib.optionalString (project.webhookTokens != null) '' Require expr "req('Access-Key') in { ${builtins.concatStringsSep ", " (map (x: "'${x}'") project.webhookTokens)} }" ''; - dest = "buildbot/${project.name}/webhook-httpd-include"; - } + }) + (lib.nameValuePair "buildbot/${project.name}/environment_file" { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = let + project_env = with lib.attrsets; + mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" (if builtins.isFunction v then v pkgs else v)) project.environment // + { + BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; + BUILDBOT_WORKER_PORT = builtins.toString project.workerPort; + BUILDBOT_HOST = config.hostEnv.fqdn; + BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system"; + }; + in builtins.concatStringsSep "\n" + (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env); + }) ] ) config.myEnv.buildbot.projects ) - ) ++ [ - { + ) // { + "buildbot/ldap" = { permissions = "0600"; user = "buildbot"; group = "buildbot"; text = config.myEnv.buildbot.ldap.password; - dest = "buildbot/ldap"; - } - { + }; + "buildbot/worker_password" = { permissions = "0600"; user = "buildbot"; group = "buildbot"; - text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key"; - dest = "buildbot/ssh_key"; - } - ]; + text = config.myEnv.buildbot.workerPassword; + }; + "buildbot/ssh_key" = { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = config.myEnv.buildbot.ssh_key.private; + }; + "buildbot/ssh_known_hosts" = { + permissions = "0644"; + user = "buildbot"; + group = "buildbot"; + text = '' + git.immae.eu ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF + eldiron ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF + phare.normalesup.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN2GomItXICXpCtCFRMT2xuerqx2nLMO/3mNUuWyzFr1 + ''; + }; + }; services.filesWatcher = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { restart = true; paths = [ - "/var/secrets/buildbot/ldap" - "/var/secrets/buildbot/ssh_key" - ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets; + config.secrets.fullPaths."buildbot/ldap" + config.secrets.fullPaths."buildbot/worker_password" + config.secrets.fullPaths."buildbot/ssh_key" + config.secrets.fullPaths."buildbot/${project.name}/environment_file" + ] ++ lib.attrsets.mapAttrsToList (k: v: config.secrets.fullPaths."buildbot/${project.name}/${k}") project.secrets; }) config.myEnv.buildbot.projects; + systemd.slices.buildbot = { + description = "buildbot slice"; + }; + + networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects; systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { description = "Buildbot Continuous Integration Server ${project.name}."; after = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; - path = project.packages pkgs ++ (project.pythonPackages buildbot.pythonModule pkgs); + path = project.packages pkgs; preStart = let - master-cfg = "${buildbot_common}/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common/master.cfg"; + master-cfg = "${buildbot.buildbot_common}/${bb-python.pythonForBuild.sitePackages}/buildbot_common/master.cfg"; tac_file = pkgs.writeText "buildbot.tac" '' import os @@ -173,39 +253,27 @@ in fi ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac # different buildbots may be trying that simultaneously, add the || true to avoid complaining in case of race - install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ssh_key ${varDir}/buildbot_key || true + install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ssh_key"} ${varDir}/buildbot_key || true + install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ssh_known_hosts"} ${varDir}/buildbot_hosts || true buildbot_secrets=${varDir}/${project.name}/secrets install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets - install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap + install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ldap"} $buildbot_secrets/ldap + install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/worker_password"} $buildbot_secrets/worker_password ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList - (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets + (k: v: "install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/${project.name}/${k}"} $buildbot_secrets/${k}") project.secrets )} ${buildbot}/bin/buildbot upgrade-master ${varDir}/${project.name} ''; environment = let - project_env = with lib.attrsets; - mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment // - mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) // - { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; }; - buildbot_config = pkgs.python3Packages.buildPythonPackage (rec { - name = "buildbot_config-${project.name}"; - src = ./projects + "/${project.name}"; - format = "other"; - installPhase = '' - mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages} - cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_config - ''; - }); HOME = "${varDir}/${project.name}"; - PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [ - pkgs.python3Packages.wokkel - pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot - pkgs.python3Packages.buildbot-worker - buildbot_common buildbot_config - ])}/${buildbot.pythonModule.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${pkgs.python3.pythonForBuild.sitePackages}" else ""}"; - in project_env // { inherit PYTHONPATH HOME; }; + PYTHONPATH = "${bb-python.withPackages (self: + buildbot.common_packages self ++ + [ (buildbot.buildbot_config project) ] + )}/${bb-python.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${bb-python.sitePackages}" else ""}"; + in { inherit PYTHONPATH HOME; }; serviceConfig = { + Slice = "buildbot.slice"; Type = "forking"; User = "buildbot"; Group = "buildbot"; @@ -215,6 +283,7 @@ in SupplementaryGroups = "keys"; WorkingDirectory = "${varDir}/${project.name}"; ExecStart = "${buildbot}/bin/buildbot start"; + EnvironmentFile = config.secrets.fullPaths."buildbot/${project.name}/environment_file"; }; }) config.myEnv.buildbot.projects; };