X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=systems%2Feldiron%2Fbuildbot%2Fdefault.nix;h=e86b081758c0a6ba91add4d0e5be917f72421109;hb=1a64deeb894dc95e2645a75771732c6cc53a79ad;hpb=fa25ffd4583cc362075cd5e1b4130f33306103f0;p=perso%2FImmae%2FConfig%2FNix.git
diff --git a/systems/eldiron/buildbot/default.nix b/systems/eldiron/buildbot/default.nix
new file mode 100644
index 0000000..e86b081
--- /dev/null
+++ b/systems/eldiron/buildbot/default.nix
@@ -0,0 +1,310 @@
+{ lib, pkgs, config, buildbot, ... }:
+let
+ varDir = "/var/lib/buildbot";
+ bb-python = buildbot.pythonModule;
+in
+{
+ options = {
+ myServices.buildbot.enable = lib.mkOption {
+ type = lib.types.bool;
+ default = false;
+ description = ''
+ Whether to enable buildbot.
+ '';
+ };
+ };
+
+ config = lib.mkIf config.myServices.buildbot.enable {
+ myEnv.buildbot.projects.test = {
+ name = "test";
+ workerPort = config.myEnv.ports.buildbot_test;
+ packages = [ pkgs.git pkgs.gzip pkgs.openssh ];
+ pythonPathHome = false;
+ secrets = {
+ apprise_webhooks = builtins.concatStringsSep "\n" [
+ "{{ .apprise_webhooks.matrix_immae_eu_alert }}"
+ ];
+ notify_xmpp_password = "{{ .xmpp.notify_bot }}";
+ };
+ activationScript = ''
+ install -m 0755 -o buildbot -g buildbot -d /var/lib/ftp/release.immae.eu/test
+ '';
+ webhookTokens = [
+ "{{ .buildbot.webhookTokens.Immae }}"
+ "{{ .buildbot.webhookTokens.Immae }}"
+ ];
+ };
+
+ myServices.chatonsProperties.hostings.buildbot = {
+ file.datetime = "2022-08-21T10:37:00";
+ hosting = {
+ name = "Buildbot";
+ description = "Python-based continuous integration testing framework";
+ type = "INSTANCE";
+ website = "https://git.immae.eu";
+ logo = "https://www.buildbot.net/img/icon.png";
+ status.level = "OK";
+ status.description = "OK";
+ registration.load = "OPEN";
+ install.type = "PACKAGE";
+ guide.user = "https://www.immae.eu/docs/forge-logicielle.html";
+ };
+ software = {
+ name = "Buildbot";
+ website = "https://www.buildbot.net/";
+ license.url = "https://github.com/buildbot/buildbot/blob/master/LICENSE";
+ license.name = "GNU General Public License v2.0";
+ version = pkgs.buildbot.version;
+ source.url = "https://github.com/buildbot/buildbot";
+ };
+ };
+ nixpkgs.overlays = [
+ (self: super: {
+ follow-systemd-unit = self.writeScriptBin "follow-systemd-unit" ''
+ #!${self.stdenv.shell}
+
+ set -euo pipefail
+
+ service=$1
+ before_invocation_id=$2
+
+ get_id() {
+ systemctl show -p InvocationID --value "$service"
+ }
+
+ while [ "$(get_id)" = "$before_invocation_id" ]; do sleep 1; done
+
+ invocation_id="$(get_id)"
+ cursor="$(mktemp)"
+ trap "rm -f $cursor" EXIT
+
+ get_logs() {
+ journalctl --quiet --cursor-file=$cursor INVOCATION_ID=$invocation_id + _SYSTEMD_INVOCATION_ID=$invocation_id
+ }
+
+ while [ -n "$(systemctl show -p Job --value "$service")" ]; do
+ get_logs
+ done
+ get_logs
+ '';
+ })
+ ];
+ ids.uids.buildbot = config.myEnv.buildbot.user.uid;
+ ids.gids.buildbot = config.myEnv.buildbot.user.gid;
+
+ users.groups.buildbot.gid = config.ids.gids.buildbot;
+ users.users.buildbot = {
+ name = "buildbot";
+ uid = config.ids.uids.buildbot;
+ group = "buildbot";
+ description = "Buildbot user";
+ home = varDir;
+ extraGroups = [ "keys" "systemd-journal" ];
+ useDefaultShell = true;
+ openssh.authorizedKeys.keys = [ config.myEnv.buildbot.ssh_key.public ];
+ };
+
+ services.websites.env.tools.watchPaths = lib.attrsets.mapAttrsToList
+ (k: project: config.secrets.fullPaths."buildbot/${project.name}/webhook-httpd-include")
+ config.myEnv.buildbot.projects;
+
+ services.websites.env.tools.vhostConfs.git.extraConfig = lib.attrsets.mapAttrsToList (k: project: ''
+ RedirectMatch permanent "^/buildbot/${project.name}$" "/buildbot/${project.name}/"
+ RewriteEngine On
+ RewriteRule ^/buildbot/${project.name}/ws(.*)$ unix:///run/buildbot/${project.name}.sock|ws://git.immae.eu/ws$1 [P,NE,QSA,L]
+ ProxyPass /buildbot/${project.name}/ unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
+ ProxyPassReverse /buildbot/${project.name}/ unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
+
+ Use LDAPConnect
+ Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
+
+ SetEnvIf X-Url-Scheme https HTTPS=1
+ ProxyPreserveHost On
+
+
+
+ Require local
+ Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
+ Include ${config.secrets.fullPaths."buildbot/${project.name}/webhook-httpd-include"}
+
+
+ '') config.myEnv.buildbot.projects;
+
+ system.activationScripts = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
+ deps = [ "users" "wrappers" ];
+ text = ''
+ install -m 755 -o buildbot -g buildbot -d ${varDir}/${project.name}
+
+ ${project.activationScript}
+ '';
+ }) config.myEnv.buildbot.projects;
+
+ secrets.keys = lib.listToAttrs (
+ lib.lists.flatten (
+ lib.attrsets.mapAttrsToList (k: project:
+ lib.attrsets.mapAttrsToList (k: v:
+ (lib.nameValuePair "buildbot/${project.name}/${k}" {
+ permissions = "0600";
+ user = "buildbot";
+ group = "buildbot";
+ text = v;
+ })
+ ) project.secrets
+ ++ [
+ (lib.nameValuePair "buildbot/${project.name}/webhook-httpd-include" {
+ permissions = "0600";
+ user = "wwwrun";
+ group = "wwwrun";
+ text = lib.optionalString (project.webhookTokens != null) ''
+ Require expr "req('Access-Key') in { ${builtins.concatStringsSep ", " (map (x: "'${x}'") project.webhookTokens)} }"
+ '';
+ })
+ (lib.nameValuePair "buildbot/${project.name}/environment_file" {
+ permissions = "0600";
+ user = "buildbot";
+ group = "buildbot";
+ keyDependencies = [ (buildbot.buildbot_config project).src ] ++ project.secretsDeps;
+ text = let
+ project_env = with lib.attrsets;
+ mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment //
+ {
+ BUILDBOT_PROJECT_DIR = (buildbot.buildbot_config project).src;
+ BUILDBOT_WORKER_PORT = builtins.toString project.workerPort;
+ BUILDBOT_HOST = config.hostEnv.fqdn;
+ BUILDBOT_VIRT_URL = "qemu+ssh://libvirt@dilion.immae.eu/system";
+ };
+ in builtins.concatStringsSep "\n"
+ (lib.mapAttrsToList (envK: envV: "${envK}=${envV}") project_env);
+ })
+ ]
+ ) config.myEnv.buildbot.projects
+ )
+ ) // {
+ "buildbot/ldap" = {
+ permissions = "0600";
+ user = "buildbot";
+ group = "buildbot";
+ text = config.myEnv.buildbot.ldap.password;
+ };
+ "buildbot/worker_password" = {
+ permissions = "0600";
+ user = "buildbot";
+ group = "buildbot";
+ text = config.myEnv.buildbot.workerPassword;
+ };
+ "buildbot/ssh_key" = {
+ permissions = "0600";
+ user = "buildbot";
+ group = "buildbot";
+ text = config.myEnv.buildbot.ssh_key.private;
+ };
+ "buildbot/ssh_known_hosts" = {
+ permissions = "0644";
+ user = "buildbot";
+ group = "buildbot";
+ text = ''
+ git.immae.eu ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF
+ eldiron ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF
+ phare.normalesup.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN2GomItXICXpCtCFRMT2xuerqx2nLMO/3mNUuWyzFr1
+ '';
+ };
+ };
+
+ services.filesWatcher = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
+ restart = true;
+ paths = [
+ config.secrets.fullPaths."buildbot/ldap"
+ config.secrets.fullPaths."buildbot/worker_password"
+ config.secrets.fullPaths."buildbot/ssh_key"
+ config.secrets.fullPaths."buildbot/${project.name}/environment_file"
+ ] ++ lib.attrsets.mapAttrsToList (k: v: config.secrets.fullPaths."buildbot/${project.name}/${k}") project.secrets;
+ }) config.myEnv.buildbot.projects;
+
+ systemd.slices.buildbot = {
+ description = "buildbot slice";
+ };
+
+ networking.firewall.allowedTCPPorts = lib.attrsets.mapAttrsToList (k: v: v.workerPort) config.myEnv.buildbot.projects;
+ systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
+ description = "Buildbot Continuous Integration Server ${project.name}.";
+ after = [ "network-online.target" ];
+ wantedBy = [ "multi-user.target" ];
+ path = project.packages;
+ preStart = let
+ master-cfg = "${buildbot.buildbot_common}/${bb-python.pythonForBuild.sitePackages}/buildbot_common/master.cfg";
+ tac_file = pkgs.writeText "buildbot.tac" ''
+ import os
+
+ from twisted.application import service
+ from buildbot.master import BuildMaster
+
+ basedir = '${varDir}/${project.name}'
+ rotateLength = 10000000
+ maxRotatedFiles = 10
+ configfile = '${master-cfg}'
+
+ # Default umask for server
+ umask = None
+
+ # if this is a relocatable tac file, get the directory containing the TAC
+ if basedir == '.':
+ import os
+ basedir = os.path.abspath(os.path.dirname(__file__))
+
+ # note: this line is matched against to check that this is a buildmaster
+ # directory; do not edit it.
+ application = service.Application('buildmaster')
+ from twisted.python.logfile import LogFile
+ from twisted.python.log import ILogObserver, FileLogObserver
+ logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+ maxRotatedFiles=maxRotatedFiles)
+ application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+ m = BuildMaster(basedir, configfile, umask)
+ m.setServiceParent(application)
+ m.log_rotation.rotateLength = rotateLength
+ m.log_rotation.maxRotatedFiles = maxRotatedFiles
+ '';
+ in ''
+ if [ ! -f ${varDir}/${project.name}/buildbot.tac ]; then
+ ${buildbot}/bin/buildbot create-master -c "${master-cfg}" "${varDir}/${project.name}"
+ rm -f ${varDir}/${project.name}/master.cfg.sample
+ rm -f ${varDir}/${project.name}/buildbot.tac
+ fi
+ ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac
+ # different buildbots may be trying that simultaneously, add the || true to avoid complaining in case of race
+ install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ssh_key"} ${varDir}/buildbot_key || true
+ install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ssh_known_hosts"} ${varDir}/buildbot_hosts || true
+ buildbot_secrets=${varDir}/${project.name}/secrets
+ install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
+ install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ldap"} $buildbot_secrets/ldap
+ install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/worker_password"} $buildbot_secrets/worker_password
+ ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
+ (k: v: "install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/${project.name}/${k}"} $buildbot_secrets/${k}") project.secrets
+ )}
+ ${buildbot}/bin/buildbot upgrade-master ${varDir}/${project.name}
+ '';
+ environment = let
+ HOME = "${varDir}/${project.name}";
+ PYTHONPATH = "${bb-python.withPackages (self:
+ buildbot.common_packages self ++
+ [ (buildbot.buildbot_config project) ]
+ )}/${bb-python.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${bb-python.sitePackages}" else ""}";
+ in { inherit PYTHONPATH HOME; };
+
+ serviceConfig = {
+ Slice = "buildbot.slice";
+ Type = "forking";
+ User = "buildbot";
+ Group = "buildbot";
+ RuntimeDirectory = "buildbot";
+ RuntimeDirectoryPreserve = "yes";
+ StateDirectory = "buildbot";
+ SupplementaryGroups = "keys";
+ WorkingDirectory = "${varDir}/${project.name}";
+ ExecStart = "${buildbot}/bin/buildbot start";
+ EnvironmentFile = config.secrets.fullPaths."buildbot/${project.name}/environment_file";
+ };
+ }) config.myEnv.buildbot.projects;
+ };
+}