X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=nixops%2Fmodules%2Fbuildbot%2Fdefault.nix;h=d7dd47943aac86676b582d8f08135ba3714d8417;hb=b798cf6d60baf5aad5438ba5936259c64d87299c;hp=cd5b2609e86484789f56f0a05604d720fd9f8541;hpb=9fb4205e2ceadb79a93cbe44bd77ebebe8c94625;p=perso%2FImmae%2FConfig%2FNix.git diff --git a/nixops/modules/buildbot/default.nix b/nixops/modules/buildbot/default.nix index cd5b260..d7dd479 100644 --- a/nixops/modules/buildbot/default.nix +++ b/nixops/modules/buildbot/default.nix @@ -1,47 +1,16 @@ -{ lib, pkgs, pkgsNext, config, myconfig, mylibs, ... }: +{ lib, pkgs, config, myconfig, mylibs, ... }: let varDir = "/var/lib/buildbot"; - buildslist_src = mylibs.fetchedGitPrivate ./buildslist.json; - buildslist_yarn = pkgsNext.yarn2nix.mkYarnModules { - name = "buildslist-yarn-modules"; - packageJSON = "${buildslist_src.src}/package.json"; - yarnLock = "${buildslist_src.src}/yarn.lock"; - }; - buildslist_bower = pkgsNext.buildBowerComponents { - name = "buildslist"; - generated = ./bower.nix; - src = "${buildslist_src.src}/guanlecoja/"; - }; - - buildslist = pkgsNext.python3Packages.buildPythonPackage rec { - pname = "buildbot-buildslist"; - inherit (pkgsNext.buildbot-pkg) version; - - preConfigure = '' - export HOME=$PWD - cp -a ${buildslist_yarn}/node_modules . - chmod -R u+w node_modules - cp -a ${buildslist_bower}/bower_components ./libs - chmod -R u+w libs - ''; - propagatedBuildInputs = with pkgsNext.python3Packages; [ - (klein.overridePythonAttrs(old: { checkPhase = ""; })) - buildbot-pkg - ]; - nativeBuildInputs = with pkgsNext; [ yarn nodejs ]; - buildInputs = [ buildslist_yarn buildslist_bower ]; - - doCheck = false; - src = buildslist_src.src; - }; - buildbot_common = pkgsNext.python3Packages.buildPythonPackage (mylibs.fetchedGitPrivate ./buildbot_common.json // rec { + buildbot_common = pkgs.python3Packages.buildPythonPackage rec { + name = "buildbot_common"; + src = ./common; format = "other"; installPhase = '' - mkdir -p $out/${pkgsNext.python3.pythonForBuild.sitePackages} - cp -a $src $out/${pkgsNext.python3.pythonForBuild.sitePackages}/buildbot_common + mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages} + cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common ''; - }); - buildbot = pkgsNext.python3Packages.buildbot-full.withPlugins ([ buildslist ]); + }; + buildbot = pkgs.python3Packages.buildbot-full; in { options = { @@ -65,6 +34,7 @@ in group = "buildbot"; description = "Buildbot user"; home = varDir; + extraGroups = [ "keys" ]; }; services.myWebsites.tools.vhostConfs.git.extraConfig = lib.attrsets.mapAttrsToList (k: project: '' @@ -75,69 +45,152 @@ in ProxyPassReverse /buildbot/${project.name}/ unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/ Use LDAPConnect - Require ldap-group cn=users,cn=buildbot,ou=services,dc=immae,dc=eu + Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu SetEnvIf X-Url-Scheme https HTTPS=1 ProxyPreserveHost On - Require local + + Require local + Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu + Include /var/secrets/buildbot/${project.name}/webhook-httpd-include + '') myconfig.env.buildbot.projects; system.activationScripts = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { deps = [ "users" "wrappers" ]; - text = let - master-cfg = "${buildbot_common}/${pkgsNext.python3.pythonForBuild.sitePackages}/buildbot_common/master.cfg"; - puppet_notify = pkgs.writeText "puppet_notify" (builtins.readFile "${myconfig.privateFiles}/buildbot_puppet_notify"); - in '' + text = '' install -m 0755 -o buildbot -g buildbot -d /run/buildbot/ install -m 0755 -o buildbot -g buildbot -d ${varDir} - if [ ! -f ${varDir}/${project.name}/buildbot.tac ]; then - $wrapperDir/sudo -u buildbot ${buildbot}/bin/buildbot create-master -c "${master-cfg}" "${varDir}/${project.name}" - rm -f ${varDir}/${project.name}/master.cfg.sample - fi - install -Dm600 -o buildbot -g buildbot -T ${puppet_notify} ${varDir}/puppet_notify - buildbot_secrets=${varDir}/${project.name}/secrets - install -m 0600 -o buildbot -g buildbot -d $buildbot_secrets - echo "${myconfig.env.buildbot.ldap.password}" > $buildbot_secrets/ldap - ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList - (k: v: "echo ${lib.strings.escapeShellArg v} > $buildbot_secrets/${k}") project.secrets - )} - chown -R buildbot:buildbot $buildbot_secrets - chmod -R u=rX,go=- $buildbot_secrets ${project.activationScript} ''; }) myconfig.env.buildbot.projects; + secrets.keys = ( + lib.lists.flatten ( + lib.attrsets.mapAttrsToList (k: project: + lib.attrsets.mapAttrsToList (k: v: + { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = v; + dest = "buildbot/${project.name}/${k}"; + } + ) project.secrets + ++ [ + { + permissions = "0600"; + user = "wwwrun"; + group = "wwwrun"; + text = lib.optionalString (lib.attrsets.hasAttr "webhookTokens" project) '' + Require expr "req('Access-Key') in { ${builtins.concatStringsSep ", " (map (x: "'${x}'") project.webhookTokens)} }" + ''; + dest = "buildbot/${project.name}/webhook-httpd-include"; + } + ] + ) myconfig.env.buildbot.projects + ) + ) ++ [ + { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = myconfig.env.buildbot.ldap.password; + dest = "buildbot/ldap"; + } + { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = builtins.readFile "${myconfig.privateFiles}/buildbot_ssh_key"; + dest = "buildbot/ssh_key"; + } + ]; + systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { description = "Buildbot Continuous Integration Server ${project.name}."; after = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; - path = project.packages pkgs ++ (project.pythonPackages buildbot.pythonModule pkgsNext); + path = project.packages pkgs ++ (project.pythonPackages buildbot.pythonModule pkgs); + preStart = let + master-cfg = "${buildbot_common}/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common/master.cfg"; + tac_file = pkgs.writeText "buildbot.tac" '' + import os + + from twisted.application import service + from buildbot.master import BuildMaster + + basedir = '${varDir}/${project.name}' + rotateLength = 10000000 + maxRotatedFiles = 10 + configfile = '${master-cfg}' + + # Default umask for server + umask = None + + # if this is a relocatable tac file, get the directory containing the TAC + if basedir == '.': + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + + # note: this line is matched against to check that this is a buildmaster + # directory; do not edit it. + application = service.Application('buildmaster') + from twisted.python.logfile import LogFile + from twisted.python.log import ILogObserver, FileLogObserver + logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, + maxRotatedFiles=maxRotatedFiles) + application.setComponent(ILogObserver, FileLogObserver(logfile).emit) + + m = BuildMaster(basedir, configfile, umask) + m.setServiceParent(application) + m.log_rotation.rotateLength = rotateLength + m.log_rotation.maxRotatedFiles = maxRotatedFiles + ''; + in '' + if [ ! -f ${varDir}/${project.name}/buildbot.tac ]; then + ${buildbot}/bin/buildbot create-master -c "${master-cfg}" "${varDir}/${project.name}" + rm -f ${varDir}/${project.name}/master.cfg.sample + rm -f ${varDir}/${project.name}/buildbot.tac + fi + ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac + # different buildbots may be trying that simultaneously, add the || true to avoid complaining in case of race + install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ssh_key ${varDir}/buildbot_key || true + buildbot_secrets=${varDir}/${project.name}/secrets + install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets + install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap + ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList + (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets + )} + ''; environment = let project_env = lib.attrsets.mapAttrs' (k: v: lib.attrsets.nameValuePair "BUILDBOT_${k}" v) project.environment; - buildbot_config = pkgsNext.python3Packages.buildPythonPackage (rec { + buildbot_config = pkgs.python3Packages.buildPythonPackage (rec { name = "buildbot_config-${project.name}"; - src = "${./projects}/${project.name}"; + src = ./projects + "/${project.name}"; format = "other"; installPhase = '' - mkdir -p $out/${pkgsNext.python3.pythonForBuild.sitePackages} - cp -a $src $out/${pkgsNext.python3.pythonForBuild.sitePackages}/buildbot_config + mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages} + cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_config ''; }); HOME = "${varDir}/${project.name}"; - PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgsNext ++ [ - pkgsNext.python3Packages.treq pkgsNext.python3Packages.ldap3 buildbot - pkgsNext.python3Packages.buildbot-worker + PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [ + pkgs.python3Packages.wokkel + pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot + pkgs.python3Packages.buildbot-worker buildbot_common buildbot_config - ])}/${buildbot.pythonModule.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${pkgsNext.python3.pythonForBuild.sitePackages}" else ""}"; + ])}/${buildbot.pythonModule.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${pkgs.python3.pythonForBuild.sitePackages}" else ""}"; in project_env // { inherit PYTHONPATH HOME; }; serviceConfig = { Type = "forking"; User = "buildbot"; Group = "buildbot"; + SupplementaryGroups = "keys"; WorkingDirectory = "${varDir}/${project.name}"; ExecStart = "${buildbot}/bin/buildbot start"; };