nixops/modules/buildbot/default.nix

   1 { lib, pkgs, config, myconfig, mylibs, ... }:
   2 let
   3   varDir = "/var/lib/buildbot";
   4   buildslist_src = mylibs.fetchedGitPrivate ./buildslist.json;
   5   buildslist_yarn = mylibs.yarn2nixPackage.mkYarnModules rec {
   6     name = "buildslist-yarn-modules";
   7     pname = name;
   8     inherit (pkgs.buildbot-pkg) version;
   9     packageJSON = "${buildslist_src.src}/package.json";
  10     yarnLock = "${buildslist_src.src}/yarn.lock";
  11   };
  12   buildslist_bower = pkgs.buildBowerComponents {
  13     name = "buildslist";
  14     generated = ./bower.nix;
  15     src = "${buildslist_src.src}/guanlecoja/";
  16   };
  17
  18   buildslist = pkgs.python3Packages.buildPythonPackage rec {
  19     pname = "buildbot-buildslist";
  20     inherit (pkgs.buildbot-pkg) version;
  21
  22     preConfigure = ''
  23       export HOME=$PWD
  24       cp -a ${buildslist_yarn}/node_modules .
  25       chmod -R u+w node_modules
  26       cp -a ${buildslist_bower}/bower_components ./libs
  27       chmod -R u+w libs
  28       '';
  29     propagatedBuildInputs = with pkgs.python3Packages; [
  30       (klein.overridePythonAttrs(old: { checkPhase = ""; }))
  31       buildbot-pkg
  32     ];
  33     nativeBuildInputs = with pkgs; [ yarn nodejs ];
  34     buildInputs = [ buildslist_yarn buildslist_bower ];
  35
  36     doCheck = false;
  37     src = buildslist_src.src;
  38   };
  39   buildbot_common = pkgs.python3Packages.buildPythonPackage rec {
  40     name = "buildbot_common";
  41     src = ./common;
  42     format = "other";
  43     installPhase = ''
  44       mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages}
  45       cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common
  46       '';
  47   };
  48   buildbot = pkgs.python3Packages.buildbot-full.withPlugins ([ buildslist ]);
  49 in
  50 {
  51   options = {
  52     services.buildbot.enable = lib.mkOption {
  53       type = lib.types.bool;
  54       default = false;
  55       description = ''
  56         Whether to enable buildbot.
  57       '';
  58     };
  59   };
  60
  61   config = lib.mkIf config.services.buildbot.enable {
  62     nixpkgs.overlays = [ (self: super: rec {
  63       python3 = super.python3.override {
  64         packageOverrides = python-self: python-super: {
  65           wokkel = python-self.buildPythonPackage rec {
  66             pname = "wokkel";
  67             version = "18.0.0";
  68             src = python-self.fetchPypi {
  69               inherit pname version;
  70               sha256 = "1spq44gg8gsviqx1dvlmjpgfc0wk0jpyx4ap01y2pad1ai9cw016";
  71             };
  72             propagatedBuildInputs = with python-self; [ twisted.extras.tls twisted incremental dateutil ];
  73             doChecks = false;
  74           };
  75           apprise = python-self.buildPythonPackage rec {
  76             pname = "apprise";
  77             version = "0.7.4";
  78             src = (mylibs.fetchedGithub ./apprise.json).src;
  79             propagatedBuildInputs = with python-self; [ decorator
  80             requests requests_oauthlib oauthlib urllib3 six click
  81             markdown pyyaml sleekxmpp
  82             ];
  83             doChecks = false;
  84           };
  85         };
  86       };
  87     }) ];
  88
  89     ids.uids.buildbot = myconfig.env.buildbot.user.uid;
  90     ids.gids.buildbot = myconfig.env.buildbot.user.gid;
  91
  92     users.groups.buildbot.gid = config.ids.gids.buildbot;
  93     users.users.buildbot = {
  94       name = "buildbot";
  95       uid = config.ids.uids.buildbot;
  96       group = "buildbot";
  97       description = "Buildbot user";
  98       home = varDir;
  99     };
 100
 101     services.myWebsites.tools.vhostConfs.git.extraConfig = lib.attrsets.mapAttrsToList (k: project: ''
 102         RedirectMatch permanent "^/buildbot/${project.name}$" "/buildbot/${project.name}/"
 103         RewriteEngine On
 104         RewriteRule ^/buildbot/${project.name}/ws(.*)$   unix:///run/buildbot/${project.name}.sock|ws://git.immae.eu/ws$1 [P,NE,QSA,L]
 105         ProxyPass /buildbot/${project.name}/             unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
 106         ProxyPassReverse /buildbot/${project.name}/      unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
 107         <Location /buildbot/${project.name}/>
 108           Use LDAPConnect
 109           Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
 110
 111           SetEnvIf X-Url-Scheme https HTTPS=1
 112           ProxyPreserveHost On
 113         </Location>
 114         <Location /buildbot/${project.name}/change_hook/base>
 115           <RequireAny>
 116             Require local
 117             Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
 118             ${if lib.attrsets.hasAttr "webhookTokens" project then ''
 119               Require expr "req('Access-Key') in { ${builtins.concatStringsSep ", " (map (x: "'${x}'") project.webhookTokens)} }"
 120               '' else ""}
 121           </RequireAny>
 122         </Location>
 123         '') myconfig.env.buildbot.projects;
 124
 125     system.activationScripts = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
 126       deps = [ "users" "wrappers" ];
 127       text = let
 128         master-cfg = "${buildbot_common}/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common/master.cfg";
 129         buildbot_key = pkgs.writeText "buildbot_key" (builtins.readFile "${myconfig.privateFiles}/buildbot_ssh_key");
 130         tac_file = pkgs.writeText "buildbot.tac" ''
 131             import os
 132
 133             from twisted.application import service
 134             from buildbot.master import BuildMaster
 135
 136             basedir = '${varDir}/${project.name}'
 137             rotateLength = 10000000
 138             maxRotatedFiles = 10
 139             configfile = '${master-cfg}'
 140
 141             # Default umask for server
 142             umask = None
 143
 144             # if this is a relocatable tac file, get the directory containing the TAC
 145             if basedir == '.':
 146                 import os
 147                 basedir = os.path.abspath(os.path.dirname(__file__))
 148
 149             # note: this line is matched against to check that this is a buildmaster
 150             # directory; do not edit it.
 151             application = service.Application('buildmaster')
 152             from twisted.python.logfile import LogFile
 153             from twisted.python.log import ILogObserver, FileLogObserver
 154             logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
 155                                             maxRotatedFiles=maxRotatedFiles)
 156             application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
 157
 158             m = BuildMaster(basedir, configfile, umask)
 159             m.setServiceParent(application)
 160             m.log_rotation.rotateLength = rotateLength
 161             m.log_rotation.maxRotatedFiles = maxRotatedFiles
 162           '';
 163       in ''
 164       install -m 0755 -o buildbot -g buildbot -d /run/buildbot/
 165       install -m 0755 -o buildbot -g buildbot -d ${varDir}
 166       if [ ! -f ${varDir}/${project.name}/buildbot.tac ]; then
 167         $wrapperDir/sudo -u buildbot ${buildbot}/bin/buildbot create-master -c "${master-cfg}" "${varDir}/${project.name}"
 168         rm -f ${varDir}/${project.name}/master.cfg.sample
 169         rm -f ${varDir}/${project.name}/buildbot.tac
 170       fi
 171       ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac
 172       install -Dm600 -o buildbot -g buildbot -T ${buildbot_key} ${varDir}/buildbot_key
 173       buildbot_secrets=${varDir}/${project.name}/secrets
 174       install -m 0600 -o buildbot -g buildbot -d $buildbot_secrets
 175       echo "${myconfig.env.buildbot.ldap.password}" > $buildbot_secrets/ldap
 176       ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
 177         (k: v: "echo ${lib.strings.escapeShellArg v} > $buildbot_secrets/${k}") project.secrets
 178       )}
 179       chown -R buildbot:buildbot $buildbot_secrets
 180       chmod -R u=rX,go=- $buildbot_secrets
 181       ${project.activationScript}
 182       '';
 183     }) myconfig.env.buildbot.projects;
 184
 185     systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
 186       description = "Buildbot Continuous Integration Server ${project.name}.";
 187       after = [ "network-online.target" ];
 188       wantedBy = [ "multi-user.target" ];
 189       path = project.packages pkgs ++ (project.pythonPackages buildbot.pythonModule pkgs);
 190       environment = let
 191         project_env = lib.attrsets.mapAttrs' (k: v: lib.attrsets.nameValuePair "BUILDBOT_${k}" v) project.environment;
 192         buildbot_config = pkgs.python3Packages.buildPythonPackage (rec {
 193           name = "buildbot_config-${project.name}";
 194           src = ./projects + "/${project.name}";
 195           format = "other";
 196           installPhase = ''
 197             mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages}
 198             cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_config
 199             '';
 200         });
 201         HOME = "${varDir}/${project.name}";
 202         PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
 203           pkgs.python3Packages.wokkel
 204           pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
 205           pkgs.python3Packages.buildbot-worker
 206           buildbot_common buildbot_config
 207         ])}/${buildbot.pythonModule.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${pkgs.python3.pythonForBuild.sitePackages}" else ""}";
 208       in project_env // { inherit PYTHONPATH HOME; };
 209
 210       serviceConfig = {
 211         Type = "forking";
 212         User = "buildbot";
 213         Group = "buildbot";
 214         WorkingDirectory = "${varDir}/${project.name}";
 215         ExecStart = "${buildbot}/bin/buildbot start";
 216       };
 217     }) myconfig.env.buildbot.projects;
 218   };
 219 }