nixops/modules/buildbot/default.nix

   1 { lib, pkgs, config, myconfig, mylibs, ... }:
   2 let
   3   varDir = "/var/lib/buildbot";
   4   buildslist_src = mylibs.fetchedGitPrivate ./buildslist.json;
   5   buildslist_yarn = mylibs.yarn2nixPackage.mkYarnModules rec {
   6     name = "buildslist-yarn-modules";
   7     pname = name;
   8     inherit (pkgs.buildbot-pkg) version;
   9     packageJSON = "${buildslist_src.src}/package.json";
  10     yarnLock = "${buildslist_src.src}/yarn.lock";
  11     pkgConfig = {
  12       all = { buildInputs = [ mylibs.yarn2nixPackage.src ]; };
  13     };
  14   };
  15   buildslist_bower = pkgs.buildBowerComponents {
  16     name = "buildslist";
  17     generated = ./bower.nix;
  18     src = "${buildslist_src.src}/guanlecoja/";
  19   };
  20
  21   buildslist = pkgs.python3Packages.buildPythonPackage rec {
  22     pname = "buildbot-buildslist";
  23     inherit (pkgs.buildbot-pkg) version;
  24
  25     preConfigure = ''
  26       export HOME=$PWD
  27       cp -a ${buildslist_yarn}/node_modules .
  28       chmod -R u+w node_modules
  29       cp -a ${buildslist_bower}/bower_components ./libs
  30       chmod -R u+w libs
  31       '';
  32     propagatedBuildInputs = with pkgs.python3Packages; [
  33       (klein.overridePythonAttrs(old: { checkPhase = ""; }))
  34       buildbot-pkg
  35     ];
  36     nativeBuildInputs = with pkgs; [ yarn nodejs ];
  37     buildInputs = [ buildslist_yarn buildslist_bower ];
  38
  39     doCheck = false;
  40     src = buildslist_src.src;
  41   };
  42   buildbot_common = pkgs.python3Packages.buildPythonPackage rec {
  43     name = "buildbot_common";
  44     src = ./common;
  45     format = "other";
  46     installPhase = ''
  47       mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages}
  48       cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common
  49       '';
  50   };
  51   buildbot = pkgs.python3Packages.buildbot-full.withPlugins ([ buildslist ]);
  52 in
  53 {
  54   options = {
  55     services.buildbot.enable = lib.mkOption {
  56       type = lib.types.bool;
  57       default = false;
  58       description = ''
  59         Whether to enable buildbot.
  60       '';
  61     };
  62   };
  63
  64   config = lib.mkIf config.services.buildbot.enable {
  65     nixpkgs.overlays = [ (self: super: rec {
  66       python3 = super.python3.override {
  67         packageOverrides = python-self: python-super: {
  68           wokkel = python-self.buildPythonPackage rec {
  69             pname = "wokkel";
  70             version = "18.0.0";
  71             src = python-self.fetchPypi {
  72               inherit pname version;
  73               sha256 = "1spq44gg8gsviqx1dvlmjpgfc0wk0jpyx4ap01y2pad1ai9cw016";
  74             };
  75             propagatedBuildInputs = with python-self; [ twisted.extras.tls twisted incremental dateutil ];
  76             doChecks = false;
  77           };
  78           apprise = python-self.buildPythonPackage rec {
  79             pname = "apprise";
  80             version = "0.7.4";
  81             src = (mylibs.fetchedGithub ./apprise.json).src;
  82             propagatedBuildInputs = with python-self; [ decorator
  83             requests requests_oauthlib oauthlib urllib3 six click
  84             markdown pyyaml sleekxmpp
  85             ];
  86             doChecks = false;
  87           };
  88         };
  89       };
  90     }) ];
  91
  92     ids.uids.buildbot = myconfig.env.buildbot.user.uid;
  93     ids.gids.buildbot = myconfig.env.buildbot.user.gid;
  94
  95     users.groups.buildbot.gid = config.ids.gids.buildbot;
  96     users.users.buildbot = {
  97       name = "buildbot";
  98       uid = config.ids.uids.buildbot;
  99       group = "buildbot";
 100       description = "Buildbot user";
 101       home = varDir;
 102       extraGroups = [ "keys" ];
 103     };
 104
 105     services.myWebsites.tools.vhostConfs.git.extraConfig = lib.attrsets.mapAttrsToList (k: project: ''
 106         RedirectMatch permanent "^/buildbot/${project.name}$" "/buildbot/${project.name}/"
 107         RewriteEngine On
 108         RewriteRule ^/buildbot/${project.name}/ws(.*)$   unix:///run/buildbot/${project.name}.sock|ws://git.immae.eu/ws$1 [P,NE,QSA,L]
 109         ProxyPass /buildbot/${project.name}/             unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
 110         ProxyPassReverse /buildbot/${project.name}/      unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
 111         <Location /buildbot/${project.name}/>
 112           Use LDAPConnect
 113           Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
 114
 115           SetEnvIf X-Url-Scheme https HTTPS=1
 116           ProxyPreserveHost On
 117         </Location>
 118         <Location /buildbot/${project.name}/change_hook/base>
 119           <RequireAny>
 120             Require local
 121             Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
 122             Include /var/secrets/buildbot/${project.name}/webhook-httpd-include
 123           </RequireAny>
 124         </Location>
 125         '') myconfig.env.buildbot.projects;
 126
 127     system.activationScripts = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
 128       deps = [ "users" "wrappers" ];
 129       text = ''
 130       install -m 0755 -o buildbot -g buildbot -d /run/buildbot/
 131       install -m 0755 -o buildbot -g buildbot -d ${varDir}
 132       ${project.activationScript}
 133       '';
 134     }) myconfig.env.buildbot.projects;
 135
 136     mySecrets.keys = (
 137       lib.lists.flatten (
 138         lib.attrsets.mapAttrsToList (k: project:
 139           lib.attrsets.mapAttrsToList (k: v:
 140             {
 141               permissions = "0600";
 142               user = "buildbot";
 143               group = "buildbot";
 144               text = v;
 145               dest = "buildbot/${project.name}/${k}";
 146             }
 147           ) project.secrets
 148           ++ [
 149             {
 150               permissions = "0600";
 151               user = "wwwrun";
 152               group = "wwwrun";
 153               text = lib.optionalString (lib.attrsets.hasAttr "webhookTokens" project) ''
 154                 Require expr "req('Access-Key') in { ${builtins.concatStringsSep ", " (map (x: "'${x}'") project.webhookTokens)} }"
 155                 '';
 156               dest = "buildbot/${project.name}/webhook-httpd-include";
 157             }
 158           ]
 159         ) myconfig.env.buildbot.projects
 160       )
 161     ) ++ [
 162       {
 163         permissions = "0600";
 164         user = "buildbot";
 165         group = "buildbot";
 166         text = myconfig.env.buildbot.ldap.password;
 167         dest = "buildbot/ldap";
 168       }
 169       {
 170         permissions = "0600";
 171         user = "buildbot";
 172         group = "buildbot";
 173         text = builtins.readFile "${myconfig.privateFiles}/buildbot_ssh_key";
 174         dest = "buildbot/ssh_key";
 175       }
 176     ];
 177
 178     systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
 179       description = "Buildbot Continuous Integration Server ${project.name}.";
 180       after = [ "network-online.target" ];
 181       wantedBy = [ "multi-user.target" ];
 182       path = project.packages pkgs ++ (project.pythonPackages buildbot.pythonModule pkgs);
 183       preStart = let
 184         master-cfg = "${buildbot_common}/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common/master.cfg";
 185         tac_file = pkgs.writeText "buildbot.tac" ''
 186             import os
 187
 188             from twisted.application import service
 189             from buildbot.master import BuildMaster
 190
 191             basedir = '${varDir}/${project.name}'
 192             rotateLength = 10000000
 193             maxRotatedFiles = 10
 194             configfile = '${master-cfg}'
 195
 196             # Default umask for server
 197             umask = None
 198
 199             # if this is a relocatable tac file, get the directory containing the TAC
 200             if basedir == '.':
 201                 import os
 202                 basedir = os.path.abspath(os.path.dirname(__file__))
 203
 204             # note: this line is matched against to check that this is a buildmaster
 205             # directory; do not edit it.
 206             application = service.Application('buildmaster')
 207             from twisted.python.logfile import LogFile
 208             from twisted.python.log import ILogObserver, FileLogObserver
 209             logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
 210                                             maxRotatedFiles=maxRotatedFiles)
 211             application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
 212
 213             m = BuildMaster(basedir, configfile, umask)
 214             m.setServiceParent(application)
 215             m.log_rotation.rotateLength = rotateLength
 216             m.log_rotation.maxRotatedFiles = maxRotatedFiles
 217           '';
 218       in ''
 219       if [ ! -f ${varDir}/${project.name}/buildbot.tac ]; then
 220         ${buildbot}/bin/buildbot create-master -c "${master-cfg}" "${varDir}/${project.name}"
 221         rm -f ${varDir}/${project.name}/master.cfg.sample
 222         rm -f ${varDir}/${project.name}/buildbot.tac
 223       fi
 224       ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac
 225       # different buildbots may be trying that simultaneously, add the || true to avoid complaining in case of race
 226       install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ssh_key ${varDir}/buildbot_key || true
 227       buildbot_secrets=${varDir}/${project.name}/secrets
 228       install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
 229       install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap
 230       ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
 231         (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets
 232       )}
 233       '';
 234       environment = let
 235         project_env = lib.attrsets.mapAttrs' (k: v: lib.attrsets.nameValuePair "BUILDBOT_${k}" v) project.environment;
 236         buildbot_config = pkgs.python3Packages.buildPythonPackage (rec {
 237           name = "buildbot_config-${project.name}";
 238           src = ./projects + "/${project.name}";
 239           format = "other";
 240           installPhase = ''
 241             mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages}
 242             cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_config
 243             '';
 244         });
 245         HOME = "${varDir}/${project.name}";
 246         PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
 247           pkgs.python3Packages.wokkel
 248           pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
 249           pkgs.python3Packages.buildbot-worker
 250           buildbot_common buildbot_config
 251         ])}/${buildbot.pythonModule.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${pkgs.python3.pythonForBuild.sitePackages}" else ""}";
 252       in project_env // { inherit PYTHONPATH HOME; };
 253
 254       serviceConfig = {
 255         Type = "forking";
 256         User = "buildbot";
 257         Group = "buildbot";
 258         SupplementaryGroups = "keys";
 259         WorkingDirectory = "${varDir}/${project.name}";
 260         ExecStart = "${buildbot}/bin/buildbot start";
 261       };
 262     }) myconfig.env.buildbot.projects;
 263   };
 264 }