nixops/modules/buildbot/default.nix

   1 { lib, pkgs, config, myconfig, mylibs, ... }:
   2 let
   3   varDir = "/var/lib/buildbot";
   4   buildslist_src = mylibs.fetchedGitPrivate ./buildslist.json;
   5   buildslist_yarn = mylibs.yarn2nixPackage.mkYarnModules rec {
   6     name = "buildslist-yarn-modules";
   7     pname = name;
   8     inherit (pkgs.buildbot-pkg) version;
   9     packageJSON = "${buildslist_src.src}/package.json";
  10     yarnLock = "${buildslist_src.src}/yarn.lock";
  11   };
  12   buildslist_bower = pkgs.buildBowerComponents {
  13     name = "buildslist";
  14     generated = ./bower.nix;
  15     src = "${buildslist_src.src}/guanlecoja/";
  16   };
  17
  18   buildslist = pkgs.python3Packages.buildPythonPackage rec {
  19     pname = "buildbot-buildslist";
  20     inherit (pkgs.buildbot-pkg) version;
  21
  22     preConfigure = ''
  23       export HOME=$PWD
  24       cp -a ${buildslist_yarn}/node_modules .
  25       chmod -R u+w node_modules
  26       cp -a ${buildslist_bower}/bower_components ./libs
  27       chmod -R u+w libs
  28       '';
  29     propagatedBuildInputs = with pkgs.python3Packages; [
  30       (klein.overridePythonAttrs(old: { checkPhase = ""; }))
  31       buildbot-pkg
  32     ];
  33     nativeBuildInputs = with pkgs; [ yarn nodejs ];
  34     buildInputs = [ buildslist_yarn buildslist_bower ];
  35
  36     doCheck = false;
  37     src = buildslist_src.src;
  38   };
  39   buildbot_common = pkgs.python3Packages.buildPythonPackage rec {
  40     name = "buildbot_common";
  41     src = ./common;
  42     format = "other";
  43     installPhase = ''
  44       mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages}
  45       cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common
  46       '';
  47   };
  48   buildbot = pkgs.python3Packages.buildbot-full.withPlugins ([ buildslist ]);
  49 in
  50 {
  51   options = {
  52     services.buildbot.enable = lib.mkOption {
  53       type = lib.types.bool;
  54       default = false;
  55       description = ''
  56         Whether to enable buildbot.
  57       '';
  58     };
  59   };
  60
  61   config = lib.mkIf config.services.buildbot.enable {
  62     nixpkgs.overlays = [ (self: super: rec {
  63       python3 = super.python3.override {
  64         packageOverrides = python-self: python-super: {
  65           wokkel = python-self.buildPythonPackage rec {
  66             pname = "wokkel";
  67             version = "18.0.0";
  68             src = python-self.fetchPypi {
  69               inherit pname version;
  70               sha256 = "1spq44gg8gsviqx1dvlmjpgfc0wk0jpyx4ap01y2pad1ai9cw016";
  71             };
  72             propagatedBuildInputs = with python-self; [ twisted.extras.tls twisted incremental dateutil ];
  73             doChecks = false;
  74           };
  75           apprise = python-self.buildPythonPackage rec {
  76             pname = "apprise";
  77             version = "0.7.4";
  78             src = (mylibs.fetchedGithub ./apprise.json).src;
  79             propagatedBuildInputs = with python-self; [ decorator
  80             requests requests_oauthlib oauthlib urllib3 six click
  81             markdown pyyaml sleekxmpp
  82             ];
  83             doChecks = false;
  84           };
  85         };
  86       };
  87     }) ];
  88
  89     ids.uids.buildbot = myconfig.env.buildbot.user.uid;
  90     ids.gids.buildbot = myconfig.env.buildbot.user.gid;
  91
  92     users.groups.buildbot.gid = config.ids.gids.buildbot;
  93     users.users.buildbot = {
  94       name = "buildbot";
  95       uid = config.ids.uids.buildbot;
  96       group = "buildbot";
  97       description = "Buildbot user";
  98       home = varDir;
  99       extraGroups = [ "keys" ];
 100     };
 101
 102     services.myWebsites.tools.vhostConfs.git.extraConfig = lib.attrsets.mapAttrsToList (k: project: ''
 103         RedirectMatch permanent "^/buildbot/${project.name}$" "/buildbot/${project.name}/"
 104         RewriteEngine On
 105         RewriteRule ^/buildbot/${project.name}/ws(.*)$   unix:///run/buildbot/${project.name}.sock|ws://git.immae.eu/ws$1 [P,NE,QSA,L]
 106         ProxyPass /buildbot/${project.name}/             unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
 107         ProxyPassReverse /buildbot/${project.name}/      unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
 108         <Location /buildbot/${project.name}/>
 109           Use LDAPConnect
 110           Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
 111
 112           SetEnvIf X-Url-Scheme https HTTPS=1
 113           ProxyPreserveHost On
 114         </Location>
 115         <Location /buildbot/${project.name}/change_hook/base>
 116           <RequireAny>
 117             Require local
 118             Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
 119             Include /run/keys/buildbot/${project.name}/buildbot-${project.name}-webhook-httpd-include
 120           </RequireAny>
 121         </Location>
 122         '') myconfig.env.buildbot.projects;
 123
 124     system.activationScripts = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
 125       deps = [ "users" "wrappers" ];
 126       text = ''
 127       install -m 0755 -o buildbot -g buildbot -d /run/buildbot/
 128       install -m 0755 -o buildbot -g buildbot -d ${varDir}
 129       ${project.activationScript}
 130       '';
 131     }) myconfig.env.buildbot.projects;
 132
 133     deployment.keys = lib.attrsets.listToAttrs (
 134       lib.lists.flatten (
 135         lib.attrsets.mapAttrsToList (k: project:
 136           lib.attrsets.mapAttrsToList (k: v:
 137             lib.attrsets.nameValuePair "buildbot-${project.name}-${k}" {
 138               permissions = "0600";
 139               user = "buildbot";
 140               group = "buildbot";
 141               text = v;
 142               destDir = "/run/keys/buildbot/${project.name}";
 143             }
 144           ) project.secrets
 145           ++ [
 146             (lib.attrsets.nameValuePair "buildbot-${project.name}-webhook-httpd-include" {
 147               permissions = "0600";
 148               user = "wwwrun";
 149               group = "wwwrun";
 150               text = lib.optionalString (lib.attrsets.hasAttr "webhookTokens" project) ''
 151                 Require expr "req('Access-Key') in { ${builtins.concatStringsSep ", " (map (x: "'${x}'") project.webhookTokens)} }"
 152                 '';
 153               destDir = "/run/keys/buildbot/${project.name}";
 154             })
 155           ]
 156         ) myconfig.env.buildbot.projects
 157       )
 158     ) // {
 159       buildbot-ldap = {
 160         permissions = "0600";
 161         user = "buildbot";
 162         group = "buildbot";
 163         text = myconfig.env.buildbot.ldap.password;
 164         destDir = "/run/keys/buildbot";
 165       };
 166       buildbot-ssh-key = {
 167         permissions = "0600";
 168         user = "buildbot";
 169         group = "buildbot";
 170         text = builtins.readFile "${myconfig.privateFiles}/buildbot_ssh_key";
 171         destDir = "/run/keys/buildbot";
 172       };
 173     };
 174
 175     systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
 176       description = "Buildbot Continuous Integration Server ${project.name}.";
 177       after = [ "network-online.target" "keys.target" ];
 178       wants = [ "keys.target" ];
 179       wantedBy = [ "multi-user.target" ];
 180       path = project.packages pkgs ++ (project.pythonPackages buildbot.pythonModule pkgs);
 181       preStart = let
 182         master-cfg = "${buildbot_common}/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common/master.cfg";
 183         tac_file = pkgs.writeText "buildbot.tac" ''
 184             import os
 185
 186             from twisted.application import service
 187             from buildbot.master import BuildMaster
 188
 189             basedir = '${varDir}/${project.name}'
 190             rotateLength = 10000000
 191             maxRotatedFiles = 10
 192             configfile = '${master-cfg}'
 193
 194             # Default umask for server
 195             umask = None
 196
 197             # if this is a relocatable tac file, get the directory containing the TAC
 198             if basedir == '.':
 199                 import os
 200                 basedir = os.path.abspath(os.path.dirname(__file__))
 201
 202             # note: this line is matched against to check that this is a buildmaster
 203             # directory; do not edit it.
 204             application = service.Application('buildmaster')
 205             from twisted.python.logfile import LogFile
 206             from twisted.python.log import ILogObserver, FileLogObserver
 207             logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
 208                                             maxRotatedFiles=maxRotatedFiles)
 209             application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
 210
 211             m = BuildMaster(basedir, configfile, umask)
 212             m.setServiceParent(application)
 213             m.log_rotation.rotateLength = rotateLength
 214             m.log_rotation.maxRotatedFiles = maxRotatedFiles
 215           '';
 216       in ''
 217       if [ ! -f ${varDir}/${project.name}/buildbot.tac ]; then
 218         ${buildbot}/bin/buildbot create-master -c "${master-cfg}" "${varDir}/${project.name}"
 219         rm -f ${varDir}/${project.name}/master.cfg.sample
 220         rm -f ${varDir}/${project.name}/buildbot.tac
 221       fi
 222       ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac
 223       install -Dm600 -o buildbot -g buildbot -T /run/keys/buildbot/buildbot-ssh-key ${varDir}/buildbot_key
 224       buildbot_secrets=${varDir}/${project.name}/secrets
 225       install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
 226       install -Dm600 -o buildbot -g buildbot -T /run/keys/buildbot/buildbot-ldap $buildbot_secrets/ldap
 227       ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
 228         (k: v: "install -Dm600 -o buildbot -g buildbot -T /run/keys/buildbot/${project.name}/buildbot-${project.name}-${k} $buildbot_secrets/${k}") project.secrets
 229       )}
 230       '';
 231       environment = let
 232         project_env = lib.attrsets.mapAttrs' (k: v: lib.attrsets.nameValuePair "BUILDBOT_${k}" v) project.environment;
 233         buildbot_config = pkgs.python3Packages.buildPythonPackage (rec {
 234           name = "buildbot_config-${project.name}";
 235           src = ./projects + "/${project.name}";
 236           format = "other";
 237           installPhase = ''
 238             mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages}
 239             cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_config
 240             '';
 241         });
 242         HOME = "${varDir}/${project.name}";
 243         PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
 244           pkgs.python3Packages.wokkel
 245           pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
 246           pkgs.python3Packages.buildbot-worker
 247           buildbot_common buildbot_config
 248         ])}/${buildbot.pythonModule.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${pkgs.python3.pythonForBuild.sitePackages}" else ""}";
 249       in project_env // { inherit PYTHONPATH HOME; };
 250
 251       serviceConfig = {
 252         Type = "forking";
 253         User = "buildbot";
 254         Group = "buildbot";
 255         SupplementaryGroups = "keys";
 256         WorkingDirectory = "${varDir}/${project.name}";
 257         ExecStart = "${buildbot}/bin/buildbot start";
 258       };
 259     }) myconfig.env.buildbot.projects;
 260   };
 261 }