]> git.immae.eu Git - perso/Immae/Config/Nix.git/blame - modules/private/buildbot/default.nix
Fix maison bbc website and monitoring
[perso/Immae/Config/Nix.git] / modules / private / buildbot / default.nix
CommitLineData
ab8f306d 1{ lib, pkgs, config, ... }:
9fb4205e
IB
2let
3 varDir = "/var/lib/buildbot";
caa08508 4 buildbot_common = pkgs.python3Packages.buildPythonPackage rec {
e2b96bf5
IB
5 name = "buildbot_common";
6 src = ./common;
9fb4205e
IB
7 format = "other";
8 installPhase = ''
caa08508
IB
9 mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages}
10 cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common
9fb4205e 11 '';
e2b96bf5 12 };
b798cf6d 13 buildbot = pkgs.python3Packages.buildbot-full;
9fb4205e
IB
14in
15{
16 options = {
8d213e2b 17 myServices.buildbot.enable = lib.mkOption {
9fb4205e
IB
18 type = lib.types.bool;
19 default = false;
20 description = ''
21 Whether to enable buildbot.
22 '';
23 };
24 };
25
8d213e2b 26 config = lib.mkIf config.myServices.buildbot.enable {
d2e703c5 27 services.duplyBackup.profiles.buildbot = {
6a8252b1
IB
28 rootDir = varDir;
29 };
ab8f306d
IB
30 ids.uids.buildbot = config.myEnv.buildbot.user.uid;
31 ids.gids.buildbot = config.myEnv.buildbot.user.gid;
9fb4205e
IB
32
33 users.groups.buildbot.gid = config.ids.gids.buildbot;
34 users.users.buildbot = {
35 name = "buildbot";
36 uid = config.ids.uids.buildbot;
37 group = "buildbot";
38 description = "Buildbot user";
39 home = varDir;
6984f454 40 extraGroups = [ "keys" ];
9fb4205e
IB
41 };
42
29f8cb85 43 services.websites.env.tools.watchPaths = lib.attrsets.mapAttrsToList
17f6eae9 44 (k: project: "/var/secrets/buildbot/${project.name}/webhook-httpd-include")
ab8f306d 45 config.myEnv.buildbot.projects;
17f6eae9 46
29f8cb85 47 services.websites.env.tools.vhostConfs.git.extraConfig = lib.attrsets.mapAttrsToList (k: project: ''
9fb4205e
IB
48 RedirectMatch permanent "^/buildbot/${project.name}$" "/buildbot/${project.name}/"
49 RewriteEngine On
50 RewriteRule ^/buildbot/${project.name}/ws(.*)$ unix:///run/buildbot/${project.name}.sock|ws://git.immae.eu/ws$1 [P,NE,QSA,L]
51 ProxyPass /buildbot/${project.name}/ unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
52 ProxyPassReverse /buildbot/${project.name}/ unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/
53 <Location /buildbot/${project.name}/>
54 Use LDAPConnect
e2b96bf5 55 Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
9fb4205e
IB
56
57 SetEnvIf X-Url-Scheme https HTTPS=1
58 ProxyPreserveHost On
59 </Location>
60 <Location /buildbot/${project.name}/change_hook/base>
85817848
IB
61 <RequireAny>
62 Require local
63 Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu
ca330baa 64 Include /var/secrets/buildbot/${project.name}/webhook-httpd-include
85817848 65 </RequireAny>
9fb4205e 66 </Location>
ab8f306d 67 '') config.myEnv.buildbot.projects;
9fb4205e
IB
68
69 system.activationScripts = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
70 deps = [ "users" "wrappers" ];
8fa7ff2c
IB
71 text = ''
72 install -m 755 -o buildbot -g buildbot -d ${varDir}/${project.name}
73
74 ${project.activationScript}
75 '';
ab8f306d 76 }) config.myEnv.buildbot.projects;
6984f454 77
1a718805 78 secrets.keys = (
6984f454
IB
79 lib.lists.flatten (
80 lib.attrsets.mapAttrsToList (k: project:
81 lib.attrsets.mapAttrsToList (k: v:
ca330baa 82 {
6984f454
IB
83 permissions = "0600";
84 user = "buildbot";
85 group = "buildbot";
86 text = v;
ca330baa 87 dest = "buildbot/${project.name}/${k}";
6984f454
IB
88 }
89 ) project.secrets
90 ++ [
ca330baa 91 {
6984f454
IB
92 permissions = "0600";
93 user = "wwwrun";
94 group = "wwwrun";
ab8f306d 95 text = lib.optionalString (project.webhookTokens != null) ''
6984f454
IB
96 Require expr "req('Access-Key') in { ${builtins.concatStringsSep ", " (map (x: "'${x}'") project.webhookTokens)} }"
97 '';
ca330baa
IB
98 dest = "buildbot/${project.name}/webhook-httpd-include";
99 }
6984f454 100 ]
ab8f306d 101 ) config.myEnv.buildbot.projects
6984f454 102 )
ca330baa
IB
103 ) ++ [
104 {
6984f454
IB
105 permissions = "0600";
106 user = "buildbot";
107 group = "buildbot";
ab8f306d 108 text = config.myEnv.buildbot.ldap.password;
ca330baa
IB
109 dest = "buildbot/ldap";
110 }
111 {
6984f454
IB
112 permissions = "0600";
113 user = "buildbot";
114 group = "buildbot";
ab8f306d 115 text = builtins.readFile "${config.myEnv.privateFiles}/buildbot_ssh_key";
ca330baa
IB
116 dest = "buildbot/ssh_key";
117 }
118 ];
6984f454 119
17f6eae9
IB
120 services.filesWatcher = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
121 restart = true;
122 paths = [
123 "/var/secrets/buildbot/ldap"
124 "/var/secrets/buildbot/ssh_key"
125 ] ++ lib.attrsets.mapAttrsToList (k: v: "/var/secrets/buildbot/${project.name}/${k}") project.secrets;
ab8f306d 126 }) config.myEnv.buildbot.projects;
17f6eae9 127
6984f454
IB
128 systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" {
129 description = "Buildbot Continuous Integration Server ${project.name}.";
ca330baa 130 after = [ "network-online.target" ];
6984f454
IB
131 wantedBy = [ "multi-user.target" ];
132 path = project.packages pkgs ++ (project.pythonPackages buildbot.pythonModule pkgs);
133 preStart = let
caa08508 134 master-cfg = "${buildbot_common}/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common/master.cfg";
e2b96bf5
IB
135 tac_file = pkgs.writeText "buildbot.tac" ''
136 import os
137
138 from twisted.application import service
139 from buildbot.master import BuildMaster
140
141 basedir = '${varDir}/${project.name}'
142 rotateLength = 10000000
143 maxRotatedFiles = 10
144 configfile = '${master-cfg}'
145
146 # Default umask for server
147 umask = None
148
149 # if this is a relocatable tac file, get the directory containing the TAC
150 if basedir == '.':
151 import os
152 basedir = os.path.abspath(os.path.dirname(__file__))
153
154 # note: this line is matched against to check that this is a buildmaster
155 # directory; do not edit it.
156 application = service.Application('buildmaster')
157 from twisted.python.logfile import LogFile
158 from twisted.python.log import ILogObserver, FileLogObserver
159 logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
160 maxRotatedFiles=maxRotatedFiles)
161 application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
162
163 m = BuildMaster(basedir, configfile, umask)
164 m.setServiceParent(application)
165 m.log_rotation.rotateLength = rotateLength
166 m.log_rotation.maxRotatedFiles = maxRotatedFiles
167 '';
9fb4205e 168 in ''
9fb4205e 169 if [ ! -f ${varDir}/${project.name}/buildbot.tac ]; then
6984f454 170 ${buildbot}/bin/buildbot create-master -c "${master-cfg}" "${varDir}/${project.name}"
9fb4205e 171 rm -f ${varDir}/${project.name}/master.cfg.sample
e2b96bf5 172 rm -f ${varDir}/${project.name}/buildbot.tac
9fb4205e 173 fi
e2b96bf5 174 ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac
ca330baa
IB
175 # different buildbots may be trying that simultaneously, add the || true to avoid complaining in case of race
176 install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ssh_key ${varDir}/buildbot_key || true
9fb4205e 177 buildbot_secrets=${varDir}/${project.name}/secrets
6984f454 178 install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets
ca330baa 179 install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap
9fb4205e 180 ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList
ca330baa 181 (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets
9fb4205e 182 )}
9fb4205e 183 '';
9fb4205e 184 environment = let
0012da0f
IB
185 project_env = with lib.attrsets;
186 mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment //
60d328e2
IB
187 mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) //
188 { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; };
caa08508 189 buildbot_config = pkgs.python3Packages.buildPythonPackage (rec {
9fb4205e 190 name = "buildbot_config-${project.name}";
e2b96bf5 191 src = ./projects + "/${project.name}";
9fb4205e
IB
192 format = "other";
193 installPhase = ''
caa08508
IB
194 mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages}
195 cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_config
9fb4205e
IB
196 '';
197 });
198 HOME = "${varDir}/${project.name}";
caa08508 199 PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [
256d607c 200 pkgs.python3Packages.wokkel
caa08508
IB
201 pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot
202 pkgs.python3Packages.buildbot-worker
9fb4205e 203 buildbot_common buildbot_config
caa08508 204 ])}/${buildbot.pythonModule.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${pkgs.python3.pythonForBuild.sitePackages}" else ""}";
9fb4205e
IB
205 in project_env // { inherit PYTHONPATH HOME; };
206
207 serviceConfig = {
208 Type = "forking";
209 User = "buildbot";
210 Group = "buildbot";
81b9ff89
IB
211 RuntimeDirectory = "buildbot";
212 RuntimeDirectoryPreserve = "yes";
213 StateDirectory = "buildbot";
6984f454 214 SupplementaryGroups = "keys";
9fb4205e
IB
215 WorkingDirectory = "${varDir}/${project.name}";
216 ExecStart = "${buildbot}/bin/buildbot start";
217 };
ab8f306d 218 }) config.myEnv.buildbot.projects;
9fb4205e
IB
219 };
220}