]> git.immae.eu Git - perso/Immae/Config/Nix.git/blame - modules/private/buildbot/common/build_helpers.py
Fix ttrss not synchronizing
[perso/Immae/Config/Nix.git] / modules / private / buildbot / common / build_helpers.py
CommitLineData
e2b96bf5
IB
1from buildbot.plugins import util, steps, schedulers
2from buildbot_buildslist import BuildsList
6b017278 3from shutil import which
e2b96bf5
IB
4
5__all__ = [
f6e48e3a 6 "force_scheduler", "deploy_scheduler", "git_hook_scheduler",
256d607c 7 "clean_branch", "package_and_upload", "SlackStatusPush",
bd0cb07b 8 "XMPPStatusPush", "NixShellCommand",
f6e48e3a
IB
9 "all_builder_names", "compute_build_infos", "deploy_ssh_command",
10 "configure_slack_push", "configure_xmpp_push", "deploy_hook_scheduler",
e2b96bf5
IB
11 ]
12
13# Small helpers"
14@util.renderer
15def clean_branch(props):
16 if props.hasProperty("branch") and len(props["branch"]) > 0:
17 return props["branch"].replace("/", "_")
18 else:
19 return "HEAD"
20
21def package_and_upload(package, package_dest, package_url):
22 return [
23 steps.ShellCommand(name="build package",
f6e48e3a 24 logEnviron=False, haltOnFailure=True,
e2b96bf5
IB
25 command=["git", "archive", "HEAD", "-o", package]),
26
27 steps.FileUpload(name="upload package", workersrc=package,
f6e48e3a 28 masterdest=package_dest,
e2b96bf5
IB
29 url=package_url, mode=0o644),
30
31 steps.ShellCommand(name="cleanup package", logEnviron=False,
f6e48e3a 32 haltOnFailure=True, alwaysRun=True,
e2b96bf5
IB
33 command=["rm", "-f", package]),
34 ]
35
6b017278
IB
36# Steps
37class NixShellCommand(steps.ShellCommand):
bd0cb07b 38 def __init__(self, command=None, nixPackages=[], pure=True, nixFile=None, nixIncludes={}, nixArgs={}, **kwargs):
6b017278
IB
39 oldpath = kwargs.get("env", {}).get("PATH", None)
40 if which("nix-shell", path=oldpath) is None:
41 kwargs["env"] = kwargs.get("env", {})
42 if isinstance(oldpath, str):
43 kwargs["env"]["PATH"] = "/run/current-system/sw/bin:" + oldpath
44 elif isinstance(oldpath, list):
45 kwargs["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath
46 nixcommand = ["nix-shell"]
bd0cb07b
IB
47 for k, v in nixArgs.items():
48 nixcommand.append("--arg")
49 nixcommand.append(k)
50 nixcommand.append(v)
6b017278
IB
51 if pure:
52 nixcommand.append("--pure")
bd0cb07b
IB
53 for k, v in nixIncludes.items():
54 nixcommand.append("-I")
55 nixcommand.append("{}={}".format(k, v))
6b017278
IB
56 nixcommand.append("--run")
57 nixcommand.append(command)
bd0cb07b
IB
58 if len(nixPackages) > 0:
59 nixcommand.append("-p")
60 nixcommand += nixPackages
61 elif nixFile is not None:
62 nixcommand.append(nixFile)
6b017278
IB
63 super().__init__(command=nixcommand, **kwargs)
64
e2b96bf5 65# Schedulers
f6e48e3a
IB
66def force_scheduler(name, builders, nobranch=False):
67 if nobranch:
68 branch = util.FixedParameter(name="branch", default="")
69 else:
70 branch=util.StringParameter(name="branch", label="Git reference (tag, branch)", required=True)
71
e2b96bf5
IB
72 return schedulers.ForceScheduler(name=name,
73 label="Force build", buttonName="Force build",
74 reason=util.StringParameter(name="reason", label="Reason", default="Force build"),
75 codebases=[
76 util.CodebaseParameter("",
f6e48e3a 77 branch=branch,
e2b96bf5
IB
78 revision=util.FixedParameter(name="revision", default=""),
79 repository=util.FixedParameter(name="repository", default=""),
80 project=util.FixedParameter(name="project", default=""),
81 ),
82 ],
83 username=util.FixedParameter(name="username", default="Web button"),
84 builderNames=builders)
85
86def deploy_scheduler(name, builders):
87 return schedulers.ForceScheduler(name=name,
88 builderNames=builders,
89 label="Deploy built package", buttonName="Deploy",
90 username=util.FixedParameter(name="username", default="Web button"),
91 codebases=[
92 util.CodebaseParameter(codebase="",
93 branch=util.FixedParameter(name="branch", default=""),
94 revision=util.FixedParameter(name="revision", default=""),
95 repository=util.FixedParameter(name="repository", default=""),
96 project=util.FixedParameter(name="project", default=""))],
97 reason=util.FixedParameter(name="reason", default="Deploy"),
98 properties=[
99 util.ChoiceStringParameter(label="Environment",
100 name="environment", default="integration",
101 choices=["integration", "production"]),
102 BuildsList(label="Build to deploy", name="build"),
103 ]
104 )
105
f6e48e3a
IB
106def git_hook_scheduler(project, builders=[], timer=1):
107 if len(builders) == 0:
108 builders = ["{}_build".format(project)]
109 return schedulers.AnyBranchScheduler(
110 change_filter=util.ChangeFilter(category="gitolite-hooks", project=project),
111 name="{}_git_hook".format(project), treeStableTimer=timer, builderNames=builders)
112
113def deploy_hook_scheduler(project, builders, timer=1):
e2b96bf5 114 return schedulers.AnyBranchScheduler(
f6e48e3a
IB
115 change_filter=util.ChangeFilter(category="deploy_webhook", project=project),
116 name="{}_deploy".format(project), treeStableTimer=timer, builderNames=builders)
117
118# Builders
119def all_builder_names(c):
120 return [builder.name for builder in c['builders']]
e2b96bf5 121
256d607c 122# Slack/XMPP status push
e2b96bf5
IB
123from buildbot.reporters.http import HttpStatusPushBase
124from twisted.internet import defer
125from twisted.python import log
126from buildbot.util import httpclientservice
127from buildbot.reporters import utils
128from buildbot.process import results
256d607c
IB
129from twisted.words.protocols.jabber.jid import JID
130from wokkel import client, xmppim
7158b76e 131from functools import partial
e2b96bf5
IB
132
133class SlackStatusPush(HttpStatusPushBase):
134 name = "SlackStatusPush"
135
136 @defer.inlineCallbacks
137 def reconfigService(self, serverUrl, **kwargs):
138 yield HttpStatusPushBase.reconfigService(self, **kwargs)
139 self._http = yield httpclientservice.HTTPClientService.getService(
140 self.master, serverUrl)
141
142 @defer.inlineCallbacks
143 def send(self, build):
144 yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
145 response = yield self._http.post("", json=self.format(build))
146 if response.code != 200:
147 log.msg("%s: unable to upload status: %s" %
148 (response.code, response.content))
149
150 def format(self, build):
151 colors = [
152 "#36A64F", # success
153 "#F1E903", # warnings
154 "#DA0505", # failure
155 "#FFFFFF", # skipped
156 "#000000", # exception
157 "#FFFFFF", # retry
158 "#D02CA9", # cancelled
159 ]
160
161 if "environment" in build["properties"]:
162 msg = "{} environment".format(build["properties"]["environment"][0])
163 if "build" in build["properties"]:
164 msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
e42f8192 165 elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
e2b96bf5
IB
166 msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
167 else:
168 msg = "build"
169
170 if build["complete"]:
171 timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
172 hours, rest = divmod(timedelta, 3600)
173 minutes, seconds = divmod(rest, 60)
174 if hours > 0:
175 duration = "{}h {}min {}s".format(hours, minutes, seconds)
176 elif minutes > 0:
177 duration = "{}min {}s".format(minutes, seconds)
178 else:
179 duration = "{}s".format(seconds)
180
181 text = "Build <{}|{}> of {}'s {} was {} in {}.".format(
182 build["url"], build["buildid"],
183 build["builder"]["name"],
184 msg,
185 results.Results[build["results"]],
186 duration,
187 )
188 fields = [
189 {
190 "title": "Build",
191 "value": "<{}|{}>".format(build["url"], build["buildid"]),
192 "short": True,
193 },
194 {
195 "title": "Project",
196 "value": build["builder"]["name"],
197 "short": True,
198 },
199 {
200 "title": "Build status",
201 "value": results.Results[build["results"]],
202 "short": True,
203 },
204 {
205 "title": "Build duration",
206 "value": duration,
207 "short": True,
208 },
209 ]
210 if "environment" in build["properties"]:
211 fields.append({
212 "title": "Environment",
213 "value": build["properties"]["environment"][0],
214 "short": True,
215 })
216 if "build" in build["properties"]:
217 fields.append({
218 "title": "Archive",
219 "value": build["properties"]["build"][0],
220 "short": True,
221 })
222 attachments = [{
223 "fallback": "",
224 "color": colors[build["results"]],
225 "fields": fields
226 }]
227 else:
228 text = "Build <{}|{}> of {}'s {} started.".format(
229 build["url"], build["buildid"],
230 build["builder"]["name"],
231 msg,
232 )
233 attachments = []
234
235 return {
236 "username": "Buildbot",
237 "icon_url": "http://docs.buildbot.net/current/_static/icon.png",
238 "text": text,
239 "attachments": attachments,
240 }
241
f6e48e3a
IB
242def configure_slack_push(c, secrets_file, builders):
243 c['services'].append(SlackStatusPush(
244 name="slack_status", builders=builders,
245 serverUrl=open(secrets_file + "/slack_webhook", "r").read().rstrip()))
246
256d607c
IB
247class XMPPStatusPush(HttpStatusPushBase):
248 name = "XMPPStatusPush"
249
250 @defer.inlineCallbacks
251 def reconfigService(self, password, recipients, **kwargs):
252 yield HttpStatusPushBase.reconfigService(self, **kwargs)
253 self.password = password
254 self.recipients = recipients
255
256 @defer.inlineCallbacks
257 def send(self, build):
258 yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
259 body = self.format(build)
260 factory = client.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self.password)
256d607c 261 d = client.clientCreator(factory)
7158b76e 262 def send_message(recipient, stream):
256d607c
IB
263 message = xmppim.Message(recipient=JID(recipient), body=body)
264 message.stanzaType = 'chat'
265 stream.send(message.toElement())
266 # To allow chaining
267 return stream
268 for recipient in self.recipients:
7158b76e 269 d.addCallback(partial(send_message, recipient))
256d607c
IB
270 d.addCallback(lambda _: factory.streamManager.xmlstream.sendFooter())
271 d.addErrback(log.err)
272
273 def format(self, build):
274 if "environment" in build["properties"]:
275 msg = "{} environment".format(build["properties"]["environment"][0])
276 if "build" in build["properties"]:
277 msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
e42f8192 278 elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
256d607c
IB
279 msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
280 else:
281 msg = "build"
282
283 if build["complete"]:
284 timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
285 hours, rest = divmod(timedelta, 3600)
286 minutes, seconds = divmod(rest, 60)
287 if hours > 0:
288 duration = "{}h {}min {}s".format(hours, minutes, seconds)
289 elif minutes > 0:
290 duration = "{}min {}s".format(minutes, seconds)
291 else:
292 duration = "{}s".format(seconds)
e2b96bf5 293
256d607c
IB
294 text = "Build {} ( {} ) of {}'s {} was {} in {}.".format(
295 build["buildid"], build["url"],
296 build["builder"]["name"],
297 msg,
298 results.Results[build["results"]],
299 duration,
300 )
301 else:
302 text = "Build {} ( {} ) of {}'s {} started.".format(
303 build["buildid"], build["url"],
304 build["builder"]["name"],
305 msg,
306 )
e2b96bf5 307
256d607c 308 return text
f6e48e3a
IB
309
310def configure_xmpp_push(c, secrets_file, builders, recipients):
311 c['services'].append(XMPPStatusPush(
312 name="xmpp_status", builders=builders, recipients=recipients,
313 password=open(secrets_file + "/notify_xmpp_password", "r").read().rstrip()))
314
315# LDAP edit
316from buildbot.process.buildstep import FAILURE
317from buildbot.process.buildstep import SUCCESS
318from buildbot.process.buildstep import BuildStep
319
f6e48e3a
IB
320def compute_build_infos(prefix, release_path):
321 @util.renderer
322 def compute(props):
323 import re, hashlib
324 build_file = props.getProperty("build")
325 package_dest = "{}/{}".format(release_path, build_file)
326 version = re.match(r"{0}_(.*).tar.gz".format(prefix), build_file).group(1)
327 with open(package_dest, "rb") as f:
328 sha = hashlib.sha256(f.read()).hexdigest()
329 return {
330 "build_version": version,
331 "build_hash": sha,
332 }
333 return compute
334
335def deploy_ssh_command(ssh_key_path, deploy_hosts):
336 @util.renderer
337 def compute(props):
338 environment = props["environment"] if props.hasProperty("environment") else "integration"
339 ssh_command = [
340 "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no",
341 "-i", ssh_key_path ]
342 return ssh_command + deploy_hosts.get(environment, ["host.invalid"])
343 return compute