]>
git.immae.eu Git - perso/Immae/Config/Nix.git/blob - modules/private/buildbot/common/build_helpers.py
1 from buildbot
.plugins
import util
, steps
, schedulers
2 from buildbot_buildslist
import BuildsList
3 from shutil
import which
6 "force_scheduler", "deploy_scheduler", "git_hook_scheduler",
7 "clean_branch", "package_and_upload", "AppriseStatusPush",
8 "XMPPStatusPush", "NixShellCommand",
9 "all_builder_names", "compute_build_infos", "deploy_ssh_command",
10 "configure_apprise_push",
11 "configure_xmpp_push", "deploy_hook_scheduler",
16 def clean_branch(props
):
17 if props
.hasProperty("branch") and len(props
["branch"]) > 0:
18 return props
["branch"].replace("/", "_")
22 def package_and_upload(package
, package_dest
, package_url
):
24 steps
.ShellCommand(name
="build package",
25 logEnviron
=False, haltOnFailure
=True,
26 command
=["git", "archive", "HEAD", "-o", package
]),
28 steps
.FileUpload(name
="upload package", workersrc
=package
,
29 masterdest
=package_dest
,
30 url
=package_url
, mode
=0o644),
32 steps
.ShellCommand(name
="cleanup package", logEnviron
=False,
33 haltOnFailure
=True, alwaysRun
=True,
34 command
=["rm", "-f", package
]),
38 class NixShellCommand(steps
.ShellCommand
):
39 def __init__(self
, command
=None, nixPackages
=[], pure
=True, nixFile
=None, nixIncludes
={}, nixArgs={}
, **kwargs
):
40 oldpath
= kwargs
.get("env", {}).get("PATH", None)
41 if which("nix-shell", path
=oldpath
) is None:
42 kwargs
["env"] = kwargs
.get("env", {})
43 if isinstance(oldpath
, str):
44 kwargs
["env"]["PATH"] = "/run/current-system/sw/bin:" + oldpath
45 elif isinstance(oldpath
, list):
46 kwargs
["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath
47 nixcommand
= ["nix-shell"]
48 for k
, v
in nixArgs
.items():
49 nixcommand
.append("--arg")
53 nixcommand
.append("--pure")
54 for k
, v
in nixIncludes
.items():
55 nixcommand
.append("-I")
56 nixcommand
.append("{}={}".format(k
, v
))
57 nixcommand
.append("--run")
58 nixcommand
.append(command
)
59 if len(nixPackages
) > 0:
60 nixcommand
.append("-p")
61 nixcommand
+= nixPackages
62 elif nixFile
is not None:
63 nixcommand
.append(nixFile
)
64 super().__init
__(command
=nixcommand
, **kwargs
)
67 def force_scheduler(name
, builders
, nobranch
=False):
69 branch
= util
.FixedParameter(name
="branch", default
="")
71 branch
=util
.StringParameter(name
="branch", label
="Git reference (tag, branch)", required
=True)
73 return schedulers
.ForceScheduler(name
=name
,
74 label
="Force build", buttonName
="Force build",
75 reason
=util
.StringParameter(name
="reason", label
="Reason", default
="Force build"),
77 util
.CodebaseParameter("",
79 revision
=util
.FixedParameter(name
="revision", default
=""),
80 repository
=util
.FixedParameter(name
="repository", default
=""),
81 project
=util
.FixedParameter(name
="project", default
=""),
84 username
=util
.FixedParameter(name
="username", default
="Web button"),
85 builderNames
=builders
)
87 def deploy_scheduler(name
, builders
):
88 return schedulers
.ForceScheduler(name
=name
,
89 builderNames
=builders
,
90 label
="Deploy built package", buttonName
="Deploy",
91 username
=util
.FixedParameter(name
="username", default
="Web button"),
93 util
.CodebaseParameter(codebase
="",
94 branch
=util
.FixedParameter(name
="branch", default
=""),
95 revision
=util
.FixedParameter(name
="revision", default
=""),
96 repository
=util
.FixedParameter(name
="repository", default
=""),
97 project
=util
.FixedParameter(name
="project", default
=""))],
98 reason
=util
.FixedParameter(name
="reason", default
="Deploy"),
100 util
.ChoiceStringParameter(label
="Environment",
101 name
="environment", default
="integration",
102 choices
=["integration", "production"]),
103 BuildsList(label
="Build to deploy", name
="build"),
107 def git_hook_scheduler(project
, builders
=[], timer
=1):
108 if len(builders
) == 0:
109 builders
= ["{}_build".format(project
)]
110 return schedulers
.AnyBranchScheduler(
111 change_filter
=util
.ChangeFilter(category
="gitolite-hooks", project
=project
),
112 name
="{}_git_hook".format(project
), treeStableTimer
=timer
, builderNames
=builders
)
114 def deploy_hook_scheduler(project
, builders
, timer
=1):
115 return schedulers
.AnyBranchScheduler(
116 change_filter
=util
.ChangeFilter(category
="deploy_webhook", project
=project
),
117 name
="{}_deploy".format(project
), treeStableTimer
=timer
, builderNames
=builders
)
120 def all_builder_names(c
):
121 return [builder
.name
for builder
in c
['builders']]
123 # Apprise/XMPP status push
124 from buildbot
.reporters
.http
import HttpStatusPushBase
125 from twisted
.internet
import defer
126 from twisted
.python
import log
127 from buildbot
.reporters
import utils
128 from buildbot
.process
import results
129 from twisted
.words
.protocols
.jabber
.jid
import JID
130 from wokkel
import client
, xmppim
131 from functools
import partial
134 class AppriseStatusPush(HttpStatusPushBase
):
135 name
= "AppriseStatusPush"
137 @defer.inlineCallbacks
138 def reconfigService(self
, appriseUrls
, **kwargs
):
139 self
.appriseUrls
= appriseUrls
140 yield HttpStatusPushBase
.reconfigService(self
, **kwargs
)
142 @defer.inlineCallbacks
143 def send(self
, build
):
144 yield utils
.getDetailsForBuild(self
.master
, build
, wantProperties
=True)
145 appobject
= apprise
.Apprise()
146 message
= self
.format(build
)
147 for url
in self
.appriseUrls
:
148 appobject
.add(url
.format(**message
))
149 yield appobject
.notify(title
=message
["title"], body
=message
["text"])
151 def format(self
, build
):
152 if "environment" in build
["properties"]:
153 msg
= "{} environment".format(build
["properties"]["environment"][0])
154 if "build" in build
["properties"]:
155 msg
= "of archive {} in ".format(build
["properties"]["build"][0]) + msg
156 elif len(build
["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
157 msg
= "revision {}".format(build
["buildset"]["sourcestamps"][0]["branch"])
161 if build
["complete"]:
162 timedelta
= int((build
["complete_at"] - build
["started_at"]).total_seconds())
163 hours
, rest
= divmod(timedelta
, 3600)
164 minutes
, seconds
= divmod(rest
, 60)
166 duration
= "{}h {}min {}s".format(hours
, minutes
, seconds
)
168 duration
= "{}min {}s".format(minutes
, seconds
)
170 duration
= "{}s".format(seconds
)
172 text
= "Build {} ({}) of {}'s {} was {} in {}.".format(
173 build
["number"], build
["url"],
174 build
["builder"]["name"],
176 results
.Results
[build
["results"]],
180 text
= "Build {} ({}) of {}'s {} started.".format(
181 build
["number"], build
["url"],
182 build
["builder"]["name"],
186 "username": "Buildbot",
187 "image_url": "http://docs.buildbot.net/current/_static/icon.png",
192 def configure_apprise_push(c
, secrets_file
, builders
):
193 c
['services'].append(AppriseStatusPush(
194 name
="apprise_status", builders
=builders
,
195 appriseUrls
=open(secrets_file
+ "/apprise_webhooks", "r").read().split("\n")))
197 class XMPPStatusPush(HttpStatusPushBase
):
198 name
= "XMPPStatusPush"
200 @defer.inlineCallbacks
201 def reconfigService(self
, password
, recipients
, **kwargs
):
202 yield HttpStatusPushBase
.reconfigService(self
, **kwargs
)
203 self
.password
= password
204 self
.recipients
= recipients
206 @defer.inlineCallbacks
207 def send(self
, build
):
208 yield utils
.getDetailsForBuild(self
.master
, build
, wantProperties
=True)
209 body
= self
.format(build
)
210 factory
= client
.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self
.password
)
211 d
= client
.clientCreator(factory
)
212 def send_message(recipient
, stream
):
213 message
= xmppim
.Message(recipient
=JID(recipient
), body
=body
)
214 message
.stanzaType
= 'chat'
215 stream
.send(message
.toElement())
218 for recipient
in self
.recipients
:
219 d
.addCallback(partial(send_message
, recipient
))
220 d
.addCallback(lambda _
: factory
.streamManager
.xmlstream
.sendFooter())
221 d
.addErrback(log
.err
)
223 def format(self
, build
):
224 if "environment" in build
["properties"]:
225 msg
= "{} environment".format(build
["properties"]["environment"][0])
226 if "build" in build
["properties"]:
227 msg
= "of archive {} in ".format(build
["properties"]["build"][0]) + msg
228 elif len(build
["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
229 msg
= "revision {}".format(build
["buildset"]["sourcestamps"][0]["branch"])
233 if build
["complete"]:
234 timedelta
= int((build
["complete_at"] - build
["started_at"]).total_seconds())
235 hours
, rest
= divmod(timedelta
, 3600)
236 minutes
, seconds
= divmod(rest
, 60)
238 duration
= "{}h {}min {}s".format(hours
, minutes
, seconds
)
240 duration
= "{}min {}s".format(minutes
, seconds
)
242 duration
= "{}s".format(seconds
)
244 text
= "Build {} ( {} ) of {}'s {} was {} in {}.".format(
245 build
["number"], build
["url"],
246 build
["builder"]["name"],
248 results
.Results
[build
["results"]],
252 text
= "Build {} ( {} ) of {}'s {} started.".format(
253 build
["number"], build
["url"],
254 build
["builder"]["name"],
260 def configure_xmpp_push(c
, secrets_file
, builders
, recipients
):
261 c
['services'].append(XMPPStatusPush(
262 name
="xmpp_status", builders
=builders
, recipients
=recipients
,
263 password
=open(secrets_file
+ "/notify_xmpp_password", "r").read().rstrip()))
266 from buildbot
.process
.buildstep
import FAILURE
267 from buildbot
.process
.buildstep
import SUCCESS
268 from buildbot
.process
.buildstep
import BuildStep
270 def compute_build_infos(prefix
, release_path
):
274 build_file
= props
.getProperty("build")
275 package_dest
= "{}/{}".format(release_path
, build_file
)
276 version
= re
.match(r
"{0}_(.*).tar.gz".format(prefix
), build_file
).group(1)
277 with open(package_dest
, "rb") as f
:
278 sha
= hashlib
.sha256(f
.read()).hexdigest()
280 "build_version": version
,
285 def deploy_ssh_command(ssh_key_path
, deploy_hosts
):
288 environment
= props
["environment"] if props
.hasProperty("environment") else "integration"
290 "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no",
292 return ssh_command
+ deploy_hosts
.get(environment
, ["host.invalid"])