| | 1 | from buildbot.plugins import util, steps, schedulers |
| | 2 | from buildbot_buildslist import BuildsList |
| | 3 | from shutil import which |
| | 4 | |
| | 5 | __all__ = [ |
| | 6 | "force_scheduler", "deploy_scheduler", "git_hook_scheduler", |
| | 7 | "clean_branch", "package_and_upload", "SlackStatusPush", |
| | 8 | "XMPPStatusPush", "NixShellCommand", |
| | 9 | "all_builder_names", "compute_build_infos", "deploy_ssh_command", |
| | 10 | "configure_slack_push", "configure_xmpp_push", "deploy_hook_scheduler", |
| | 11 | ] |
| | 12 | |
| | 13 | # Small helpers" |
| | 14 | @util.renderer |
| | 15 | def clean_branch(props): |
| | 16 | if props.hasProperty("branch") and len(props["branch"]) > 0: |
| | 17 | return props["branch"].replace("/", "_") |
| | 18 | else: |
| | 19 | return "HEAD" |
| | 20 | |
| | 21 | def package_and_upload(package, package_dest, package_url): |
| | 22 | return [ |
| | 23 | steps.ShellCommand(name="build package", |
| | 24 | logEnviron=False, haltOnFailure=True, |
| | 25 | command=["git", "archive", "HEAD", "-o", package]), |
| | 26 | |
| | 27 | steps.FileUpload(name="upload package", workersrc=package, |
| | 28 | masterdest=package_dest, |
| | 29 | url=package_url, mode=0o644), |
| | 30 | |
| | 31 | steps.ShellCommand(name="cleanup package", logEnviron=False, |
| | 32 | haltOnFailure=True, alwaysRun=True, |
| | 33 | command=["rm", "-f", package]), |
| | 34 | ] |
| | 35 | |
| | 36 | # Steps |
| | 37 | class NixShellCommand(steps.ShellCommand): |
| | 38 | def __init__(self, command=None, nixPackages=[], pure=True, nixFile=None, nixIncludes={}, nixArgs={}, **kwargs): |
| | 39 | oldpath = kwargs.get("env", {}).get("PATH", None) |
| | 40 | if which("nix-shell", path=oldpath) is None: |
| | 41 | kwargs["env"] = kwargs.get("env", {}) |
| | 42 | if isinstance(oldpath, str): |
| | 43 | kwargs["env"]["PATH"] = "/run/current-system/sw/bin:" + oldpath |
| | 44 | elif isinstance(oldpath, list): |
| | 45 | kwargs["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath |
| | 46 | nixcommand = ["nix-shell"] |
| | 47 | for k, v in nixArgs.items(): |
| | 48 | nixcommand.append("--arg") |
| | 49 | nixcommand.append(k) |
| | 50 | nixcommand.append(v) |
| | 51 | if pure: |
| | 52 | nixcommand.append("--pure") |
| | 53 | for k, v in nixIncludes.items(): |
| | 54 | nixcommand.append("-I") |
| | 55 | nixcommand.append("{}={}".format(k, v)) |
| | 56 | nixcommand.append("--run") |
| | 57 | nixcommand.append(command) |
| | 58 | if len(nixPackages) > 0: |
| | 59 | nixcommand.append("-p") |
| | 60 | nixcommand += nixPackages |
| | 61 | elif nixFile is not None: |
| | 62 | nixcommand.append(nixFile) |
| | 63 | super().__init__(command=nixcommand, **kwargs) |
| | 64 | |
| | 65 | # Schedulers |
| | 66 | def force_scheduler(name, builders, nobranch=False): |
| | 67 | if nobranch: |
| | 68 | branch = util.FixedParameter(name="branch", default="") |
| | 69 | else: |
| | 70 | branch=util.StringParameter(name="branch", label="Git reference (tag, branch)", required=True) |
| | 71 | |
| | 72 | return schedulers.ForceScheduler(name=name, |
| | 73 | label="Force build", buttonName="Force build", |
| | 74 | reason=util.StringParameter(name="reason", label="Reason", default="Force build"), |
| | 75 | codebases=[ |
| | 76 | util.CodebaseParameter("", |
| | 77 | branch=branch, |
| | 78 | revision=util.FixedParameter(name="revision", default=""), |
| | 79 | repository=util.FixedParameter(name="repository", default=""), |
| | 80 | project=util.FixedParameter(name="project", default=""), |
| | 81 | ), |
| | 82 | ], |
| | 83 | username=util.FixedParameter(name="username", default="Web button"), |
| | 84 | builderNames=builders) |
| | 85 | |
| | 86 | def deploy_scheduler(name, builders): |
| | 87 | return schedulers.ForceScheduler(name=name, |
| | 88 | builderNames=builders, |
| | 89 | label="Deploy built package", buttonName="Deploy", |
| | 90 | username=util.FixedParameter(name="username", default="Web button"), |
| | 91 | codebases=[ |
| | 92 | util.CodebaseParameter(codebase="", |
| | 93 | branch=util.FixedParameter(name="branch", default=""), |
| | 94 | revision=util.FixedParameter(name="revision", default=""), |
| | 95 | repository=util.FixedParameter(name="repository", default=""), |
| | 96 | project=util.FixedParameter(name="project", default=""))], |
| | 97 | reason=util.FixedParameter(name="reason", default="Deploy"), |
| | 98 | properties=[ |
| | 99 | util.ChoiceStringParameter(label="Environment", |
| | 100 | name="environment", default="integration", |
| | 101 | choices=["integration", "production"]), |
| | 102 | BuildsList(label="Build to deploy", name="build"), |
| | 103 | ] |
| | 104 | ) |
| | 105 | |
| | 106 | def git_hook_scheduler(project, builders=[], timer=1): |
| | 107 | if len(builders) == 0: |
| | 108 | builders = ["{}_build".format(project)] |
| | 109 | return schedulers.AnyBranchScheduler( |
| | 110 | change_filter=util.ChangeFilter(category="gitolite-hooks", project=project), |
| | 111 | name="{}_git_hook".format(project), treeStableTimer=timer, builderNames=builders) |
| | 112 | |
| | 113 | def deploy_hook_scheduler(project, builders, timer=1): |
| | 114 | return schedulers.AnyBranchScheduler( |
| | 115 | change_filter=util.ChangeFilter(category="deploy_webhook", project=project), |
| | 116 | name="{}_deploy".format(project), treeStableTimer=timer, builderNames=builders) |
| | 117 | |
| | 118 | # Builders |
| | 119 | def all_builder_names(c): |
| | 120 | return [builder.name for builder in c['builders']] |
| | 121 | |
| | 122 | # Slack/XMPP status push |
| | 123 | from buildbot.reporters.http import HttpStatusPushBase |
| | 124 | from twisted.internet import defer |
| | 125 | from twisted.python import log |
| | 126 | from buildbot.util import httpclientservice |
| | 127 | from buildbot.reporters import utils |
| | 128 | from buildbot.process import results |
| | 129 | from twisted.words.protocols.jabber.jid import JID |
| | 130 | from wokkel import client, xmppim |
| | 131 | from functools import partial |
| | 132 | |
| | 133 | class SlackStatusPush(HttpStatusPushBase): |
| | 134 | name = "SlackStatusPush" |
| | 135 | |
| | 136 | @defer.inlineCallbacks |
| | 137 | def reconfigService(self, serverUrl, **kwargs): |
| | 138 | yield HttpStatusPushBase.reconfigService(self, **kwargs) |
| | 139 | self._http = yield httpclientservice.HTTPClientService.getService( |
| | 140 | self.master, serverUrl) |
| | 141 | |
| | 142 | @defer.inlineCallbacks |
| | 143 | def send(self, build): |
| | 144 | yield utils.getDetailsForBuild(self.master, build, wantProperties=True) |
| | 145 | response = yield self._http.post("", json=self.format(build)) |
| | 146 | if response.code != 200: |
| | 147 | log.msg("%s: unable to upload status: %s" % |
| | 148 | (response.code, response.content)) |
| | 149 | |
| | 150 | def format(self, build): |
| | 151 | colors = [ |
| | 152 | "#36A64F", # success |
| | 153 | "#F1E903", # warnings |
| | 154 | "#DA0505", # failure |
| | 155 | "#FFFFFF", # skipped |
| | 156 | "#000000", # exception |
| | 157 | "#FFFFFF", # retry |
| | 158 | "#D02CA9", # cancelled |
| | 159 | ] |
| | 160 | |
| | 161 | if "environment" in build["properties"]: |
| | 162 | msg = "{} environment".format(build["properties"]["environment"][0]) |
| | 163 | if "build" in build["properties"]: |
| | 164 | msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg |
| | 165 | elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0: |
| | 166 | msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"]) |
| | 167 | else: |
| | 168 | msg = "build" |
| | 169 | |
| | 170 | if build["complete"]: |
| | 171 | timedelta = int((build["complete_at"] - build["started_at"]).total_seconds()) |
| | 172 | hours, rest = divmod(timedelta, 3600) |
| | 173 | minutes, seconds = divmod(rest, 60) |
| | 174 | if hours > 0: |
| | 175 | duration = "{}h {}min {}s".format(hours, minutes, seconds) |
| | 176 | elif minutes > 0: |
| | 177 | duration = "{}min {}s".format(minutes, seconds) |
| | 178 | else: |
| | 179 | duration = "{}s".format(seconds) |
| | 180 | |
| | 181 | text = "Build <{}|{}> of {}'s {} was {} in {}.".format( |
| | 182 | build["url"], build["buildid"], |
| | 183 | build["builder"]["name"], |
| | 184 | msg, |
| | 185 | results.Results[build["results"]], |
| | 186 | duration, |
| | 187 | ) |
| | 188 | fields = [ |
| | 189 | { |
| | 190 | "title": "Build", |
| | 191 | "value": "<{}|{}>".format(build["url"], build["buildid"]), |
| | 192 | "short": True, |
| | 193 | }, |
| | 194 | { |
| | 195 | "title": "Project", |
| | 196 | "value": build["builder"]["name"], |
| | 197 | "short": True, |
| | 198 | }, |
| | 199 | { |
| | 200 | "title": "Build status", |
| | 201 | "value": results.Results[build["results"]], |
| | 202 | "short": True, |
| | 203 | }, |
| | 204 | { |
| | 205 | "title": "Build duration", |
| | 206 | "value": duration, |
| | 207 | "short": True, |
| | 208 | }, |
| | 209 | ] |
| | 210 | if "environment" in build["properties"]: |
| | 211 | fields.append({ |
| | 212 | "title": "Environment", |
| | 213 | "value": build["properties"]["environment"][0], |
| | 214 | "short": True, |
| | 215 | }) |
| | 216 | if "build" in build["properties"]: |
| | 217 | fields.append({ |
| | 218 | "title": "Archive", |
| | 219 | "value": build["properties"]["build"][0], |
| | 220 | "short": True, |
| | 221 | }) |
| | 222 | attachments = [{ |
| | 223 | "fallback": "", |
| | 224 | "color": colors[build["results"]], |
| | 225 | "fields": fields |
| | 226 | }] |
| | 227 | else: |
| | 228 | text = "Build <{}|{}> of {}'s {} started.".format( |
| | 229 | build["url"], build["buildid"], |
| | 230 | build["builder"]["name"], |
| | 231 | msg, |
| | 232 | ) |
| | 233 | attachments = [] |
| | 234 | |
| | 235 | return { |
| | 236 | "username": "Buildbot", |
| | 237 | "icon_url": "http://docs.buildbot.net/current/_static/icon.png", |
| | 238 | "text": text, |
| | 239 | "attachments": attachments, |
| | 240 | } |
| | 241 | |
| | 242 | def configure_slack_push(c, secrets_file, builders): |
| | 243 | c['services'].append(SlackStatusPush( |
| | 244 | name="slack_status", builders=builders, |
| | 245 | serverUrl=open(secrets_file + "/slack_webhook", "r").read().rstrip())) |
| | 246 | |
| | 247 | class XMPPStatusPush(HttpStatusPushBase): |
| | 248 | name = "XMPPStatusPush" |
| | 249 | |
| | 250 | @defer.inlineCallbacks |
| | 251 | def reconfigService(self, password, recipients, **kwargs): |
| | 252 | yield HttpStatusPushBase.reconfigService(self, **kwargs) |
| | 253 | self.password = password |
| | 254 | self.recipients = recipients |
| | 255 | |
| | 256 | @defer.inlineCallbacks |
| | 257 | def send(self, build): |
| | 258 | yield utils.getDetailsForBuild(self.master, build, wantProperties=True) |
| | 259 | body = self.format(build) |
| | 260 | factory = client.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self.password) |
| | 261 | d = client.clientCreator(factory) |
| | 262 | def send_message(recipient, stream): |
| | 263 | message = xmppim.Message(recipient=JID(recipient), body=body) |
| | 264 | message.stanzaType = 'chat' |
| | 265 | stream.send(message.toElement()) |
| | 266 | # To allow chaining |
| | 267 | return stream |
| | 268 | for recipient in self.recipients: |
| | 269 | d.addCallback(partial(send_message, recipient)) |
| | 270 | d.addCallback(lambda _: factory.streamManager.xmlstream.sendFooter()) |
| | 271 | d.addErrback(log.err) |
| | 272 | |
| | 273 | def format(self, build): |
| | 274 | if "environment" in build["properties"]: |
| | 275 | msg = "{} environment".format(build["properties"]["environment"][0]) |
| | 276 | if "build" in build["properties"]: |
| | 277 | msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg |
| | 278 | elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0: |
| | 279 | msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"]) |
| | 280 | else: |
| | 281 | msg = "build" |
| | 282 | |
| | 283 | if build["complete"]: |
| | 284 | timedelta = int((build["complete_at"] - build["started_at"]).total_seconds()) |
| | 285 | hours, rest = divmod(timedelta, 3600) |
| | 286 | minutes, seconds = divmod(rest, 60) |
| | 287 | if hours > 0: |
| | 288 | duration = "{}h {}min {}s".format(hours, minutes, seconds) |
| | 289 | elif minutes > 0: |
| | 290 | duration = "{}min {}s".format(minutes, seconds) |
| | 291 | else: |
| | 292 | duration = "{}s".format(seconds) |
| | 293 | |
| | 294 | text = "Build {} ( {} ) of {}'s {} was {} in {}.".format( |
| | 295 | build["buildid"], build["url"], |
| | 296 | build["builder"]["name"], |
| | 297 | msg, |
| | 298 | results.Results[build["results"]], |
| | 299 | duration, |
| | 300 | ) |
| | 301 | else: |
| | 302 | text = "Build {} ( {} ) of {}'s {} started.".format( |
| | 303 | build["buildid"], build["url"], |
| | 304 | build["builder"]["name"], |
| | 305 | msg, |
| | 306 | ) |
| | 307 | |
| | 308 | return text |
| | 309 | |
| | 310 | def configure_xmpp_push(c, secrets_file, builders, recipients): |
| | 311 | c['services'].append(XMPPStatusPush( |
| | 312 | name="xmpp_status", builders=builders, recipients=recipients, |
| | 313 | password=open(secrets_file + "/notify_xmpp_password", "r").read().rstrip())) |
| | 314 | |
| | 315 | # LDAP edit |
| | 316 | from buildbot.process.buildstep import FAILURE |
| | 317 | from buildbot.process.buildstep import SUCCESS |
| | 318 | from buildbot.process.buildstep import BuildStep |
| | 319 | |
| | 320 | def compute_build_infos(prefix, release_path): |
| | 321 | @util.renderer |
| | 322 | def compute(props): |
| | 323 | import re, hashlib |
| | 324 | build_file = props.getProperty("build") |
| | 325 | package_dest = "{}/{}".format(release_path, build_file) |
| | 326 | version = re.match(r"{0}_(.*).tar.gz".format(prefix), build_file).group(1) |
| | 327 | with open(package_dest, "rb") as f: |
| | 328 | sha = hashlib.sha256(f.read()).hexdigest() |
| | 329 | return { |
| | 330 | "build_version": version, |
| | 331 | "build_hash": sha, |
| | 332 | } |
| | 333 | return compute |
| | 334 | |
| | 335 | def deploy_ssh_command(ssh_key_path, deploy_hosts): |
| | 336 | @util.renderer |
| | 337 | def compute(props): |
| | 338 | environment = props["environment"] if props.hasProperty("environment") else "integration" |
| | 339 | ssh_command = [ |
| | 340 | "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", |
| | 341 | "-i", ssh_key_path ] |
| | 342 | return ssh_command + deploy_hosts.get(environment, ["host.invalid"]) |
| | 343 | return compute |
projects / perso / Immae / Config / Nix.git / blame_incremental