flakes/private/buildbot/common/build_helpers.py

   1 from buildbot.plugins import util, steps, schedulers
   2 from buildbot_buildslist import BuildsList
   3 from shutil import which
   4
   5 __all__ = [
   6         "force_scheduler", "deploy_scheduler", "git_hook_scheduler",
   7         "clean_branch", "package_and_upload", "AppriseStatusPush",
   8         "XMPPStatusPush", "NixShellCommand",
   9         "all_builder_names", "compute_build_infos", "deploy_ssh_command",
  10         "configure_apprise_push",
  11         "configure_xmpp_push", "deploy_hook_scheduler",
  12         ]
  13
  14 # Small helpers"
  15 @util.renderer
  16 def clean_branch(props):
  17     if props.hasProperty("branch") and len(props["branch"]) > 0:
  18         return props["branch"].replace("/", "_")
  19     else:
  20         return "HEAD"
  21
  22 def package_and_upload(package, package_dest, package_url):
  23     return [
  24             steps.ShellCommand(name="build package",
  25                 logEnviron=False, haltOnFailure=True,
  26                 command=["git", "archive", "HEAD", "-o", package]),
  27
  28             steps.FileUpload(name="upload package", workersrc=package,
  29                 masterdest=package_dest,
  30                 url=package_url, mode=0o644),
  31
  32             steps.ShellCommand(name="cleanup package", logEnviron=False,
  33                 haltOnFailure=True, alwaysRun=True,
  34                 command=["rm", "-f", package]),
  35             ]
  36
  37 # Steps
  38 class NixShellCommand(steps.ShellCommand):
  39     def __init__(self, command=None, nixPackages=[], pure=True, nixFile=None, nixIncludes={}, nixArgs={}, **kwargs):
  40         oldpath = kwargs.get("env", {}).get("PATH", None)
  41         if which("nix-shell", path=oldpath) is None:
  42             kwargs["env"] = kwargs.get("env", {})
  43             if isinstance(oldpath, str):
  44                 kwargs["env"]["PATH"] = "/run/current-system/sw/bin:" + oldpath
  45             elif isinstance(oldpath, list):
  46                 kwargs["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath
  47         nixcommand = ["nix-shell"]
  48         for k, v in nixArgs.items():
  49             nixcommand.append("--arg")
  50             nixcommand.append(k)
  51             nixcommand.append(v)
  52         if pure:
  53             nixcommand.append("--pure")
  54         for k, v in nixIncludes.items():
  55             nixcommand.append("-I")
  56             nixcommand.append("{}={}".format(k, v))
  57         nixcommand.append("--run")
  58         nixcommand.append(command)
  59         if len(nixPackages) > 0:
  60             nixcommand.append("-p")
  61             nixcommand += nixPackages
  62         elif nixFile is not None:
  63             nixcommand.append(nixFile)
  64         super().__init__(command=nixcommand, **kwargs)
  65
  66 # Schedulers
  67 def force_scheduler(name, builders, nobranch=False):
  68     if nobranch:
  69         branch = util.FixedParameter(name="branch", default="")
  70     else:
  71         branch=util.StringParameter(name="branch", label="Git reference (tag, branch)", required=True)
  72
  73     return schedulers.ForceScheduler(name=name,
  74         label="Force build", buttonName="Force build",
  75         reason=util.StringParameter(name="reason", label="Reason", default="Force build"),
  76         codebases=[
  77             util.CodebaseParameter("",
  78                 branch=branch,
  79                 revision=util.FixedParameter(name="revision", default=""),
  80                 repository=util.FixedParameter(name="repository", default=""),
  81                 project=util.FixedParameter(name="project", default=""),
  82                 ),
  83             ],
  84         username=util.FixedParameter(name="username", default="Web button"),
  85         builderNames=builders)
  86
  87 def deploy_scheduler(name, builders):
  88     return schedulers.ForceScheduler(name=name,
  89         builderNames=builders,
  90         label="Deploy built package", buttonName="Deploy",
  91         username=util.FixedParameter(name="username", default="Web button"),
  92         codebases=[
  93             util.CodebaseParameter(codebase="",
  94                 branch=util.FixedParameter(name="branch", default=""),
  95                 revision=util.FixedParameter(name="revision", default=""),
  96                 repository=util.FixedParameter(name="repository", default=""),
  97                 project=util.FixedParameter(name="project", default=""))],
  98         reason=util.FixedParameter(name="reason", default="Deploy"),
  99         properties=[
 100             util.ChoiceStringParameter(label="Environment",
 101                 name="environment", default="integration",
 102                 choices=["integration", "production"]),
 103             BuildsList(label="Build to deploy", name="build"),
 104             ]
 105         )
 106
 107 def git_hook_scheduler(project, builders=[], timer=1):
 108     if len(builders) == 0:
 109         builders = ["{}_build".format(project)]
 110     return schedulers.AnyBranchScheduler(
 111             change_filter=util.ChangeFilter(category="gitolite-hooks", project=project),
 112             name="{}_git_hook".format(project), treeStableTimer=timer, builderNames=builders)
 113
 114 def deploy_hook_scheduler(project, builders, timer=1):
 115     return schedulers.AnyBranchScheduler(
 116             change_filter=util.ChangeFilter(category="deploy_webhook", project=project),
 117             name="{}_deploy".format(project), treeStableTimer=timer, builderNames=builders)
 118
 119 # Builders
 120 def all_builder_names(c):
 121     return [builder.name for builder in c['builders']]
 122
 123 # Apprise/XMPP status push
 124 from buildbot.reporters.http import HttpStatusPushBase
 125 from twisted.internet import defer
 126 from twisted.python import log
 127 from buildbot.reporters import utils
 128 from buildbot.process import results
 129 from twisted.words.protocols.jabber.jid import JID
 130 from wokkel import client, xmppim
 131 from functools import partial
 132 import apprise
 133
 134 class AppriseStatusPush(HttpStatusPushBase):
 135     name = "AppriseStatusPush"
 136
 137     @defer.inlineCallbacks
 138     def reconfigService(self, appriseUrls, **kwargs):
 139         self.appriseUrls = appriseUrls
 140         yield HttpStatusPushBase.reconfigService(self, **kwargs)
 141
 142     @defer.inlineCallbacks
 143     def send(self, build):
 144         yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
 145         appobject = apprise.Apprise()
 146         message = self.format(build)
 147         for url in self.appriseUrls:
 148             appobject.add(url.format(**message))
 149         yield appobject.notify(title=message["title"], body=message["text"])
 150
 151     def format(self, build):
 152         if "environment" in build["properties"]:
 153             msg = "{} environment".format(build["properties"]["environment"][0])
 154             if "build" in build["properties"]:
 155                 msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
 156         elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
 157             msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
 158         else:
 159             msg = "build"
 160
 161         if build["complete"]:
 162             timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
 163             hours, rest = divmod(timedelta, 3600)
 164             minutes, seconds = divmod(rest, 60)
 165             if hours > 0:
 166                 duration = "{}h {}min {}s".format(hours, minutes, seconds)
 167             elif minutes > 0:
 168                 duration = "{}min {}s".format(minutes, seconds)
 169             else:
 170                 duration = "{}s".format(seconds)
 171
 172             text = "Build {} ({}) of {}'s {} was {} in {}.".format(
 173                     build["number"], build["url"],
 174                     build["builder"]["name"],
 175                     msg,
 176                     results.Results[build["results"]],
 177                     duration,
 178                     )
 179         else:
 180             text = "Build {} ({}) of {}'s {} started.".format(
 181                     build["number"], build["url"],
 182                     build["builder"]["name"],
 183                     msg,
 184                     )
 185         return {
 186                 "username": "Buildbot",
 187                 "image_url": "http://docs.buildbot.net/current/_static/icon.png",
 188                 "text": text,
 189                 "title": "",
 190                 }
 191
 192 def configure_apprise_push(c, secrets_file, builders):
 193     c['services'].append(AppriseStatusPush(
 194         name="apprise_status", builders=builders,
 195         appriseUrls=open(secrets_file + "/apprise_webhooks", "r").read().split("\n")))
 196
 197 class XMPPStatusPush(HttpStatusPushBase):
 198     name = "XMPPStatusPush"
 199
 200     @defer.inlineCallbacks
 201     def reconfigService(self, password, recipients, **kwargs):
 202         yield HttpStatusPushBase.reconfigService(self, **kwargs)
 203         self.password = password
 204         self.recipients = recipients
 205
 206     @defer.inlineCallbacks
 207     def send(self, build):
 208         yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
 209         body = self.format(build)
 210         factory = client.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self.password)
 211         d = client.clientCreator(factory)
 212         def send_message(recipient, stream):
 213             message = xmppim.Message(recipient=JID(recipient), body=body)
 214             message.stanzaType = 'chat'
 215             stream.send(message.toElement())
 216             # To allow chaining
 217             return stream
 218         for recipient in self.recipients:
 219             d.addCallback(partial(send_message, recipient))
 220         d.addCallback(lambda _: factory.streamManager.xmlstream.sendFooter())
 221         d.addErrback(log.err)
 222
 223     def format(self, build):
 224         if "environment" in build["properties"]:
 225             msg = "{} environment".format(build["properties"]["environment"][0])
 226             if "build" in build["properties"]:
 227                 msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
 228         elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
 229             msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
 230         else:
 231             msg = "build"
 232
 233         if build["complete"]:
 234             timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
 235             hours, rest = divmod(timedelta, 3600)
 236             minutes, seconds = divmod(rest, 60)
 237             if hours > 0:
 238                 duration = "{}h {}min {}s".format(hours, minutes, seconds)
 239             elif minutes > 0:
 240                 duration = "{}min {}s".format(minutes, seconds)
 241             else:
 242                 duration = "{}s".format(seconds)
 243
 244             text = "Build {} ( {} ) of {}'s {} was {} in {}.".format(
 245                     build["number"], build["url"],
 246                     build["builder"]["name"],
 247                     msg,
 248                     results.Results[build["results"]],
 249                     duration,
 250                     )
 251         else:
 252             text = "Build {} ( {} ) of {}'s {} started.".format(
 253                     build["number"], build["url"],
 254                     build["builder"]["name"],
 255                     msg,
 256                     )
 257
 258         return text
 259
 260 def configure_xmpp_push(c, secrets_file, builders, recipients):
 261     c['services'].append(XMPPStatusPush(
 262         name="xmpp_status", builders=builders, recipients=recipients,
 263         password=open(secrets_file + "/notify_xmpp_password", "r").read().rstrip()))
 264
 265 # LDAP edit
 266 from buildbot.process.buildstep import FAILURE
 267 from buildbot.process.buildstep import SUCCESS
 268 from buildbot.process.buildstep import BuildStep
 269
 270 def compute_build_infos(prefix, release_path):
 271     @util.renderer
 272     def compute(props):
 273         import re, hashlib
 274         build_file = props.getProperty("build")
 275         package_dest = "{}/{}".format(release_path, build_file)
 276         version = re.match(r"{0}_(.*).tar.gz".format(prefix), build_file).group(1)
 277         with open(package_dest, "rb") as f:
 278             sha = hashlib.sha256(f.read()).hexdigest()
 279         return {
 280                 "build_version": version,
 281                 "build_hash": sha,
 282                 }
 283     return compute
 284
 285 def deploy_ssh_command(ssh_key_path, deploy_hosts):
 286     @util.renderer
 287     def compute(props):
 288         environment = props["environment"] if props.hasProperty("environment") else "integration"
 289         ssh_command = [
 290                 "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no",
 291                 "-i", ssh_key_path ]
 292         return ssh_command + deploy_hosts.get(environment, ["host.invalid"])
 293     return compute