modules/private/buildbot/common/build_helpers.py

   1 from buildbot.plugins import util, steps, schedulers
   2 from buildbot_buildslist import BuildsList
   3 from shutil import which
   4
   5 __all__ = [
   6         "force_scheduler", "deploy_scheduler", "git_hook_scheduler",
   7         "clean_branch", "package_and_upload", "SlackStatusPush",
   8         "XMPPStatusPush", "LdapEdit", "NixShellCommand",
   9         "all_builder_names", "compute_build_infos", "deploy_ssh_command",
  10         "configure_slack_push", "configure_xmpp_push", "deploy_hook_scheduler",
  11         ]
  12
  13 # Small helpers"
  14 @util.renderer
  15 def clean_branch(props):
  16     if props.hasProperty("branch") and len(props["branch"]) > 0:
  17         return props["branch"].replace("/", "_")
  18     else:
  19         return "HEAD"
  20
  21 def package_and_upload(package, package_dest, package_url):
  22     return [
  23             steps.ShellCommand(name="build package",
  24                 logEnviron=False, haltOnFailure=True,
  25                 command=["git", "archive", "HEAD", "-o", package]),
  26
  27             steps.FileUpload(name="upload package", workersrc=package,
  28                 masterdest=package_dest,
  29                 url=package_url, mode=0o644),
  30
  31             steps.ShellCommand(name="cleanup package", logEnviron=False,
  32                 haltOnFailure=True, alwaysRun=True,
  33                 command=["rm", "-f", package]),
  34             ]
  35
  36 # Steps
  37 class NixShellCommand(steps.ShellCommand):
  38     def __init__(self, command=None, pure=True, nixfile=None, **kwargs):
  39         oldpath = kwargs.get("env", {}).get("PATH", None)
  40         if which("nix-shell", path=oldpath) is None:
  41             kwargs["env"] = kwargs.get("env", {})
  42             if isinstance(oldpath, str):
  43                 kwargs["env"]["PATH"] = "/run/current-system/sw/bin:" + oldpath
  44             elif isinstance(oldpath, list):
  45                 kwargs["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath
  46         nixcommand = ["nix-shell"]
  47         if pure:
  48             nixcommand.append("--pure")
  49         nixcommand.append("--run")
  50         nixcommand.append(command)
  51         if nixfile is not None:
  52             nixcommand.append(nixfile)
  53         super().__init__(command=nixcommand, **kwargs)
  54
  55 # Schedulers
  56 def force_scheduler(name, builders, nobranch=False):
  57     if nobranch:
  58         branch = util.FixedParameter(name="branch", default="")
  59     else:
  60         branch=util.StringParameter(name="branch", label="Git reference (tag, branch)", required=True)
  61
  62     return schedulers.ForceScheduler(name=name,
  63         label="Force build", buttonName="Force build",
  64         reason=util.StringParameter(name="reason", label="Reason", default="Force build"),
  65         codebases=[
  66             util.CodebaseParameter("",
  67                 branch=branch,
  68                 revision=util.FixedParameter(name="revision", default=""),
  69                 repository=util.FixedParameter(name="repository", default=""),
  70                 project=util.FixedParameter(name="project", default=""),
  71                 ),
  72             ],
  73         username=util.FixedParameter(name="username", default="Web button"),
  74         builderNames=builders)
  75
  76 def deploy_scheduler(name, builders):
  77     return schedulers.ForceScheduler(name=name,
  78         builderNames=builders,
  79         label="Deploy built package", buttonName="Deploy",
  80         username=util.FixedParameter(name="username", default="Web button"),
  81         codebases=[
  82             util.CodebaseParameter(codebase="",
  83                 branch=util.FixedParameter(name="branch", default=""),
  84                 revision=util.FixedParameter(name="revision", default=""),
  85                 repository=util.FixedParameter(name="repository", default=""),
  86                 project=util.FixedParameter(name="project", default=""))],
  87         reason=util.FixedParameter(name="reason", default="Deploy"),
  88         properties=[
  89             util.ChoiceStringParameter(label="Environment",
  90                 name="environment", default="integration",
  91                 choices=["integration", "production"]),
  92             BuildsList(label="Build to deploy", name="build"),
  93             ]
  94         )
  95
  96 def git_hook_scheduler(project, builders=[], timer=1):
  97     if len(builders) == 0:
  98         builders = ["{}_build".format(project)]
  99     return schedulers.AnyBranchScheduler(
 100             change_filter=util.ChangeFilter(category="gitolite-hooks", project=project),
 101             name="{}_git_hook".format(project), treeStableTimer=timer, builderNames=builders)
 102
 103 def deploy_hook_scheduler(project, builders, timer=1):
 104     return schedulers.AnyBranchScheduler(
 105             change_filter=util.ChangeFilter(category="deploy_webhook", project=project),
 106             name="{}_deploy".format(project), treeStableTimer=timer, builderNames=builders)
 107
 108 # Builders
 109 def all_builder_names(c):
 110     return [builder.name for builder in c['builders']]
 111
 112 # Slack/XMPP status push
 113 from buildbot.reporters.http import HttpStatusPushBase
 114 from twisted.internet import defer
 115 from twisted.python import log
 116 from buildbot.util import httpclientservice
 117 from buildbot.reporters import utils
 118 from buildbot.process import results
 119 from twisted.words.protocols.jabber.jid import JID
 120 from wokkel import client, xmppim
 121 from functools import partial
 122
 123 class SlackStatusPush(HttpStatusPushBase):
 124     name = "SlackStatusPush"
 125
 126     @defer.inlineCallbacks
 127     def reconfigService(self, serverUrl, **kwargs):
 128         yield HttpStatusPushBase.reconfigService(self, **kwargs)
 129         self._http = yield httpclientservice.HTTPClientService.getService(
 130             self.master, serverUrl)
 131
 132     @defer.inlineCallbacks
 133     def send(self, build):
 134         yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
 135         response = yield self._http.post("", json=self.format(build))
 136         if response.code != 200:
 137             log.msg("%s: unable to upload status: %s" %
 138                     (response.code, response.content))
 139
 140     def format(self, build):
 141         colors = [
 142                 "#36A64F", # success
 143                 "#F1E903", # warnings
 144                 "#DA0505", # failure
 145                 "#FFFFFF", # skipped
 146                 "#000000", # exception
 147                 "#FFFFFF", # retry
 148                 "#D02CA9", # cancelled
 149                 ]
 150
 151         if "environment" in build["properties"]:
 152             msg = "{} environment".format(build["properties"]["environment"][0])
 153             if "build" in build["properties"]:
 154                 msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
 155         elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
 156             msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
 157         else:
 158             msg = "build"
 159
 160         if build["complete"]:
 161             timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
 162             hours, rest = divmod(timedelta, 3600)
 163             minutes, seconds = divmod(rest, 60)
 164             if hours > 0:
 165                 duration = "{}h {}min {}s".format(hours, minutes, seconds)
 166             elif minutes > 0:
 167                 duration = "{}min {}s".format(minutes, seconds)
 168             else:
 169                 duration = "{}s".format(seconds)
 170
 171             text = "Build <{}|{}> of {}'s {} was {} in {}.".format(
 172                     build["url"], build["buildid"],
 173                     build["builder"]["name"],
 174                     msg,
 175                     results.Results[build["results"]],
 176                     duration,
 177                     )
 178             fields = [
 179                     {
 180                         "title": "Build",
 181                         "value": "<{}|{}>".format(build["url"], build["buildid"]),
 182                         "short": True,
 183                         },
 184                     {
 185                         "title": "Project",
 186                         "value": build["builder"]["name"],
 187                         "short": True,
 188                         },
 189                     {
 190                         "title": "Build status",
 191                         "value": results.Results[build["results"]],
 192                         "short": True,
 193                         },
 194                     {
 195                         "title": "Build duration",
 196                         "value": duration,
 197                         "short": True,
 198                         },
 199                     ]
 200             if "environment" in build["properties"]:
 201                 fields.append({
 202                         "title": "Environment",
 203                         "value": build["properties"]["environment"][0],
 204                         "short": True,
 205                     })
 206             if "build" in build["properties"]:
 207                 fields.append({
 208                         "title": "Archive",
 209                         "value": build["properties"]["build"][0],
 210                         "short": True,
 211                     })
 212             attachments = [{
 213                     "fallback": "",
 214                     "color": colors[build["results"]],
 215                     "fields": fields
 216                     }]
 217         else:
 218             text = "Build <{}|{}> of {}'s {} started.".format(
 219                     build["url"], build["buildid"],
 220                     build["builder"]["name"],
 221                     msg,
 222                     )
 223             attachments = []
 224
 225         return {
 226                 "username": "Buildbot",
 227                 "icon_url": "http://docs.buildbot.net/current/_static/icon.png",
 228                 "text": text,
 229                 "attachments": attachments,
 230                 }
 231
 232 def configure_slack_push(c, secrets_file, builders):
 233     c['services'].append(SlackStatusPush(
 234         name="slack_status", builders=builders,
 235         serverUrl=open(secrets_file + "/slack_webhook", "r").read().rstrip()))
 236
 237 class XMPPStatusPush(HttpStatusPushBase):
 238     name = "XMPPStatusPush"
 239
 240     @defer.inlineCallbacks
 241     def reconfigService(self, password, recipients, **kwargs):
 242         yield HttpStatusPushBase.reconfigService(self, **kwargs)
 243         self.password = password
 244         self.recipients = recipients
 245
 246     @defer.inlineCallbacks
 247     def send(self, build):
 248         yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
 249         body = self.format(build)
 250         factory = client.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self.password)
 251         d = client.clientCreator(factory)
 252         def send_message(recipient, stream):
 253             message = xmppim.Message(recipient=JID(recipient), body=body)
 254             message.stanzaType = 'chat'
 255             stream.send(message.toElement())
 256             # To allow chaining
 257             return stream
 258         for recipient in self.recipients:
 259             d.addCallback(partial(send_message, recipient))
 260         d.addCallback(lambda _: factory.streamManager.xmlstream.sendFooter())
 261         d.addErrback(log.err)
 262
 263     def format(self, build):
 264         if "environment" in build["properties"]:
 265             msg = "{} environment".format(build["properties"]["environment"][0])
 266             if "build" in build["properties"]:
 267                 msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
 268         elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
 269             msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
 270         else:
 271             msg = "build"
 272
 273         if build["complete"]:
 274             timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
 275             hours, rest = divmod(timedelta, 3600)
 276             minutes, seconds = divmod(rest, 60)
 277             if hours > 0:
 278                 duration = "{}h {}min {}s".format(hours, minutes, seconds)
 279             elif minutes > 0:
 280                 duration = "{}min {}s".format(minutes, seconds)
 281             else:
 282                 duration = "{}s".format(seconds)
 283
 284             text = "Build {} ( {} ) of {}'s {} was {} in {}.".format(
 285                     build["buildid"], build["url"],
 286                     build["builder"]["name"],
 287                     msg,
 288                     results.Results[build["results"]],
 289                     duration,
 290                     )
 291         else:
 292             text = "Build {} ( {} ) of {}'s {} started.".format(
 293                     build["buildid"], build["url"],
 294                     build["builder"]["name"],
 295                     msg,
 296                     )
 297
 298         return text
 299
 300 def configure_xmpp_push(c, secrets_file, builders, recipients):
 301     c['services'].append(XMPPStatusPush(
 302         name="xmpp_status", builders=builders, recipients=recipients,
 303         password=open(secrets_file + "/notify_xmpp_password", "r").read().rstrip()))
 304
 305 # LDAP edit
 306 from buildbot.process.buildstep import FAILURE
 307 from buildbot.process.buildstep import SUCCESS
 308 from buildbot.process.buildstep import BuildStep
 309
 310 class LdapEdit(BuildStep):
 311     name = "LdapEdit"
 312     renderables = ["environment", "build_version", "build_hash", "ldap_password"]
 313
 314     def __init__(self, **kwargs):
 315         self.environment = kwargs.pop("environment")
 316         self.build_version = kwargs.pop("build_version")
 317         self.build_hash = kwargs.pop("build_hash")
 318         self.ldap_password = kwargs.pop("ldap_password")
 319         self.ldap_host = kwargs.pop("ldap_host")
 320         self.ldap_dn = kwargs.pop("ldap_dn")
 321         self.ldap_roles_base = kwargs.pop("ldap_roles_base")
 322         self.ldap_cn_template = kwargs.pop("ldap_cn_template")
 323         self.config_key = kwargs.pop("config_key")
 324         super().__init__(**kwargs)
 325
 326     def run(self):
 327         import json
 328         from ldap3 import Reader, Writer, Server, Connection, ObjectDef
 329         server = Server(self.ldap_host)
 330         conn = Connection(server,
 331                 user=self.ldap_dn,
 332                 password=self.ldap_password)
 333         conn.bind()
 334         obj = ObjectDef("immaePuppetClass", conn)
 335         r = Reader(conn, obj,
 336                 "cn={},{}".format(self.ldap_cn_template.format(self.environment), self.ldap_roles_base))
 337         r.search()
 338         if len(r) > 0:
 339             w = Writer.from_cursor(r)
 340             for value in w[0].immaePuppetJson.values:
 341                 config = json.loads(value)
 342                 if "{}_version".format(self.config_key) in config:
 343                     config["{}_version".format(self.config_key)] = self.build_version
 344                     config["{}_sha256".format(self.config_key)] = self.build_hash
 345                     w[0].immaePuppetJson -= value
 346                     w[0].immaePuppetJson += json.dumps(config, indent="  ")
 347                     w.commit()
 348                     return defer.succeed(SUCCESS)
 349         return defer.succeed(FAILURE)
 350
 351 def compute_build_infos(prefix, release_path):
 352     @util.renderer
 353     def compute(props):
 354         import re, hashlib
 355         build_file = props.getProperty("build")
 356         package_dest = "{}/{}".format(release_path, build_file)
 357         version = re.match(r"{0}_(.*).tar.gz".format(prefix), build_file).group(1)
 358         with open(package_dest, "rb") as f:
 359             sha = hashlib.sha256(f.read()).hexdigest()
 360         return {
 361                 "build_version": version,
 362                 "build_hash": sha,
 363                 }
 364     return compute
 365
 366 def deploy_ssh_command(ssh_key_path, deploy_hosts):
 367     @util.renderer
 368     def compute(props):
 369         environment = props["environment"] if props.hasProperty("environment") else "integration"
 370         ssh_command = [
 371                 "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no",
 372                 "-i", ssh_key_path ]
 373         return ssh_command + deploy_hosts.get(environment, ["host.invalid"])
 374     return compute