nixops/modules/buildbot/projects/test/__init__.py

   1 from buildbot.plugins import *
   2 from buildbot_common.build_helpers import *
   3 import os
   4
   5 __all__ = [ "configure", "E" ]
   6
   7 class E():
   8     PROJECT       = "test"
   9     BUILDBOT_URL  = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
  10     SOCKET        = "unix:/run/buildbot/{}.sock".format(PROJECT)
  11     PB_SOCKET     = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT)
  12     RELEASE_PATH  = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT)
  13     RELEASE_URL   = "https://release.immae.eu/{}".format(PROJECT)
  14     GIT_URL       = "https://git.immae.eu/perso/Immae/TestProject.git"
  15     SSH_KEY_PATH  = "/var/lib/buildbot/buildbot_key"
  16     PUPPET_HOST   = "root@backup-1.v.immae.eu"
  17     LDAP_HOST     = "ldap.immae.eu"
  18     LDAP_DN       = "cn=buildbot,ou=services,dc=immae,dc=eu"
  19     LDAP_ROLES_BASE = "ou=roles,ou=hosts,dc=immae,dc=eu"
  20
  21     # master.cfg
  22     SECRETS_FILE       = os.getcwd() + "/secrets"
  23     LDAP_URL           = "ldaps://ldap.immae.eu:636"
  24     LDAP_ADMIN_USER    = "cn=buildbot,ou=services,dc=immae,dc=eu"
  25     LDAP_BASE          = "dc=immae,dc=eu"
  26     LDAP_PATTERN       = "(uid=%(username)s)"
  27     LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=test,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))"
  28     TITLE_URL          = "https://git.immae.eu/?p=perso/Immae/TestProject.git;a=summary"
  29     TITLE              = "Test project"
  30
  31 def configure(c):
  32     c["buildbotURL"] = E.BUILDBOT_URL
  33     c["www"]["port"] = E.SOCKET
  34
  35     c['workers'].append(worker.LocalWorker("generic-worker-test"))
  36     c['workers'].append(worker.LocalWorker("deploy-worker-test"))
  37
  38     c['schedulers'].append(hook_scheduler("TestProject", timer=1))
  39     c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"]))
  40     c['schedulers'].append(deploy_scheduler("deploy_test", ["TestProject_deploy"]))
  41
  42     c['builders'].append(factory())
  43     c['builders'].append(deploy_factory())
  44
  45     c['services'].append(SlackStatusPush(
  46         name="slack_status_test_project",
  47         builders=["TestProject_build", "TestProject_deploy"],
  48         serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip()))
  49
  50 def factory():
  51     package = util.Interpolate("test_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch)
  52     package_dest = util.Interpolate("{}/test_%(kw:clean_branch)s.tar.gz".format(E.RELEASE_PATH), clean_branch=clean_branch)
  53     package_url = util.Interpolate("{}/test_%(kw:clean_branch)s.tar.gz".format(E.RELEASE_URL), clean_branch=clean_branch)
  54
  55     factory = util.BuildFactory()
  56     factory.addStep(steps.Git(logEnviron=False,
  57         repourl=E.GIT_URL, mode="full", method="copy"))
  58     factory.addStep(steps.ShellCommand(name="env",
  59         logEnviron=False, command=["env"]))
  60     factory.addStep(steps.ShellCommand(name="pwd",
  61         logEnviron=False, command=["pwd"]))
  62     factory.addStep(steps.ShellCommand(name="true",
  63         logEnviron=False, command=["true"]))
  64     factory.addStep(steps.ShellCommand(name="echo",
  65         logEnviron=False, command=["echo", package]))
  66     factory.addSteps(package_and_upload(package, package_dest, package_url))
  67
  68     return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory)
  69
  70
  71 def compute_build_infos():
  72     @util.renderer
  73     def compute(props):
  74         import re, hashlib
  75         build_file = props.getProperty("build")
  76         package_dest = "{}/{}".format(E.RELEASE_PATH, build_file)
  77         version = re.match(r"{0}_(.*).tar.gz".format("test"), build_file).group(1)
  78         with open(package_dest, "rb") as f:
  79             sha = hashlib.sha256(f.read()).hexdigest()
  80         return {
  81                 "build_version": version,
  82                 "build_hash": sha,
  83                 }
  84     return compute
  85
  86 @util.renderer
  87 def puppet_host(props):
  88     return E.PUPPET_HOST
  89
  90 def deploy_factory():
  91     package_dest = util.Interpolate("{}/%(prop:build)s".format(E.RELEASE_PATH))
  92
  93     factory = util.BuildFactory()
  94     factory.addStep(steps.MasterShellCommand(command=["test", "-f", package_dest]))
  95     factory.addStep(steps.SetProperties(properties=compute_build_infos()))
  96     factory.addStep(LdapPush(environment=util.Property("environment"),
  97         build_version=util.Property("build_version"),
  98         build_hash=util.Property("build_hash"),
  99         ldap_password=util.Secret("ldap")))
 100     factory.addStep(steps.MasterShellCommand(command=[
 101         "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host]))
 102     return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory)
 103
 104 from twisted.internet import defer
 105 from buildbot.process.buildstep import FAILURE
 106 from buildbot.process.buildstep import SUCCESS
 107 from buildbot.process.buildstep import BuildStep
 108
 109 class LdapPush(BuildStep):
 110     name = "LdapPush"
 111     renderables = ["environment", "build_version", "build_hash", "ldap_password"]
 112
 113     def __init__(self, **kwargs):
 114         self.environment = kwargs.pop("environment")
 115         self.build_version = kwargs.pop("build_version")
 116         self.build_hash = kwargs.pop("build_hash")
 117         self.ldap_password = kwargs.pop("ldap_password")
 118         self.ldap_host = kwargs.pop("ldap_host", E.LDAP_HOST)
 119         super().__init__(**kwargs)
 120
 121     def run(self):
 122         import json
 123         from ldap3 import Reader, Writer, Server, Connection, ObjectDef
 124         server = Server(self.ldap_host)
 125         conn = Connection(server,
 126                 user=E.LDAP_DN,
 127                 password=self.ldap_password)
 128         conn.bind()
 129         obj = ObjectDef("immaePuppetClass", conn)
 130         r = Reader(conn, obj,
 131                 "cn=test.{},{}".format(self.environment, E.LDAP_ROLES_BASE))
 132         r.search()
 133         if len(r) > 0:
 134             w = Writer.from_cursor(r)
 135             for value in w[0].immaePuppetJson.values:
 136                 config = json.loads(value)
 137                 if "test_version" in config:
 138                     config["test_version"] = self.build_version
 139                     config["test_sha256"] = self.build_hash
 140                     w[0].immaePuppetJson -= value
 141                     w[0].immaePuppetJson += json.dumps(config, indent="  ")
 142                     w.commit()
 143                     return defer.succeed(SUCCESS)
 144         return defer.succeed(FAILURE)