[perso/Immae/Config/Nix.git] / flakes / private / buildbot / common / build_helpers.py

from buildbot.plugins import util, steps, schedulers
from buildbot_buildslist import BuildsList
from shutil import which

__all__ = [
        "force_scheduler", "deploy_scheduler", "git_hook_scheduler",
        "clean_branch", "package_and_upload", "AppriseStatusPush",
        "XMPPStatusPush", "NixShellCommand",
        "all_builder_names", "compute_build_infos", "deploy_ssh_command",
        "configure_apprise_push",
        "configure_xmpp_push", "deploy_hook_scheduler",
        ]

# Small helpers"
@util.renderer
def clean_branch(props):
    if props.hasProperty("branch") and len(props["branch"]) > 0:
        return props["branch"].replace("/", "_")
    else:
        return "HEAD"

def package_and_upload(package, package_dest, package_url):
    return [
            steps.ShellCommand(name="build package",
                logEnviron=False, haltOnFailure=True,
                command=["git", "archive", "HEAD", "-o", package]),

            steps.FileUpload(name="upload package", workersrc=package,
                masterdest=package_dest,
                url=package_url, mode=0o644),

            steps.ShellCommand(name="cleanup package", logEnviron=False,
                haltOnFailure=True, alwaysRun=True,
                command=["rm", "-f", package]),
            ]

# Steps
class NixShellCommand(steps.ShellCommand):
    def __init__(self, command=None, nixPackages=[], pure=True, nixFile=None, nixIncludes={}, nixArgs={}, **kwargs):
        oldpath = kwargs.get("env", {}).get("PATH", None)
        if which("nix-shell", path=oldpath) is None:
            kwargs["env"] = kwargs.get("env", {})
            if isinstance(oldpath, str):
                kwargs["env"]["PATH"] = "/run/current-system/sw/bin:" + oldpath
            elif isinstance(oldpath, list):
                kwargs["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath
        nixcommand = ["nix-shell"]
        for k, v in nixArgs.items():
            nixcommand.append("--arg")
            nixcommand.append(k)
            nixcommand.append(v)
        if pure:
            nixcommand.append("--pure")
        for k, v in nixIncludes.items():
            nixcommand.append("-I")
            nixcommand.append("{}={}".format(k, v))
        nixcommand.append("--run")
        nixcommand.append(command)
        if len(nixPackages) > 0:
            nixcommand.append("-p")
            nixcommand += nixPackages
        elif nixFile is not None:
            nixcommand.append(nixFile)
        super().__init__(command=nixcommand, **kwargs)

# Schedulers
def force_scheduler(name, builders, nobranch=False):
    if nobranch:
        branch = util.FixedParameter(name="branch", default="")
    else:
        branch=util.StringParameter(name="branch", label="Git reference (tag, branch)", required=True)

    return schedulers.ForceScheduler(name=name,
        label="Force build", buttonName="Force build",
        reason=util.StringParameter(name="reason", label="Reason", default="Force build"),
        codebases=[
            util.CodebaseParameter("",
                branch=branch,
                revision=util.FixedParameter(name="revision", default=""),
                repository=util.FixedParameter(name="repository", default=""),
                project=util.FixedParameter(name="project", default=""),
                ),
            ],
        username=util.FixedParameter(name="username", default="Web button"),
        builderNames=builders)

def deploy_scheduler(name, builders):
    return schedulers.ForceScheduler(name=name,
        builderNames=builders,
        label="Deploy built package", buttonName="Deploy",
        username=util.FixedParameter(name="username", default="Web button"),
        codebases=[
            util.CodebaseParameter(codebase="",
                branch=util.FixedParameter(name="branch", default=""),
                revision=util.FixedParameter(name="revision", default=""),
                repository=util.FixedParameter(name="repository", default=""),
                project=util.FixedParameter(name="project", default=""))],
        reason=util.FixedParameter(name="reason", default="Deploy"),
        properties=[
            util.ChoiceStringParameter(label="Environment",
                name="environment", default="integration",
                choices=["integration", "production"]),
            BuildsList(label="Build to deploy", name="build"),
            ]
        )

def git_hook_scheduler(project, builders=[], timer=1):
    if len(builders) == 0:
        builders = ["{}_build".format(project)]
    return schedulers.AnyBranchScheduler(
            change_filter=util.ChangeFilter(category="gitolite-hooks", project=project),
            name="{}_git_hook".format(project), treeStableTimer=timer, builderNames=builders)

def deploy_hook_scheduler(project, builders, timer=1):
    return schedulers.AnyBranchScheduler(
            change_filter=util.ChangeFilter(category="deploy_webhook", project=project),
            name="{}_deploy".format(project), treeStableTimer=timer, builderNames=builders)

# Builders
def all_builder_names(c):
    return [builder.name for builder in c['builders']]

# Apprise/XMPP status push
from buildbot.reporters.http import HttpStatusPushBase
from twisted.internet import defer
from twisted.python import log
from buildbot.reporters import utils
from buildbot.process import results
from twisted.words.protocols.jabber.jid import JID
from wokkel import client, xmppim
from functools import partial
import apprise

class AppriseStatusPush(HttpStatusPushBase):
    name = "AppriseStatusPush"

    @defer.inlineCallbacks
    def reconfigService(self, appriseUrls, **kwargs):
        self.appriseUrls = appriseUrls
        yield HttpStatusPushBase.reconfigService(self, **kwargs)

    @defer.inlineCallbacks
    def send(self, build):
        yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
        appobject = apprise.Apprise()
        message = self.format(build)
        for url in self.appriseUrls:
            appobject.add(url.format(**message))
        yield appobject.notify(title=message["title"], body=message["text"])

    def format(self, build):
        if "environment" in build["properties"]:
            msg = "{} environment".format(build["properties"]["environment"][0])
            if "build" in build["properties"]:
                msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
        elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
            msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
        else:
            msg = "build"

        if build["complete"]:
            timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
            hours, rest = divmod(timedelta, 3600)
            minutes, seconds = divmod(rest, 60)
            if hours > 0:
                duration = "{}h {}min {}s".format(hours, minutes, seconds)
            elif minutes > 0:
                duration = "{}min {}s".format(minutes, seconds)
            else:
                duration = "{}s".format(seconds)

            text = "Build {} ({}) of {}'s {} was {} in {}.".format(
                    build["number"], build["url"],
                    build["builder"]["name"],
                    msg,
                    results.Results[build["results"]],
                    duration,
                    )
        else:
            text = "Build {} ({}) of {}'s {} started.".format(
                    build["number"], build["url"],
                    build["builder"]["name"],
                    msg,
                    )
        return {
                "username": "Buildbot",
                "image_url": "http://docs.buildbot.net/current/_static/icon.png",
                "text": text,
                "title": "",
                }

def configure_apprise_push(c, secrets_file, builders):
    c['services'].append(AppriseStatusPush(
        name="apprise_status", builders=builders,
        appriseUrls=open(secrets_file + "/apprise_webhooks", "r").read().split("\n")))

class XMPPStatusPush(HttpStatusPushBase):
    name = "XMPPStatusPush"

    @defer.inlineCallbacks
    def reconfigService(self, password, recipients, **kwargs):
        yield HttpStatusPushBase.reconfigService(self, **kwargs)
        self.password = password
        self.recipients = recipients

    @defer.inlineCallbacks
    def send(self, build):
        yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
        body = self.format(build)
        factory = client.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self.password)
        d = client.clientCreator(factory)
        def send_message(recipient, stream):
            message = xmppim.Message(recipient=JID(recipient), body=body)
            message.stanzaType = 'chat'
            stream.send(message.toElement())
            # To allow chaining
            return stream
        for recipient in self.recipients:
            d.addCallback(partial(send_message, recipient))
        d.addCallback(lambda _: factory.streamManager.xmlstream.sendFooter())
        d.addErrback(log.err)

    def format(self, build):
        if "environment" in build["properties"]:
            msg = "{} environment".format(build["properties"]["environment"][0])
            if "build" in build["properties"]:
                msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
        elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
            msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
        else:
            msg = "build"

        if build["complete"]:
            timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
            hours, rest = divmod(timedelta, 3600)
            minutes, seconds = divmod(rest, 60)
            if hours > 0:
                duration = "{}h {}min {}s".format(hours, minutes, seconds)
            elif minutes > 0:
                duration = "{}min {}s".format(minutes, seconds)
            else:
                duration = "{}s".format(seconds)

            text = "Build {} ( {} ) of {}'s {} was {} in {}.".format(
                    build["number"], build["url"],
                    build["builder"]["name"],
                    msg,
                    results.Results[build["results"]],
                    duration,
                    )
        else:
            text = "Build {} ( {} ) of {}'s {} started.".format(
                    build["number"], build["url"],
                    build["builder"]["name"],
                    msg,
                    )

        return text

def configure_xmpp_push(c, secrets_file, builders, recipients):
    c['services'].append(XMPPStatusPush(
        name="xmpp_status", builders=builders, recipients=recipients,
        password=open(secrets_file + "/notify_xmpp_password", "r").read().rstrip()))

# LDAP edit
from buildbot.process.buildstep import FAILURE
from buildbot.process.buildstep import SUCCESS
from buildbot.process.buildstep import BuildStep

def compute_build_infos(prefix, release_path):
    @util.renderer
    def compute(props):
        import re, hashlib
        build_file = props.getProperty("build")
        package_dest = "{}/{}".format(release_path, build_file)
        version = re.match(r"{0}_(.*).tar.gz".format(prefix), build_file).group(1)
        with open(package_dest, "rb") as f:
            sha = hashlib.sha256(f.read()).hexdigest()
        return {
                "build_version": version,
                "build_hash": sha,
                }
    return compute

def deploy_ssh_command(ssh_key_path, deploy_hosts):
    @util.renderer
    def compute(props):
        environment = props["environment"] if props.hasProperty("environment") else "integration"
        ssh_command = [
                "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no",
                "-i", ssh_key_path ]
        return ssh_command + deploy_hosts.get(environment, ["host.invalid"])
    return compute
Commit	Line	Data
e2b96bf5 IB	1	from buildbot.plugins import util, steps, schedulers
e2b96bf5 IB	2	from buildbot_buildslist import BuildsList
6b017278	3	from shutil import which
e2b96bf5 IB	4
e2b96bf5 IB	5	__all__ = [
1a64deeb IB	6	"force_scheduler", "deploy_scheduler", "git_hook_scheduler",
	7	"clean_branch", "package_and_upload", "AppriseStatusPush",
	8	"XMPPStatusPush", "NixShellCommand",
	9	"all_builder_names", "compute_build_infos", "deploy_ssh_command",
	10	"configure_apprise_push",
	11	"configure_xmpp_push", "deploy_hook_scheduler",
e2b96bf5 IB	12	]
	13
	14	# Small helpers"
	15	@util.renderer
	16	def clean_branch(props):
	17	if props.hasProperty("branch") and len(props["branch"]) > 0:
	18	return props["branch"].replace("/", "_")
	19	else:
	20	return "HEAD"
	21
	22	def package_and_upload(package, package_dest, package_url):
	23	return [
	24	steps.ShellCommand(name="build package",
1a64deeb	25	logEnviron=False, haltOnFailure=True,
e2b96bf5 IB	26	command=["git", "archive", "HEAD", "-o", package]),
	27
	28	steps.FileUpload(name="upload package", workersrc=package,
1a64deeb	29	masterdest=package_dest,
e2b96bf5 IB	30	url=package_url, mode=0o644),
	31
	32	steps.ShellCommand(name="cleanup package", logEnviron=False,
1a64deeb	33	haltOnFailure=True, alwaysRun=True,
e2b96bf5 IB	34	command=["rm", "-f", package]),
	35	]
	36
6b017278 IB	37	# Steps
6b017278 IB	38	class NixShellCommand(steps.ShellCommand):
1a64deeb	39	def __init__(self, command=None, nixPackages=[], pure=True, nixFile=None, nixIncludes={}, nixArgs={}, **kwargs):
6b017278 IB	40	oldpath = kwargs.get("env", {}).get("PATH", None)
	41	if which("nix-shell", path=oldpath) is None:
	42	kwargs["env"] = kwargs.get("env", {})
	43	if isinstance(oldpath, str):
	44	kwargs["env"]["PATH"] = "/run/current-system/sw/bin:" + oldpath
	45	elif isinstance(oldpath, list):
	46	kwargs["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath
	47	nixcommand = ["nix-shell"]
1a64deeb IB	48	for k, v in nixArgs.items():
	49	nixcommand.append("--arg")
	50	nixcommand.append(k)
	51	nixcommand.append(v)
6b017278 IB	52	if pure:
6b017278 IB	53	nixcommand.append("--pure")
1a64deeb IB	54	for k, v in nixIncludes.items():
	55	nixcommand.append("-I")
	56	nixcommand.append("{}={}".format(k, v))
6b017278 IB	57	nixcommand.append("--run")
6b017278 IB	58	nixcommand.append(command)
1a64deeb IB	59	if len(nixPackages) > 0:
	60	nixcommand.append("-p")
	61	nixcommand += nixPackages
	62	elif nixFile is not None:
	63	nixcommand.append(nixFile)
6b017278 IB	64	super().__init__(command=nixcommand, **kwargs)
6b017278 IB	65
e2b96bf5	66	# Schedulers
1a64deeb IB	67	def force_scheduler(name, builders, nobranch=False):
	68	if nobranch:
	69	branch = util.FixedParameter(name="branch", default="")
	70	else:
	71	branch=util.StringParameter(name="branch", label="Git reference (tag, branch)", required=True)
	72
e2b96bf5 IB	73	return schedulers.ForceScheduler(name=name,
	74	label="Force build", buttonName="Force build",
	75	reason=util.StringParameter(name="reason", label="Reason", default="Force build"),
	76	codebases=[
	77	util.CodebaseParameter("",
1a64deeb	78	branch=branch,
e2b96bf5 IB	79	revision=util.FixedParameter(name="revision", default=""),
	80	repository=util.FixedParameter(name="repository", default=""),
	81	project=util.FixedParameter(name="project", default=""),
	82	),
	83	],
	84	username=util.FixedParameter(name="username", default="Web button"),
	85	builderNames=builders)
	86
	87	def deploy_scheduler(name, builders):
	88	return schedulers.ForceScheduler(name=name,
	89	builderNames=builders,
	90	label="Deploy built package", buttonName="Deploy",
	91	username=util.FixedParameter(name="username", default="Web button"),
	92	codebases=[
	93	util.CodebaseParameter(codebase="",
	94	branch=util.FixedParameter(name="branch", default=""),
	95	revision=util.FixedParameter(name="revision", default=""),
	96	repository=util.FixedParameter(name="repository", default=""),
	97	project=util.FixedParameter(name="project", default=""))],
	98	reason=util.FixedParameter(name="reason", default="Deploy"),
	99	properties=[
	100	util.ChoiceStringParameter(label="Environment",
	101	name="environment", default="integration",
	102	choices=["integration", "production"]),
	103	BuildsList(label="Build to deploy", name="build"),
	104	]
	105	)
	106
1a64deeb IB	107	def git_hook_scheduler(project, builders=[], timer=1):
	108	if len(builders) == 0:
	109	builders = ["{}_build".format(project)]
	110	return schedulers.AnyBranchScheduler(
	111	change_filter=util.ChangeFilter(category="gitolite-hooks", project=project),
	112	name="{}_git_hook".format(project), treeStableTimer=timer, builderNames=builders)
	113
	114	def deploy_hook_scheduler(project, builders, timer=1):
e2b96bf5	115	return schedulers.AnyBranchScheduler(
1a64deeb IB	116	change_filter=util.ChangeFilter(category="deploy_webhook", project=project),
1a64deeb IB	117	name="{}_deploy".format(project), treeStableTimer=timer, builderNames=builders)
e2b96bf5	118
1a64deeb IB	119	# Builders
	120	def all_builder_names(c):
	121	return [builder.name for builder in c['builders']]
	122
	123	# Apprise/XMPP status push
e2b96bf5 IB	124	from buildbot.reporters.http import HttpStatusPushBase
	125	from twisted.internet import defer
	126	from twisted.python import log
e2b96bf5 IB	127	from buildbot.reporters import utils
e2b96bf5 IB	128	from buildbot.process import results
256d607c IB	129	from twisted.words.protocols.jabber.jid import JID
256d607c IB	130	from wokkel import client, xmppim
7158b76e	131	from functools import partial
1a64deeb	132	import apprise
e2b96bf5	133
1a64deeb IB	134	class AppriseStatusPush(HttpStatusPushBase):
1a64deeb IB	135	name = "AppriseStatusPush"
e2b96bf5 IB	136
e2b96bf5 IB	137	@defer.inlineCallbacks
1a64deeb IB	138	def reconfigService(self, appriseUrls, **kwargs):
1a64deeb IB	139	self.appriseUrls = appriseUrls
e2b96bf5	140	yield HttpStatusPushBase.reconfigService(self, **kwargs)
e2b96bf5 IB	141
	142	@defer.inlineCallbacks
	143	def send(self, build):
	144	yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
1a64deeb IB	145	appobject = apprise.Apprise()
	146	message = self.format(build)
	147	for url in self.appriseUrls:
	148	appobject.add(url.format(**message))
	149	yield appobject.notify(title=message["title"], body=message["text"])
e2b96bf5 IB	150
e2b96bf5 IB	151	def format(self, build):
e2b96bf5 IB	152	if "environment" in build["properties"]:
	153	msg = "{} environment".format(build["properties"]["environment"][0])
	154	if "build" in build["properties"]:
	155	msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
e42f8192	156	elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
e2b96bf5 IB	157	msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
	158	else:
	159	msg = "build"
	160
	161	if build["complete"]:
	162	timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
	163	hours, rest = divmod(timedelta, 3600)
	164	minutes, seconds = divmod(rest, 60)
	165	if hours > 0:
	166	duration = "{}h {}min {}s".format(hours, minutes, seconds)
	167	elif minutes > 0:
	168	duration = "{}min {}s".format(minutes, seconds)
	169	else:
	170	duration = "{}s".format(seconds)
	171
1a64deeb IB	172	text = "Build {} ({}) of {}'s {} was {} in {}.".format(
1a64deeb IB	173	build["number"], build["url"],
e2b96bf5 IB	174	build["builder"]["name"],
	175	msg,
	176	results.Results[build["results"]],
	177	duration,
	178	)
e2b96bf5	179	else:
1a64deeb IB	180	text = "Build {} ({}) of {}'s {} started.".format(
1a64deeb IB	181	build["number"], build["url"],
e2b96bf5 IB	182	build["builder"]["name"],
	183	msg,
	184	)
e2b96bf5 IB	185	return {
e2b96bf5 IB	186	"username": "Buildbot",
1a64deeb	187	"image_url": "http://docs.buildbot.net/current/_static/icon.png",
e2b96bf5	188	"text": text,
1a64deeb	189	"title": "",
e2b96bf5 IB	190	}
e2b96bf5 IB	191
1a64deeb IB	192	def configure_apprise_push(c, secrets_file, builders):
	193	c['services'].append(AppriseStatusPush(
	194	name="apprise_status", builders=builders,
	195	appriseUrls=open(secrets_file + "/apprise_webhooks", "r").read().split("\n")))
	196
256d607c IB	197	class XMPPStatusPush(HttpStatusPushBase):
	198	name = "XMPPStatusPush"
	199
	200	@defer.inlineCallbacks
	201	def reconfigService(self, password, recipients, **kwargs):
	202	yield HttpStatusPushBase.reconfigService(self, **kwargs)
	203	self.password = password
	204	self.recipients = recipients
	205
	206	@defer.inlineCallbacks
	207	def send(self, build):
	208	yield utils.getDetailsForBuild(self.master, build, wantProperties=True)
	209	body = self.format(build)
	210	factory = client.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self.password)
256d607c	211	d = client.clientCreator(factory)
7158b76e	212	def send_message(recipient, stream):
256d607c IB	213	message = xmppim.Message(recipient=JID(recipient), body=body)
	214	message.stanzaType = 'chat'
	215	stream.send(message.toElement())
	216	# To allow chaining
	217	return stream
	218	for recipient in self.recipients:
7158b76e	219	d.addCallback(partial(send_message, recipient))
256d607c IB	220	d.addCallback(lambda _: factory.streamManager.xmlstream.sendFooter())
	221	d.addErrback(log.err)
	222
	223	def format(self, build):
	224	if "environment" in build["properties"]:
	225	msg = "{} environment".format(build["properties"]["environment"][0])
	226	if "build" in build["properties"]:
	227	msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg
e42f8192	228	elif len(build["buildset"]["sourcestamps"][0]["branch"] or []) > 0:
256d607c IB	229	msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"])
	230	else:
	231	msg = "build"
	232
	233	if build["complete"]:
	234	timedelta = int((build["complete_at"] - build["started_at"]).total_seconds())
	235	hours, rest = divmod(timedelta, 3600)
	236	minutes, seconds = divmod(rest, 60)
	237	if hours > 0:
	238	duration = "{}h {}min {}s".format(hours, minutes, seconds)
	239	elif minutes > 0:
	240	duration = "{}min {}s".format(minutes, seconds)
	241	else:
	242	duration = "{}s".format(seconds)
e2b96bf5	243
256d607c	244	text = "Build {} ( {} ) of {}'s {} was {} in {}.".format(
1a64deeb	245	build["number"], build["url"],
256d607c IB	246	build["builder"]["name"],
	247	msg,
	248	results.Results[build["results"]],
	249	duration,
	250	)
	251	else:
	252	text = "Build {} ( {} ) of {}'s {} started.".format(
1a64deeb	253	build["number"], build["url"],
256d607c IB	254	build["builder"]["name"],
	255	msg,
	256	)
e2b96bf5	257
256d607c	258	return text
1a64deeb IB	259
	260	def configure_xmpp_push(c, secrets_file, builders, recipients):
	261	c['services'].append(XMPPStatusPush(
	262	name="xmpp_status", builders=builders, recipients=recipients,
	263	password=open(secrets_file + "/notify_xmpp_password", "r").read().rstrip()))
	264
	265	# LDAP edit
	266	from buildbot.process.buildstep import FAILURE
	267	from buildbot.process.buildstep import SUCCESS
	268	from buildbot.process.buildstep import BuildStep
	269
	270	def compute_build_infos(prefix, release_path):
	271	@util.renderer
	272	def compute(props):
	273	import re, hashlib
	274	build_file = props.getProperty("build")
	275	package_dest = "{}/{}".format(release_path, build_file)
	276	version = re.match(r"{0}_(.*).tar.gz".format(prefix), build_file).group(1)
	277	with open(package_dest, "rb") as f:
	278	sha = hashlib.sha256(f.read()).hexdigest()
	279	return {
	280	"build_version": version,
	281	"build_hash": sha,
	282	}
	283	return compute
	284
	285	def deploy_ssh_command(ssh_key_path, deploy_hosts):
	286	@util.renderer
	287	def compute(props):
	288	environment = props["environment"] if props.hasProperty("environment") else "integration"
	289	ssh_command = [
	290	"ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no",
	291	"-i", ssh_key_path ]
	292	return ssh_command + deploy_hosts.get(environment, ["host.invalid"])
	293	return compute