From 8d213e2b1c934f6861f76aad5eb7c11097fa97de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Wed, 22 May 2019 20:55:28 +0200 Subject: Move rest of the modules outside of nixops --- modules/private/buildbot/common/build_helpers.py | 256 +++++++++++++++++++++ modules/private/buildbot/common/master.cfg | 69 ++++++ modules/private/buildbot/default.nix | 198 ++++++++++++++++ .../private/buildbot/projects/caldance/__init__.py | 190 +++++++++++++++ .../buildbot/projects/cryptoportfolio/__init__.py | 169 ++++++++++++++ modules/private/buildbot/projects/test/__init__.py | 188 +++++++++++++++ 6 files changed, 1070 insertions(+) create mode 100644 modules/private/buildbot/common/build_helpers.py create mode 100644 modules/private/buildbot/common/master.cfg create mode 100644 modules/private/buildbot/default.nix create mode 100644 modules/private/buildbot/projects/caldance/__init__.py create mode 100644 modules/private/buildbot/projects/cryptoportfolio/__init__.py create mode 100644 modules/private/buildbot/projects/test/__init__.py (limited to 'modules/private/buildbot') diff --git a/modules/private/buildbot/common/build_helpers.py b/modules/private/buildbot/common/build_helpers.py new file mode 100644 index 0000000..384b1ac --- /dev/null +++ b/modules/private/buildbot/common/build_helpers.py @@ -0,0 +1,256 @@ +from buildbot.plugins import util, steps, schedulers +from buildbot_buildslist import BuildsList + +__all__ = [ + "force_scheduler", "deploy_scheduler", "hook_scheduler", + "clean_branch", "package_and_upload", "SlackStatusPush", + "XMPPStatusPush" + ] + +# Small helpers" +@util.renderer +def clean_branch(props): + if props.hasProperty("branch") and len(props["branch"]) > 0: + return props["branch"].replace("/", "_") + else: + return "HEAD" + +def package_and_upload(package, package_dest, package_url): + return [ + steps.ShellCommand(name="build package", + logEnviron=False, haltOnFailure=True, workdir="source", + command=["git", "archive", "HEAD", "-o", package]), + + steps.FileUpload(name="upload package", workersrc=package, + workdir="source", masterdest=package_dest, + url=package_url, mode=0o644), + + steps.ShellCommand(name="cleanup package", logEnviron=False, + haltOnFailure=True, workdir="source", alwaysRun=True, + command=["rm", "-f", package]), + ] + +# Schedulers +def force_scheduler(name, builders): + return schedulers.ForceScheduler(name=name, + label="Force build", buttonName="Force build", + reason=util.StringParameter(name="reason", label="Reason", default="Force build"), + codebases=[ + util.CodebaseParameter("", + branch=util.StringParameter( + name="branch", label="Git reference (tag, branch)", required=True), + revision=util.FixedParameter(name="revision", default=""), + repository=util.FixedParameter(name="repository", default=""), + project=util.FixedParameter(name="project", default=""), + ), + ], + username=util.FixedParameter(name="username", default="Web button"), + builderNames=builders) + +def deploy_scheduler(name, builders): + return schedulers.ForceScheduler(name=name, + builderNames=builders, + label="Deploy built package", buttonName="Deploy", + username=util.FixedParameter(name="username", default="Web button"), + codebases=[ + util.CodebaseParameter(codebase="", + branch=util.FixedParameter(name="branch", default=""), + revision=util.FixedParameter(name="revision", default=""), + repository=util.FixedParameter(name="repository", default=""), + project=util.FixedParameter(name="project", default=""))], + reason=util.FixedParameter(name="reason", default="Deploy"), + properties=[ + util.ChoiceStringParameter(label="Environment", + name="environment", default="integration", + choices=["integration", "production"]), + BuildsList(label="Build to deploy", name="build"), + ] + ) + +def hook_scheduler(project, timer=10): + return schedulers.AnyBranchScheduler( + change_filter=util.ChangeFilter(category="hooks", project=project), + name=project, treeStableTimer=timer, builderNames=["{}_build".format(project)]) + +# Slack/XMPP status push +from buildbot.reporters.http import HttpStatusPushBase +from twisted.internet import defer +from twisted.python import log +from buildbot.util import httpclientservice +from buildbot.reporters import utils +from buildbot.process import results +from twisted.words.protocols.jabber.jid import JID +from wokkel import client, xmppim +from functools import partial + +class SlackStatusPush(HttpStatusPushBase): + name = "SlackStatusPush" + + @defer.inlineCallbacks + def reconfigService(self, serverUrl, **kwargs): + yield HttpStatusPushBase.reconfigService(self, **kwargs) + self._http = yield httpclientservice.HTTPClientService.getService( + self.master, serverUrl) + + @defer.inlineCallbacks + def send(self, build): + yield utils.getDetailsForBuild(self.master, build, wantProperties=True) + response = yield self._http.post("", json=self.format(build)) + if response.code != 200: + log.msg("%s: unable to upload status: %s" % + (response.code, response.content)) + + def format(self, build): + colors = [ + "#36A64F", # success + "#F1E903", # warnings + "#DA0505", # failure + "#FFFFFF", # skipped + "#000000", # exception + "#FFFFFF", # retry + "#D02CA9", # cancelled + ] + + if "environment" in build["properties"]: + msg = "{} environment".format(build["properties"]["environment"][0]) + if "build" in build["properties"]: + msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg + elif len(build["buildset"]["sourcestamps"][0]["branch"]) > 0: + msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"]) + else: + msg = "build" + + if build["complete"]: + timedelta = int((build["complete_at"] - build["started_at"]).total_seconds()) + hours, rest = divmod(timedelta, 3600) + minutes, seconds = divmod(rest, 60) + if hours > 0: + duration = "{}h {}min {}s".format(hours, minutes, seconds) + elif minutes > 0: + duration = "{}min {}s".format(minutes, seconds) + else: + duration = "{}s".format(seconds) + + text = "Build <{}|{}> of {}'s {} was {} in {}.".format( + build["url"], build["buildid"], + build["builder"]["name"], + msg, + results.Results[build["results"]], + duration, + ) + fields = [ + { + "title": "Build", + "value": "<{}|{}>".format(build["url"], build["buildid"]), + "short": True, + }, + { + "title": "Project", + "value": build["builder"]["name"], + "short": True, + }, + { + "title": "Build status", + "value": results.Results[build["results"]], + "short": True, + }, + { + "title": "Build duration", + "value": duration, + "short": True, + }, + ] + if "environment" in build["properties"]: + fields.append({ + "title": "Environment", + "value": build["properties"]["environment"][0], + "short": True, + }) + if "build" in build["properties"]: + fields.append({ + "title": "Archive", + "value": build["properties"]["build"][0], + "short": True, + }) + attachments = [{ + "fallback": "", + "color": colors[build["results"]], + "fields": fields + }] + else: + text = "Build <{}|{}> of {}'s {} started.".format( + build["url"], build["buildid"], + build["builder"]["name"], + msg, + ) + attachments = [] + + return { + "username": "Buildbot", + "icon_url": "http://docs.buildbot.net/current/_static/icon.png", + "text": text, + "attachments": attachments, + } + +class XMPPStatusPush(HttpStatusPushBase): + name = "XMPPStatusPush" + + @defer.inlineCallbacks + def reconfigService(self, password, recipients, **kwargs): + yield HttpStatusPushBase.reconfigService(self, **kwargs) + self.password = password + self.recipients = recipients + + @defer.inlineCallbacks + def send(self, build): + yield utils.getDetailsForBuild(self.master, build, wantProperties=True) + body = self.format(build) + factory = client.DeferredClientFactory(JID("notify_bot@immae.fr/buildbot"), self.password) + d = client.clientCreator(factory) + def send_message(recipient, stream): + message = xmppim.Message(recipient=JID(recipient), body=body) + message.stanzaType = 'chat' + stream.send(message.toElement()) + # To allow chaining + return stream + for recipient in self.recipients: + d.addCallback(partial(send_message, recipient)) + d.addCallback(lambda _: factory.streamManager.xmlstream.sendFooter()) + d.addErrback(log.err) + + def format(self, build): + if "environment" in build["properties"]: + msg = "{} environment".format(build["properties"]["environment"][0]) + if "build" in build["properties"]: + msg = "of archive {} in ".format(build["properties"]["build"][0]) + msg + elif len(build["buildset"]["sourcestamps"][0]["branch"]) > 0: + msg = "revision {}".format(build["buildset"]["sourcestamps"][0]["branch"]) + else: + msg = "build" + + if build["complete"]: + timedelta = int((build["complete_at"] - build["started_at"]).total_seconds()) + hours, rest = divmod(timedelta, 3600) + minutes, seconds = divmod(rest, 60) + if hours > 0: + duration = "{}h {}min {}s".format(hours, minutes, seconds) + elif minutes > 0: + duration = "{}min {}s".format(minutes, seconds) + else: + duration = "{}s".format(seconds) + + text = "Build {} ( {} ) of {}'s {} was {} in {}.".format( + build["buildid"], build["url"], + build["builder"]["name"], + msg, + results.Results[build["results"]], + duration, + ) + else: + text = "Build {} ( {} ) of {}'s {} started.".format( + build["buildid"], build["url"], + build["builder"]["name"], + msg, + ) + + return text diff --git a/modules/private/buildbot/common/master.cfg b/modules/private/buildbot/common/master.cfg new file mode 100644 index 0000000..abe08e0 --- /dev/null +++ b/modules/private/buildbot/common/master.cfg @@ -0,0 +1,69 @@ +# -*- python -*- +# ex: set filetype=python: + +from buildbot.plugins import secrets, util, webhooks +from buildbot.util import bytes2unicode +import re +import os +from buildbot_config import E, configure +import json + +class CustomBase(webhooks.base): + def getChanges(self, request): + try: + content = request.content.read() + args = json.loads(bytes2unicode(content)) + except Exception as e: + raise ValueError("Error loading JSON: " + str(e)) + + args.setdefault("comments", "") + args.setdefault("repository", "") + args.setdefault("author", args.get("who")) + + return ([args], None) + +userInfoProvider = util.LdapUserInfo( + uri=E.LDAP_URL, + bindUser=E.LDAP_ADMIN_USER, + bindPw=open(E.SECRETS_FILE + "/ldap", "r").read().rstrip(), + accountBase=E.LDAP_BASE, + accountPattern=E.LDAP_PATTERN, + accountFullName='cn', + accountEmail='mail', + avatarData="jpegPhoto", + groupBase=E.LDAP_BASE, + groupName="cn", + groupMemberPattern=E.LDAP_GROUP_PATTERN, + ) + +c = BuildmasterConfig = { + "title": E.TITLE, + "titleURL": E.TITLE_URL, + "db": { + "db_url": "sqlite:///state.sqlite" + }, + "protocols": { "pb": { "port": E.PB_SOCKET } }, + "workers": [], + "change_source": [], + "schedulers": [], + "builders": [], + "services": [], + "secretsProviders": [ + secrets.SecretInAFile(E.SECRETS_FILE), + ], + "www": { + "change_hook_dialects": { "base": { "custom_class": CustomBase } }, + "plugins": { + "waterfall_view": {}, + "console_view": {}, + "grid_view": {}, + "buildslist": {}, + }, + "auth": util.RemoteUserAuth( + header=b"X-Remote-User", + userInfoProvider=userInfoProvider, + headerRegex=re.compile(br"(?P[^ @]+)")), + } + } + +configure(c) diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix new file mode 100644 index 0000000..fa6a6f2 --- /dev/null +++ b/modules/private/buildbot/default.nix @@ -0,0 +1,198 @@ +{ lib, pkgs, config, myconfig, ... }: +let + varDir = "/var/lib/buildbot"; + buildbot_common = pkgs.python3Packages.buildPythonPackage rec { + name = "buildbot_common"; + src = ./common; + format = "other"; + installPhase = '' + mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages} + cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common + ''; + }; + buildbot = pkgs.python3Packages.buildbot-full; +in +{ + options = { + myServices.buildbot.enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to enable buildbot. + ''; + }; + }; + + config = lib.mkIf config.myServices.buildbot.enable { + ids.uids.buildbot = myconfig.env.buildbot.user.uid; + ids.gids.buildbot = myconfig.env.buildbot.user.gid; + + users.groups.buildbot.gid = config.ids.gids.buildbot; + users.users.buildbot = { + name = "buildbot"; + uid = config.ids.uids.buildbot; + group = "buildbot"; + description = "Buildbot user"; + home = varDir; + extraGroups = [ "keys" ]; + }; + + services.websites.tools.vhostConfs.git.extraConfig = lib.attrsets.mapAttrsToList (k: project: '' + RedirectMatch permanent "^/buildbot/${project.name}$" "/buildbot/${project.name}/" + RewriteEngine On + RewriteRule ^/buildbot/${project.name}/ws(.*)$ unix:///run/buildbot/${project.name}.sock|ws://git.immae.eu/ws$1 [P,NE,QSA,L] + ProxyPass /buildbot/${project.name}/ unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/ + ProxyPassReverse /buildbot/${project.name}/ unix:///run/buildbot/${project.name}.sock|http://${project.name}-git.immae.eu/ + + Use LDAPConnect + Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu + + SetEnvIf X-Url-Scheme https HTTPS=1 + ProxyPreserveHost On + + + + Require local + Require ldap-group cn=users,ou=${project.name},cn=buildbot,ou=services,dc=immae,dc=eu + Include /var/secrets/buildbot/${project.name}/webhook-httpd-include + + + '') myconfig.env.buildbot.projects; + + system.activationScripts = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { + deps = [ "users" "wrappers" ]; + text = project.activationScript; + }) myconfig.env.buildbot.projects; + + secrets.keys = ( + lib.lists.flatten ( + lib.attrsets.mapAttrsToList (k: project: + lib.attrsets.mapAttrsToList (k: v: + { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = v; + dest = "buildbot/${project.name}/${k}"; + } + ) project.secrets + ++ [ + { + permissions = "0600"; + user = "wwwrun"; + group = "wwwrun"; + text = lib.optionalString (lib.attrsets.hasAttr "webhookTokens" project) '' + Require expr "req('Access-Key') in { ${builtins.concatStringsSep ", " (map (x: "'${x}'") project.webhookTokens)} }" + ''; + dest = "buildbot/${project.name}/webhook-httpd-include"; + } + ] + ) myconfig.env.buildbot.projects + ) + ) ++ [ + { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = myconfig.env.buildbot.ldap.password; + dest = "buildbot/ldap"; + } + { + permissions = "0600"; + user = "buildbot"; + group = "buildbot"; + text = builtins.readFile "${myconfig.privateFiles}/buildbot_ssh_key"; + dest = "buildbot/ssh_key"; + } + ]; + + systemd.services = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { + description = "Buildbot Continuous Integration Server ${project.name}."; + after = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + path = project.packages pkgs ++ (project.pythonPackages buildbot.pythonModule pkgs); + preStart = let + master-cfg = "${buildbot_common}/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_common/master.cfg"; + tac_file = pkgs.writeText "buildbot.tac" '' + import os + + from twisted.application import service + from buildbot.master import BuildMaster + + basedir = '${varDir}/${project.name}' + rotateLength = 10000000 + maxRotatedFiles = 10 + configfile = '${master-cfg}' + + # Default umask for server + umask = None + + # if this is a relocatable tac file, get the directory containing the TAC + if basedir == '.': + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + + # note: this line is matched against to check that this is a buildmaster + # directory; do not edit it. + application = service.Application('buildmaster') + from twisted.python.logfile import LogFile + from twisted.python.log import ILogObserver, FileLogObserver + logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, + maxRotatedFiles=maxRotatedFiles) + application.setComponent(ILogObserver, FileLogObserver(logfile).emit) + + m = BuildMaster(basedir, configfile, umask) + m.setServiceParent(application) + m.log_rotation.rotateLength = rotateLength + m.log_rotation.maxRotatedFiles = maxRotatedFiles + ''; + in '' + if [ ! -f ${varDir}/${project.name}/buildbot.tac ]; then + ${buildbot}/bin/buildbot create-master -c "${master-cfg}" "${varDir}/${project.name}" + rm -f ${varDir}/${project.name}/master.cfg.sample + rm -f ${varDir}/${project.name}/buildbot.tac + fi + ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac + # different buildbots may be trying that simultaneously, add the || true to avoid complaining in case of race + install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ssh_key ${varDir}/buildbot_key || true + buildbot_secrets=${varDir}/${project.name}/secrets + install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets + install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/ldap $buildbot_secrets/ldap + ${builtins.concatStringsSep "\n" (lib.attrsets.mapAttrsToList + (k: v: "install -Dm600 -o buildbot -g buildbot -T /var/secrets/buildbot/${project.name}/${k} $buildbot_secrets/${k}") project.secrets + )} + ''; + environment = let + project_env = lib.attrsets.mapAttrs' (k: v: lib.attrsets.nameValuePair "BUILDBOT_${k}" v) project.environment; + buildbot_config = pkgs.python3Packages.buildPythonPackage (rec { + name = "buildbot_config-${project.name}"; + src = ./projects + "/${project.name}"; + format = "other"; + installPhase = '' + mkdir -p $out/${pkgs.python3.pythonForBuild.sitePackages} + cp -a $src $out/${pkgs.python3.pythonForBuild.sitePackages}/buildbot_config + ''; + }); + HOME = "${varDir}/${project.name}"; + PYTHONPATH = "${buildbot.pythonModule.withPackages (self: project.pythonPackages self pkgs ++ [ + pkgs.python3Packages.wokkel + pkgs.python3Packages.treq pkgs.python3Packages.ldap3 buildbot + pkgs.python3Packages.buildbot-worker + buildbot_common buildbot_config + ])}/${buildbot.pythonModule.sitePackages}${if project.pythonPathHome then ":${varDir}/${project.name}/.local/${pkgs.python3.pythonForBuild.sitePackages}" else ""}"; + in project_env // { inherit PYTHONPATH HOME; }; + + serviceConfig = { + Type = "forking"; + User = "buildbot"; + Group = "buildbot"; + RuntimeDirectory = "buildbot"; + RuntimeDirectoryPreserve = "yes"; + StateDirectory = "buildbot"; + SupplementaryGroups = "keys"; + WorkingDirectory = "${varDir}/${project.name}"; + ExecStart = "${buildbot}/bin/buildbot start"; + }; + }) myconfig.env.buildbot.projects; + }; +} diff --git a/modules/private/buildbot/projects/caldance/__init__.py b/modules/private/buildbot/projects/caldance/__init__.py new file mode 100644 index 0000000..2c0bad5 --- /dev/null +++ b/modules/private/buildbot/projects/caldance/__init__.py @@ -0,0 +1,190 @@ +from buildbot.plugins import * +from buildbot_common.build_helpers import * +import os +from buildbot.util import bytes2unicode +import json + +__all__ = [ "configure", "E" ] + +class E(): + PROJECT = "caldance" + BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) + SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) + PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) + RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT) + RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT) + GIT_URL = "gitolite@git.immae.eu:perso/simon_descarpentries/www.cal-dance.com" + SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" + SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" + LDAP_HOST = "ldap.immae.eu" + LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" + LDAP_ROLES_BASE = "ou=roles,ou=hosts,dc=immae,dc=eu" + XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") + + PUPPET_HOST = { + "integration": "root@caldance.immae.eu", + } + + # master.cfg + SECRETS_FILE = os.getcwd() + "/secrets" + LDAP_URL = "ldaps://ldap.immae.eu:636" + LDAP_ADMIN_USER = "cn=buildbot,ou=services,dc=immae,dc=eu" + LDAP_BASE = "dc=immae,dc=eu" + LDAP_PATTERN = "(uid=%(username)s)" + LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=caldance,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))" + TITLE_URL = "https://caldance.immae.eu" + TITLE = "Caldance" + +class CustomBase(webhooks.base): + def getChanges(self, request): + try: + content = request.content.read() + args = json.loads(bytes2unicode(content)) + except Exception as e: + raise ValueError("Error loading JSON: " + str(e)) + + args.setdefault("comments", "") + args.setdefault("repository", "") + args.setdefault("author", args.get("who", "unknown")) + + if args["category"] == "deploy_webhook": + args = { + "category": "deploy_webhook", + "comments": "", + "repository": "", + "author": "webhook", + "project": "Caldance", + "properties": { + "environment": args.get("environment", "integration"), + "build": "caldance_{}.tar.gz".format(args.get("build", "master")) + } + } + + return ([args], None) + +def deploy_hook_scheduler(project, timer=1): + return schedulers.AnyBranchScheduler( + change_filter=util.ChangeFilter(category="deploy_webhook", project=project), + name="{}_deploy".format(project), treeStableTimer=timer, builderNames=["{}_deploy".format(project)]) + +def configure(c): + c["buildbotURL"] = E.BUILDBOT_URL + c["www"]["port"] = E.SOCKET + + c["www"]["change_hook_dialects"]["base"] = { + "custom_class": CustomBase + } + c['workers'].append(worker.LocalWorker("generic-worker")) + c['workers'].append(worker.LocalWorker("deploy-worker")) + + c['schedulers'].append(hook_scheduler("Caldance", timer=1)) + c['schedulers'].append(force_scheduler("force_caldance", ["Caldance_build"])) + c['schedulers'].append(deploy_scheduler("deploy_caldance", ["Caldance_deploy"])) + c['schedulers'].append(deploy_hook_scheduler("Caldance", timer=1)) + + c['builders'].append(factory("caldance")) + + c['builders'].append(deploy_factory("caldance")) + + c['services'].append(SlackStatusPush( + name="slack_status_caldance", + builders=["Caldance_build", "Caldance_deploy"], + serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) + c['services'].append(XMPPStatusPush( + name="xmpp_status_caldance", + builders=["Caldance_build", "Caldance_deploy"], + recipients=E.XMPP_RECIPIENTS, + password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip())) + +def factory(project, ignore_fails=False): + release_file = "{1}/{0}_%(kw:clean_branch)s.tar.gz" + + package = util.Interpolate("{0}_%(kw:clean_branch)s.tar.gz".format(project), clean_branch=clean_branch) + package_dest = util.Interpolate(release_file.format(project, E.RELEASE_PATH), clean_branch=clean_branch) + package_url = util.Interpolate(release_file.format(project, E.RELEASE_URL), clean_branch=clean_branch) + + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.GIT_URL, + sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) + factory.addSteps(package_and_upload(package, package_dest, package_url)) + + return util.BuilderConfig( + name="{}_build".format(project.capitalize()), + workernames=["generic-worker"], factory=factory) + +def compute_build_infos(project): + @util.renderer + def compute(props): + import re, hashlib + build_file = props.getProperty("build") + package_dest = "{1}/{0}".format(build_file, E.RELEASE_PATH) + version = re.match(r"{0}_(.*).tar.gz".format(project), build_file).group(1) + with open(package_dest, "rb") as f: + sha = hashlib.sha256(f.read()).hexdigest() + return { + "build_version": version, + "build_hash": sha, + } + return compute + +@util.renderer +def puppet_host(props): + environment = props["environment"] if props.hasProperty("environment") else "integration" + return E.PUPPET_HOST.get(environment, "host.invalid") + +def deploy_factory(project): + package_dest = util.Interpolate("{0}/%(prop:build)s".format(E.RELEASE_PATH)) + + factory = util.BuildFactory() + factory.addStep(steps.MasterShellCommand(command=["test", "-f", package_dest])) + factory.addStep(steps.SetProperties(properties=compute_build_infos(project))) + factory.addStep(LdapPush(environment=util.Property("environment"), + project=project, build_version=util.Property("build_version"), + build_hash=util.Property("build_hash"), ldap_password=util.Secret("ldap"))) + factory.addStep(steps.MasterShellCommand(command=[ + "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host])) + return util.BuilderConfig(name="{}_deploy".format(project.capitalize()), workernames=["deploy-worker"], factory=factory) + +from twisted.internet import defer +from buildbot.process.buildstep import FAILURE +from buildbot.process.buildstep import SUCCESS +from buildbot.process.buildstep import BuildStep + +class LdapPush(BuildStep): + name = "LdapPush" + renderables = ["environment", "project", "build_version", "build_hash", "ldap_password"] + + def __init__(self, **kwargs): + self.environment = kwargs.pop("environment") + self.project = kwargs.pop("project") + self.build_version = kwargs.pop("build_version") + self.build_hash = kwargs.pop("build_hash") + self.ldap_password = kwargs.pop("ldap_password") + self.ldap_host = kwargs.pop("ldap_host", E.LDAP_HOST) + super().__init__(**kwargs) + + def run(self): + import json + from ldap3 import Reader, Writer, Server, Connection, ObjectDef + server = Server(self.ldap_host) + conn = Connection(server, + user=E.LDAP_DN, + password=self.ldap_password) + conn.bind() + obj = ObjectDef("immaePuppetClass", conn) + r = Reader(conn, obj, + "cn=caldance.{},{}".format(self.environment, E.LDAP_ROLES_BASE)) + r.search() + if len(r) > 0: + w = Writer.from_cursor(r) + for value in w[0].immaePuppetJson.values: + config = json.loads(value) + if "role::caldance::{}_version".format(self.project) in config: + config["role::caldance::{}_version".format(self.project)] = self.build_version + config["role::caldance::{}_sha256".format(self.project)] = self.build_hash + w[0].immaePuppetJson -= value + w[0].immaePuppetJson += json.dumps(config, indent=" ") + w.commit() + return defer.succeed(SUCCESS) + return defer.succeed(FAILURE) diff --git a/modules/private/buildbot/projects/cryptoportfolio/__init__.py b/modules/private/buildbot/projects/cryptoportfolio/__init__.py new file mode 100644 index 0000000..5d70f95 --- /dev/null +++ b/modules/private/buildbot/projects/cryptoportfolio/__init__.py @@ -0,0 +1,169 @@ +from buildbot.plugins import * +from buildbot_common.build_helpers import * +import os + +__all__ = [ "configure", "E" ] + +class E(): + PROJECT = "cryptoportfolio" + BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) + SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) + PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) + RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT) + RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT) + GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/Cryptoportfolio/{0}.git" + SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" + LDAP_HOST = "ldap.immae.eu" + LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" + LDAP_ROLES_BASE = "ou=roles,ou=hosts,dc=immae,dc=eu" + + PUPPET_HOST = { + "production": "root@cryptoportfolio.immae.eu", + "integration": "root@cryptoportfolio-dev.immae.eu" + } + + # master.cfg + SECRETS_FILE = os.getcwd() + "/secrets" + LDAP_URL = "ldaps://ldap.immae.eu:636" + LDAP_ADMIN_USER = "cn=buildbot,ou=services,dc=immae,dc=eu" + LDAP_BASE = "dc=immae,dc=eu" + LDAP_PATTERN = "(uid=%(username)s)" + LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=cryptoportfolio,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))" + TITLE_URL = "https://git.immae.eu" + TITLE = "Cryptoportfolio" + +# eval .. dans .zshrc_local +# mkdir -p $BUILD/go +# export GOPATH=$BUILD/go +# go get -u github.com/golang/dep/cmd/dep +# export PATH=$PATH:$BUILD/go/bin +# go get git.immae.eu/Cryptoportfolio/Front.git +# cd $BUILD/go/src/git.immae.eu/Cryptoportfolio/Front.git +# git checkout dev +# dep ensure +def configure(c): + c["buildbotURL"] = E.BUILDBOT_URL + c["www"]["port"] = E.SOCKET + + c['workers'].append(worker.LocalWorker("generic-worker")) + c['workers'].append(worker.LocalWorker("deploy-worker")) + + c['schedulers'].append(hook_scheduler("Trader")) + c['schedulers'].append(hook_scheduler("Front")) + c['schedulers'].append(force_scheduler( + "force_cryptoportfolio", ["Trader_build", "Front_build"])) + c['schedulers'].append(deploy_scheduler("deploy_cryptoportfolio", + ["Trader_deploy", "Front_deploy"])) + + c['builders'].append(factory("trader")) + c['builders'].append(factory("front", ignore_fails=True)) + + c['builders'].append(deploy_factory("trader")) + c['builders'].append(deploy_factory("front")) + + c['services'].append(SlackStatusPush( + name="slack_status_cryptoportfolio", + builders=["Front_build", "Trader_build", "Front_deploy", "Trader_deploy"], + serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) + +def factory(project, ignore_fails=False): + release_file = "{1}/{0}/{0}_%(kw:clean_branch)s.tar.gz" + + url = E.GIT_URL.format(project.capitalize()) + + package = util.Interpolate("{0}_%(kw:clean_branch)s.tar.gz".format(project), clean_branch=clean_branch) + package_dest = util.Interpolate(release_file.format(project, E.RELEASE_PATH), clean_branch=clean_branch) + package_url = util.Interpolate(release_file.format(project, E.RELEASE_URL), clean_branch=clean_branch) + + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=url, + mode="full", method="copy")) + factory.addStep(steps.ShellCommand(name="make install", + logEnviron=False, haltOnFailure=(not ignore_fails), + warnOnFailure=ignore_fails, flunkOnFailure=(not ignore_fails), + command=["make", "install"])) + factory.addStep(steps.ShellCommand(name="make test", + logEnviron=False, haltOnFailure=(not ignore_fails), + warnOnFailure=ignore_fails, flunkOnFailure=(not ignore_fails), + command=["make", "test"])) + factory.addSteps(package_and_upload(package, package_dest, package_url)) + + return util.BuilderConfig( + name="{}_build".format(project.capitalize()), + workernames=["generic-worker"], factory=factory) + +def compute_build_infos(project): + @util.renderer + def compute(props): + import re, hashlib + build_file = props.getProperty("build") + package_dest = "{2}/{0}/{1}".format(project, build_file, E.RELEASE_PATH) + version = re.match(r"{0}_(.*).tar.gz".format(project), build_file).group(1) + with open(package_dest, "rb") as f: + sha = hashlib.sha256(f.read()).hexdigest() + return { + "build_version": version, + "build_hash": sha, + } + return compute + +@util.renderer +def puppet_host(props): + environment = props["environment"] if props.hasProperty("environment") else "integration" + return E.PUPPET_HOST.get(environment, "host.invalid") + +def deploy_factory(project): + package_dest = util.Interpolate("{1}/{0}/%(prop:build)s".format(project, E.RELEASE_PATH)) + + factory = util.BuildFactory() + factory.addStep(steps.MasterShellCommand(command=["test", "-f", package_dest])) + factory.addStep(steps.SetProperties(properties=compute_build_infos(project))) + factory.addStep(LdapPush(environment=util.Property("environment"), + project=project, build_version=util.Property("build_version"), + build_hash=util.Property("build_hash"), ldap_password=util.Secret("ldap"))) + factory.addStep(steps.MasterShellCommand(command=[ + "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host])) + return util.BuilderConfig(name="{}_deploy".format(project.capitalize()), workernames=["deploy-worker"], factory=factory) + +from twisted.internet import defer +from buildbot.process.buildstep import FAILURE +from buildbot.process.buildstep import SUCCESS +from buildbot.process.buildstep import BuildStep + +class LdapPush(BuildStep): + name = "LdapPush" + renderables = ["environment", "project", "build_version", "build_hash", "ldap_password"] + + def __init__(self, **kwargs): + self.environment = kwargs.pop("environment") + self.project = kwargs.pop("project") + self.build_version = kwargs.pop("build_version") + self.build_hash = kwargs.pop("build_hash") + self.ldap_password = kwargs.pop("ldap_password") + self.ldap_host = kwargs.pop("ldap_host", E.LDAP_HOST) + super().__init__(**kwargs) + + def run(self): + import json + from ldap3 import Reader, Writer, Server, Connection, ObjectDef + server = Server(self.ldap_host) + conn = Connection(server, + user=E.LDAP_DN, + password=self.ldap_password) + conn.bind() + obj = ObjectDef("immaePuppetClass", conn) + r = Reader(conn, obj, + "cn=cryptoportfolio.{},{}".format(self.environment, E.LDAP_ROLES_BASE)) + r.search() + if len(r) > 0: + w = Writer.from_cursor(r) + for value in w[0].immaePuppetJson.values: + config = json.loads(value) + if "role::cryptoportfolio::{}_version".format(self.project) in config: + config["role::cryptoportfolio::{}_version".format(self.project)] = self.build_version + config["role::cryptoportfolio::{}_sha256".format(self.project)] = self.build_hash + w[0].immaePuppetJson -= value + w[0].immaePuppetJson += json.dumps(config, indent=" ") + w.commit() + return defer.succeed(SUCCESS) + return defer.succeed(FAILURE) diff --git a/modules/private/buildbot/projects/test/__init__.py b/modules/private/buildbot/projects/test/__init__.py new file mode 100644 index 0000000..e6b8d51 --- /dev/null +++ b/modules/private/buildbot/projects/test/__init__.py @@ -0,0 +1,188 @@ +from buildbot.plugins import * +from buildbot_common.build_helpers import * +import os +from buildbot.util import bytes2unicode +import json + +__all__ = [ "configure", "E" ] + +class E(): + PROJECT = "test" + BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) + SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) + PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) + RELEASE_PATH = "/var/lib/ftp/release.immae.eu/{}".format(PROJECT) + RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT) + GIT_URL = "https://git.immae.eu/perso/Immae/TestProject.git" + SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" + PUPPET_HOST = "root@backup-1.v.immae.eu" + LDAP_HOST = "ldap.immae.eu" + LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" + LDAP_ROLES_BASE = "ou=roles,ou=hosts,dc=immae,dc=eu" + XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") + + # master.cfg + SECRETS_FILE = os.getcwd() + "/secrets" + LDAP_URL = "ldaps://ldap.immae.eu:636" + LDAP_ADMIN_USER = "cn=buildbot,ou=services,dc=immae,dc=eu" + LDAP_BASE = "dc=immae,dc=eu" + LDAP_PATTERN = "(uid=%(username)s)" + LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=test,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))" + TITLE_URL = "https://git.immae.eu/?p=perso/Immae/TestProject.git;a=summary" + TITLE = "Test project" + +class CustomBase(webhooks.base): + def getChanges(self, request): + try: + content = request.content.read() + args = json.loads(bytes2unicode(content)) + except Exception as e: + raise ValueError("Error loading JSON: " + str(e)) + + args.setdefault("comments", "") + args.setdefault("repository", "") + args.setdefault("author", args.get("who", "unknown")) + + if args["category"] == "deploy_webhook": + args = { + "category": "deploy_webhook", + "comments": "", + "repository": "", + "author": "unknown", + "project": "TestProject", + "properties": { + "environment": args.get("environment", "integration"), + "build": "test_{}.tar.gz".format(args.get("branch", "master")) + } + } + + return ([args], None) + +def deploy_hook_scheduler(project, timer=1): + return schedulers.AnyBranchScheduler( + change_filter=util.ChangeFilter(category="deploy_webhook", project=project), + name="{}_deploy".format(project), treeStableTimer=timer, builderNames=["{}_deploy".format(project)]) + +def configure(c): + c["buildbotURL"] = E.BUILDBOT_URL + c["www"]["port"] = E.SOCKET + + c["www"]["change_hook_dialects"]["base"] = { + "custom_class": CustomBase + } + c['workers'].append(worker.LocalWorker("generic-worker-test")) + c['workers'].append(worker.LocalWorker("deploy-worker-test")) + + c['schedulers'].append(hook_scheduler("TestProject", timer=1)) + c['schedulers'].append(force_scheduler("force_test", ["TestProject_build"])) + c['schedulers'].append(deploy_scheduler("deploy_test", ["TestProject_deploy"])) + c['schedulers'].append(deploy_hook_scheduler("TestProject", timer=1)) + + c['builders'].append(factory()) + c['builders'].append(deploy_factory()) + + c['services'].append(SlackStatusPush( + name="slack_status_test_project", + builders=["TestProject_build", "TestProject_deploy"], + serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) + c['services'].append(XMPPStatusPush( + name="xmpp_status_test_project", + builders=["TestProject_build", "TestProject_deploy"], + recipients=E.XMPP_RECIPIENTS, + password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip())) + +def factory(): + package = util.Interpolate("test_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) + package_dest = util.Interpolate("{}/test_%(kw:clean_branch)s.tar.gz".format(E.RELEASE_PATH), clean_branch=clean_branch) + package_url = util.Interpolate("{}/test_%(kw:clean_branch)s.tar.gz".format(E.RELEASE_URL), clean_branch=clean_branch) + + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, + repourl=E.GIT_URL, mode="full", method="copy")) + factory.addStep(steps.ShellCommand(name="env", + logEnviron=False, command=["env"])) + factory.addStep(steps.ShellCommand(name="pwd", + logEnviron=False, command=["pwd"])) + factory.addStep(steps.ShellCommand(name="true", + logEnviron=False, command=["true"])) + factory.addStep(steps.ShellCommand(name="echo", + logEnviron=False, command=["echo", package])) + factory.addSteps(package_and_upload(package, package_dest, package_url)) + + return util.BuilderConfig(name="TestProject_build", workernames=["generic-worker-test"], factory=factory) + + +def compute_build_infos(): + @util.renderer + def compute(props): + import re, hashlib + build_file = props.getProperty("build") + package_dest = "{}/{}".format(E.RELEASE_PATH, build_file) + version = re.match(r"{0}_(.*).tar.gz".format("test"), build_file).group(1) + with open(package_dest, "rb") as f: + sha = hashlib.sha256(f.read()).hexdigest() + return { + "build_version": version, + "build_hash": sha, + } + return compute + +@util.renderer +def puppet_host(props): + return E.PUPPET_HOST + +def deploy_factory(): + package_dest = util.Interpolate("{}/%(prop:build)s".format(E.RELEASE_PATH)) + + factory = util.BuildFactory() + factory.addStep(steps.MasterShellCommand(command=["test", "-f", package_dest])) + factory.addStep(steps.SetProperties(properties=compute_build_infos())) + factory.addStep(LdapPush(environment=util.Property("environment"), + build_version=util.Property("build_version"), + build_hash=util.Property("build_hash"), + ldap_password=util.Secret("ldap"))) + factory.addStep(steps.MasterShellCommand(command=[ + "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-i", E.SSH_KEY_PATH, puppet_host])) + return util.BuilderConfig(name="TestProject_deploy", workernames=["deploy-worker-test"], factory=factory) + +from twisted.internet import defer +from buildbot.process.buildstep import FAILURE +from buildbot.process.buildstep import SUCCESS +from buildbot.process.buildstep import BuildStep + +class LdapPush(BuildStep): + name = "LdapPush" + renderables = ["environment", "build_version", "build_hash", "ldap_password"] + + def __init__(self, **kwargs): + self.environment = kwargs.pop("environment") + self.build_version = kwargs.pop("build_version") + self.build_hash = kwargs.pop("build_hash") + self.ldap_password = kwargs.pop("ldap_password") + self.ldap_host = kwargs.pop("ldap_host", E.LDAP_HOST) + super().__init__(**kwargs) + + def run(self): + import json + from ldap3 import Reader, Writer, Server, Connection, ObjectDef + server = Server(self.ldap_host) + conn = Connection(server, + user=E.LDAP_DN, + password=self.ldap_password) + conn.bind() + obj = ObjectDef("immaePuppetClass", conn) + r = Reader(conn, obj, + "cn=test.{},{}".format(self.environment, E.LDAP_ROLES_BASE)) + r.search() + if len(r) > 0: + w = Writer.from_cursor(r) + for value in w[0].immaePuppetJson.values: + config = json.loads(value) + if "test_version" in config: + config["test_version"] = self.build_version + config["test_sha256"] = self.build_hash + w[0].immaePuppetJson -= value + w[0].immaePuppetJson += json.dumps(config, indent=" ") + w.commit() + return defer.succeed(SUCCESS) + return defer.succeed(FAILURE) -- cgit v1.2.3