X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=modules%2Fprivate%2Fbuildbot%2Fprojects%2FimmaeEu%2F__init__.py;h=e817ad0ac0a93a1ddd8abee7d748ec5ffdb50e44;hb=bd0cb07b13aecd16a0782492655843a1b699611d;hp=ce6cdb1e0b5573d900ccdd9f3d50b41116659ada;hpb=830b619fa6ed6b5f2acc6f2eb74e0be98c99710e;p=perso%2FImmae%2FConfig%2FNix.git diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py index ce6cdb1..e817ad0 100644 --- a/modules/private/buildbot/projects/immaeEu/__init__.py +++ b/modules/private/buildbot/projects/immaeEu/__init__.py @@ -3,6 +3,7 @@ from buildbot_common.build_helpers import * import os from buildbot.util import bytes2unicode import json +from functools import partial __all__ = [ "configure", "E" ] @@ -12,20 +13,27 @@ class E(): SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" + SSH_HOST_PATH = "/var/lib/buildbot/buildbot_hosts" SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") + BIP39_GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git" IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" + HISTORY_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/History" RECETTES_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Recettes" COURS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Cours" + DOCS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Docs" NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup" - COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/cours" + COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/cours" COURS_TARBALL_PATH = "/var/lib/ftp/release.immae.eu/cours" COURS_TARBALL_URL = "https://release.immae.eu/cours" - IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immaeEu" - RECETTES_RELEASE_PATH = "/var/lib/buildbot/outputs/recettes" - NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/recherche" + BIP39_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/bip39" + HISTORY_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/history" + IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/blog" + DOCS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/docs" + RECETTES_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recettes" + NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recherche" GSMCELLS_RELEASE_PATH = "/var/lib/ftp/release.immae.eu/gsm_cells" GSMCELLS_RELEASE_URL = "https://release.immae.eu/gsm_cells" @@ -39,181 +47,337 @@ class E(): TITLE_URL = "https://www.immae.eu" TITLE = "Immae website" -class CustomBase(webhooks.base): - def getChanges(self, request): - try: - content = request.content.read() - args = json.loads(bytes2unicode(content)) - except Exception as e: - raise ValueError("Error loading JSON: " + str(e)) - - args.setdefault("comments", "") - args.setdefault("repository", "") - args.setdefault("author", args.get("who", "unknown")) - - return ([args], None) - def configure(c): c["buildbotURL"] = E.BUILDBOT_URL c["www"]["port"] = E.SOCKET - c["www"]["change_hook_dialects"]["base"] = { - "custom_class": CustomBase + worker_name = "generic-worker-immae-eu" + c['workers'].append(worker.LocalWorker(worker_name)) + + withbranch = [] + withoutbranch = [] + + withoutbranch.append(configure_gsm_cells(c, worker_name)) + + withbranch.append(_configure("immae_eu", immae_eu_build_factory, c, worker_name)) + withbranch.append(_configure("normalesup", normalesup_build_factory, c, worker_name)) + withbranch.append(_configure("cours", cours_build_factory, c, worker_name)) + withbranch.append(_configure("recettes", recettes_build_factory, c, worker_name)) + withbranch.append(_configure("docs", docs_build_factory, c, worker_name)) + withbranch.append(_configure("history", history_build_factory, c, worker_name)) + withbranch.append(_configure("bip39", bip39_build_factory, c, worker_name)) + + withbranch.append(_configure_symfony("Chloe", c, worker_name, + "gitolite@git.immae.eu:perso/Immae/Sites/Chloe/New")) + withbranch.append(_configure_symfony("Florian", c, worker_name, + "gitolite@git.immae.eu:perso/florian_telles/stabilo")) + withbranch.append(configure_symfony_isabelle_aten(c, worker_name)) + withbranch.append(_configure_symfony("Ludivine", c, worker_name, + "gitolite@git.immae.eu:perso/Immae/Sites/Ludivine")) + withbranch.append(_configure_symfony("Connexionswing", c, worker_name, + "gitolite@git.immae.eu:perso/Immae/Projets/Connexionswing")) + withbranch.append(_configure_symfony("Piedsjaloux", c, worker_name, + "gitolite@git.immae.eu:Pieds_jaloux/NewSite")) + + c['schedulers'].append(force_scheduler("force_immae_eu", withbranch)) + c['schedulers'].append(force_scheduler("force_no_branch_immae_eu", withoutbranch, nobranch=True)) + + configure_slack_push(c, E.SECRETS_FILE, all_builder_names(c)) + configure_xmpp_push(c, E.SECRETS_FILE, all_builder_names(c), E.XMPP_RECIPIENTS) + +BRANCH_TO_SYMFONY_ENV = { + "Ludivine": { + "master": "prod", + "test": "dev", + }, + "Connexionswing": { + "master": "prod", + "test": "dev", + }, + "Piedsjaloux": { + "master": "prod", + "test": "dev", } - c['workers'].append(worker.LocalWorker("generic-worker-immae-eu")) + } + +BRANCH_TO_SERVICE_NAME = { + "Chloe": { + "test": "phpfpm-chloe_new_integration", + #"master": "phpfpm-chloe_production", + }, + "Florian": { + "stabilo_dev": "phpfpm-florian_app", + }, + "IsabelleAten": { + "test": "phpfpm-isabelle_aten_integration", + "master": "phpfpm-isabelle_aten_production", + }, + "Ludivine": { + "test": "phpfpm-ludivine_integration", + "master": "phpfpm-ludivine_production", + }, + "Connexionswing": { + "test": "phpfpm-connexionswing_integration", + "master": "phpfpm-connexionswing_production", + }, + "Piedsjaloux": { + "test": "phpfpm-piedsjaloux_integration", + "master": "phpfpm-piedsjaloux_production", + }, + } + +BRANCH_TO_POST_STEP = { + "Connexionswing": { + "master": "SYMFONY_ENV=prod php ./bin/console assetic:dump --env=prod --no-debug" + }, + } + +def need_follow_systemd(name, step): + return step.getProperty("branch") in BRANCH_TO_SERVICE_NAME.get(name, {}) + +def need_post_step(name, step): + return step.getProperty("branch") in BRANCH_TO_POST_STEP.get(name, {}) + +@util.renderer +def get_post_step_command(props, name): + if props.hasProperty("branch") and len(props["branch"]) > 0: + post_step = BRANCH_TO_POST_STEP.get(name, {}).get(props["branch"]) + if post_step is not None: + return post_step + +@util.renderer +def get_parameters_file(props, name): + if props.hasProperty("branch") and len(props["branch"]) > 0: + env = BRANCH_TO_SYMFONY_ENV.get(name, {}).get(props["branch"], "dev") + else: + env = "dev" + return E.SECRETS_FILE + "/symfony_{}_{}_parameters.yml".format(name, env) - c['schedulers'].append(hook_scheduler("ImmaeEu", timer=1)) - c['schedulers'].append(hook_scheduler("Normalesup", timer=1)) - c['schedulers'].append(hook_scheduler("Cours", timer=1)) - c['schedulers'].append(hook_scheduler("Recettes", timer=1)) +@util.renderer +def get_composer_install_command(props, name): + if props.hasProperty("branch") and len(props["branch"]) > 0: + env = BRANCH_TO_SYMFONY_ENV.get(name, {}).get(props["branch"], "dev") + else: + env = "dev" + + return "SYMFONY_ENV={} composer install".format(env) + (" --no-dev" if env == "prod" else "") + +@util.renderer +def get_systemd_service_invocation_command(props, name): + if props.hasProperty("branch") and len(props["branch"]) > 0: + service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"]) + if service is not None: + return "ssh -o UserKnownHostsFile={0} -i {1} buildbot@eldiron systemctl show -p InvocationID --value {2}.service".format(E.SSH_HOST_PATH, E.SSH_KEY_PATH, service) + +@util.renderer +def follow_systemd_command(props, name, invocation_id): + if props.hasProperty("branch") and len(props["branch"]) > 0: + service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"]) + if service is not None: + return ["follow-systemd-unit", service, util.Property("service_invocation_id")] + +def configure_gsm_cells(c, worker_name): + builder_name = "GSMCells_build" c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly", - builderNames=["GSMCells_build"], dayOfWeek=6, hour=3)) - c['schedulers'].append(force_scheduler("force_immae_eu", [ - "ImmaeEu_build", "Normalesup_build", "Cours_build", "Recettes_build" - ])) - c['schedulers'].append(schedulers.ForceScheduler( - name="GSMCells-force", label="Force build", - buttonName="Force build", - reason=util.StringParameter(name="reason", label="Reason", default="Force build"), - codebases=[ - util.CodebaseParameter("", - branch=util.FixedParameter(name="branch", default=""), - revision=util.FixedParameter(name="revision", default=""), - repository=util.FixedParameter(name="repository", default=""), - project=util.FixedParameter(name="project", default=""), - ), - ], - username=util.FixedParameter(name="username", default="Web button"), - builderNames=["GSMCells_build"] - )) - - c['builders'].append(immae_eu_factory()) - c['builders'].append(normalesup_factory()) - c['builders'].append(cours_factory()) - c['builders'].append(gsm_cells_factory()) - c['builders'].append(recettes_factory()) - - c['services'].append(SlackStatusPush( - name="slack_status_immae_eu_project", - builders=[ - "ImmaeEu_build", "Normalesup_build", "Cours_build", - "GSMCells_build", "Recettes_build" - ], - serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) - c['services'].append(XMPPStatusPush( - name="xmpp_status_immae_eu_project", - builders=[ - "ImmaeEu_build", "Normalesup_build", "Cours_build", - "GSMCells_build", "Recettes_build" - ], - recipients=E.XMPP_RECIPIENTS, - password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip())) - -def recettes_factory(): + builderNames=[builder_name], dayOfWeek=6, hour=3)) + c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=gsm_cells_build_factory())) + + return builder_name + +def _configure(name, factory, c, worker_name): + if name == "bip39": + capitalized = "BIP39" + else: + capitalized = "".join([n.capitalize() for n in name.split('_')]) + builder_name = "{}_build".format(capitalized) + + c['schedulers'].append(git_hook_scheduler(capitalized, [builder_name])) + c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=factory())) + + return builder_name + +def configure_symfony_isabelle_aten(c, worker_name): + return _configure_symfony("IsabelleAten", c, worker_name, + "gitolite@git.immae.eu:perso/Immae/Sites/Aten", parameters_path=None, + other_steps=lambda path_env : [ + NixShellCommand(name="Install yarn", + logEnviron=False, haltOnFailure=True, + env=path_env, command="yarn install"), + NixShellCommand(name="Build frontend", + logEnviron=False, haltOnFailure=True, + env=path_env, command="yarn run encore production") + ] + ) + +def _configure_symfony(name, c, worker_name, *args, **kwargs): + builder_name = "Symfony_{}_build".format(name) + + c['schedulers'].append(git_hook_scheduler("Symfony_{}".format(name), [builder_name])) + c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=symfony_project_factory(name, *args, **kwargs))) + + return builder_name + +def history_build_factory(): + path_env = { + "PATH": "/run/current-system/sw/bin" + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, + env=path_env, command="jekyll build")) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH))) + factory.addStep(steps.DirectoryUpload(workersrc="_site", + masterdest=E.HISTORY_RELEASE_PATH, + url="https://www.immae.eu/history")) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.HISTORY_RELEASE_PATH))) + + return factory + +def docs_build_factory(): + path_env = { + "PATH": "/run/current-system/sw/bin" + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, + env=path_env, command="make html")) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH))) + factory.addStep(steps.DirectoryUpload(workersrc="_build/html", + masterdest=E.DOCS_RELEASE_PATH, + url="https://www.immae.eu/docs")) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.DOCS_RELEASE_PATH))) + + return factory + +def recettes_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - env=path_env, command=["jekyll", "build", "--baseurl", "/recettes"])) + sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, + env=path_env, command="jekyll build --trace --baseurl /recettes")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RECETTES_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/_site", + factory.addStep(steps.DirectoryUpload(workersrc="_site", masterdest=E.RECETTES_RELEASE_PATH, url="https://www.immae.eu/recettes")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RECETTES_RELEASE_PATH))) - return util.BuilderConfig(name="Recettes_build", workernames=["generic-worker-immae-eu"], factory=factory) + return factory -def immae_eu_factory(): +def bip39_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin", + "NIX_PATH": "nixpkgs=channel:nixos-unstable", + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL, + submodules=True, mode="full", method="fresh")) + factory.addStep(NixShellCommand(name="build file", nixPackages=["python3"], + logEnviron=False, haltOnFailure=True, env=path_env, command="python compile.py")) + factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html", + masterdest=E.BIP39_RELEASE_PATH + "/index.html", + url="https://tools.immae.eu/BIP39", mode=0o644)) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.BIP39_RELEASE_PATH))) + + return factory + +def immae_eu_build_factory(): + path_env = { + "PATH": "/run/current-system/sw/bin", } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - env=path_env, command=["make", "html"])) + sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=False, + env=path_env, command="make html")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/output", + factory.addStep(steps.DirectoryUpload(workersrc="output", masterdest=E.IMMAE_EU_RELEASE_PATH, url="https://www.immae.eu")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH))) - return util.BuilderConfig(name="ImmaeEu_build", workernames=["generic-worker-immae-eu"], factory=factory) + return factory -def cours_factory(): +def cours_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}", + "PATH": "/run/current-system/sw/bin", "CI": "yes" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "build"], env=path_env)) + sshHostKey=E.SSH_HOST_KEY, mode="incremental")) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=True, + command="make build", env=path_env)) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/build", + factory.addStep(steps.DirectoryUpload(workersrc="build", masterdest=E.COURS_RELEASE_PATH, url="https://www.immae.eu/cours")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) - factory.addStep(steps.ShellCommand(name="build pdfs", - logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "pdfs"], env=path_env)) + factory.addStep(NixShellCommand(name="build pdfs", + logEnviron=False, haltOnFailure=True, pure=True, + command="make pdfs", env=path_env)) package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch) package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch) factory.addStep(steps.ShellCommand(name="build pdf tarball", - logEnviron=False, haltOnFailure=True, workdir="source", + logEnviron=False, haltOnFailure=True, command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env)) factory.addStep(steps.FileUpload(name="upload package", workersrc=package, - workdir="source", masterdest=package_dest, - url=package_url, mode=0o644)) + masterdest=package_dest, url=package_url, mode=0o644)) - return util.BuilderConfig(name="Cours_build", workernames=["generic-worker-immae-eu"], factory=factory) + return factory -def normalesup_factory(): +def normalesup_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin", + "GIT_SSH_COMMAND": "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH), } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "build"], env=path_env)) + sshHostKey=E.SSH_HOST_KEY, mode="incremental")) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=False, + command="make build", env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", - logEnviron=False, haltOnFailure=True, workdir="source", + logEnviron=False, haltOnFailure=True, command="chmod -R a+rX build", env=path_env)) factory.addStep(steps.ShellCommand(name="synchronize with phare", - logEnviron=False, haltOnFailure=True, workdir="source", + logEnviron=False, haltOnFailure=True, env=path_env, command=[ "rsync", "-av", "--delete", - "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH), + "-e", "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH), "build/", os.environ["BUILDBOT_NORMALESUP_HOST"] ])) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/build", masterdest=E.NORMALESUP_RELEASE_PATH, + factory.addStep(steps.DirectoryUpload(workersrc="build", masterdest=E.NORMALESUP_RELEASE_PATH, url="https://www.immae.eu/recherche")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH))) - return util.BuilderConfig(name="Normalesup_build", workernames=["generic-worker-immae-eu"], factory=factory) + return factory -def gsm_cells_factory(): +def gsm_cells_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}", + "PATH": "/run/current-system/sw/bin", + "NIX_PATH": "nixpkgs=channel:nixos-unstable", "IN_BUILDBOT": "yes", } master_env = { @@ -228,8 +392,10 @@ def gsm_cells_factory(): script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" factory = util.BuildFactory() - factory.addStep(steps.ShellCommand(name="download files", - logEnviron=False, haltOnFailure=True, command=[script], env=path_env)) + factory.addStep(steps.FileDownload(mastersrc=script, workerdest="lacells_download", mode=0o755)) + factory.addStep(NixShellCommand(name="download files", + logEnviron=False, haltOnFailure=True, command="./lacells_download", + nixPackages=["sqlite", "wget", "gzip"], pure=False, env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", logEnviron=False, haltOnFailure=True, command="chmod a+r lacells.db", env=path_env)) @@ -241,4 +407,53 @@ def gsm_cells_factory(): factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH))) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH))) - return util.BuilderConfig(name="GSMCells_build", workernames=["generic-worker-immae-eu"], factory=factory) + return factory + +def symfony_project_factory(name, repourl, parameters_path="app/config/parameters.yml", other_steps=lambda a : []): + master_path_env = { + "PATH": os.environ["BUILDBOT_SYMFONY_MASTER_PATH"] + ":${PATH}" + } + path_env = { + "PATH": "/run/current-system/sw/bin" + } + + for k, v in os.environ.items(): + if k.startswith("BUILDBOT_SYMFONY_{}_".format(name)): + path_env[k[len("BUILDBOT_SYMFONY_{}_".format(name)):]] = v + + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=repourl, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) + if parameters_path is not None: + factory.addStep(steps.FileDownload(mastersrc=get_parameters_file.withArgs(name), + workerdest=parameters_path)) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, + env=path_env, command=get_composer_install_command.withArgs(name))) + if name in BRANCH_TO_POST_STEP: + factory.addStep(NixShellCommand(name="build website post", + logEnviron=False, haltOnFailure=True, doStepIf=partial(need_post_step, name), + env=path_env, command=get_post_step_command.withArgs(name))) + if parameters_path is not None: + factory.addStep(steps.ShellCommand(name="Remove parameters.yml", + logEnviron=False, haltOnFailure=True, + command="rm -f {}".format(parameters_path))) + for step in other_steps(path_env): + factory.addStep(step) + package = util.Interpolate("{}_%(kw:clean_branch)s.tar.gz".format(name), clean_branch=clean_branch) + release_file = "{0}/{1}/%(kw:clean_branch)s.tar.gz" + package_dest = util.Interpolate(release_file.format("/var/lib/ftp/release.immae.eu/buildbot", name), clean_branch=clean_branch) + # Tar doesn’t like creating the tarball in the same directory + factory.addStep(steps.ShellCommand(name="Make a tarball 1/2", + logEnviron=False, haltOnFailure=True, env=path_env, + command=["touch", package])) + factory.addStep(steps.ShellCommand(name="Make a tarball 2/2", + logEnviron=False, haltOnFailure=True, env=path_env, + command=["tar", "--exclude", package, "-czf", package, "."])) + factory.addStep(steps.SetPropertyFromCommand(command=get_systemd_service_invocation_command.withArgs(name), + property="service_invocation_id", doStepIf=partial(need_follow_systemd, name))) + factory.addStep(steps.FileUpload(name="upload package", + workersrc=package, masterdest=package_dest, mode=0o644)) + factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=master_path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name))) + return factory