from buildbot.plugins import * from buildbot_common.build_helpers import * import os from buildbot.util import bytes2unicode import json from functools import partial __all__ = [ "configure", "E" ] class E(): PROJECT = "immaeEu" BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" SSH_HOST_PATH = "/var/lib/buildbot/buildbot_hosts" SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") BIP39_GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git" IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" HISTORY_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/History" RECETTES_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Recettes" JARDIN_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Jardin" COURS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Cours" DOCS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Docs" NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup" COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/cours" COURS_TARBALL_PATH = "/var/lib/ftp/release.immae.eu/cours" COURS_TARBALL_URL = "https://release.immae.eu/cours" BIP39_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/bip39" HISTORY_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/history" IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/blog" DOCS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/docs" RECETTES_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recettes" JARDIN_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/jardin" NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recherche" GSMCELLS_RELEASE_PATH = "/var/lib/ftp/release.immae.eu/gsm_cells" GSMCELLS_RELEASE_URL = "https://release.immae.eu/gsm_cells" # master.cfg SECRETS_FILE = os.getcwd() + "/secrets" LDAP_URL = "ldaps://ldap.immae.eu:636" LDAP_ADMIN_USER = "cn=buildbot,ou=services,dc=immae,dc=eu" LDAP_BASE = "dc=immae,dc=eu" LDAP_PATTERN = "(uid=%(username)s)" LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=immaeEu,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))" TITLE_URL = "https://www.immae.eu" TITLE = "Immae website" def configure(c): c["buildbotURL"] = E.BUILDBOT_URL c["www"]["port"] = E.SOCKET worker_name = "generic-worker-immae-eu" c['workers'].append(worker.LocalWorker(worker_name)) withbranch = [] withoutbranch = [] withoutbranch.append(configure_gsm_cells(c, worker_name)) withbranch.append(_configure("immae_eu", immae_eu_build_factory, c, worker_name)) withbranch.append(_configure("normalesup", normalesup_build_factory, c, worker_name)) withbranch.append(_configure("cours", cours_build_factory, c, worker_name)) withbranch.append(_configure("recettes", recettes_build_factory, c, worker_name)) withbranch.append(_configure("jardin", jardin_build_factory, c, worker_name)) withbranch.append(_configure("docs", docs_build_factory, c, worker_name)) withbranch.append(_configure("history", history_build_factory, c, worker_name)) withbranch.append(_configure("bip39", bip39_build_factory, c, worker_name)) withbranch.append(_configure_symfony("Chloe", c, worker_name, "gitolite@git.immae.eu:perso/Immae/Sites/Chloe/New")) withbranch.append(_configure_symfony("Florian", c, worker_name, "gitolite@git.immae.eu:perso/florian_telles/stabilo")) withbranch.append(configure_symfony_isabelle_aten(c, worker_name)) withbranch.append(_configure_symfony("Ludivine", c, worker_name, "gitolite@git.immae.eu:perso/Immae/Sites/Ludivine")) withbranch.append(_configure_symfony("Connexionswing", c, worker_name, "gitolite@git.immae.eu:perso/Immae/Projets/Connexionswing")) withbranch.append(_configure_symfony("Piedsjaloux", c, worker_name, "gitolite@git.immae.eu:Pieds_jaloux/NewSite")) c['schedulers'].append(force_scheduler("force_immae_eu", withbranch)) c['schedulers'].append(force_scheduler("force_no_branch_immae_eu", withoutbranch, nobranch=True)) configure_apprise_push(c, E.SECRETS_FILE, all_builder_names(c)) configure_xmpp_push(c, E.SECRETS_FILE, all_builder_names(c), E.XMPP_RECIPIENTS) BRANCH_TO_SYMFONY_ENV = { "Ludivine": { "master": "prod", "test": "dev", }, "Connexionswing": { "master": "prod", "test": "dev", }, "Piedsjaloux": { "master": "prod", "test": "dev", } } BRANCH_TO_SERVICE_NAME = { "Chloe": { "test": "phpfpm-chloe_new_integration", #"master": "phpfpm-chloe_production", }, "Florian": { "stabilo_dev": "phpfpm-florian_app", }, "IsabelleAten": { "test": "phpfpm-isabelle_aten_integration", "master": "phpfpm-isabelle_aten_production", }, "Ludivine": { "test": "phpfpm-ludivine_integration", "master": "phpfpm-ludivine_production", }, "Connexionswing": { "test": "phpfpm-connexionswing_integration", "master": "phpfpm-connexionswing_production", }, "Piedsjaloux": { "test": "phpfpm-piedsjaloux_integration", "master": "phpfpm-piedsjaloux_production", }, } BRANCH_TO_POST_STEP = { "Connexionswing": { "master": "SYMFONY_ENV=prod php ./bin/console assetic:dump --env=prod --no-debug" }, } def need_follow_systemd(name, step): return step.getProperty("branch") in BRANCH_TO_SERVICE_NAME.get(name, {}) def need_post_step(name, step): return step.getProperty("branch") in BRANCH_TO_POST_STEP.get(name, {}) @util.renderer def get_post_step_command(props, name): if props.hasProperty("branch") and len(props["branch"]) > 0: post_step = BRANCH_TO_POST_STEP.get(name, {}).get(props["branch"]) if post_step is not None: return post_step @util.renderer def get_parameters_file(props, name): if props.hasProperty("branch") and len(props["branch"]) > 0: env = BRANCH_TO_SYMFONY_ENV.get(name, {}).get(props["branch"], "dev") else: env = "dev" return E.SECRETS_FILE + "/symfony_{}_{}_parameters.yml".format(name, env) @util.renderer def get_composer_install_command(props, name): if props.hasProperty("branch") and len(props["branch"]) > 0: env = BRANCH_TO_SYMFONY_ENV.get(name, {}).get(props["branch"], "dev") else: env = "dev" return "SYMFONY_ENV={} composer install".format(env) + (" --no-dev" if env == "prod" else "") @util.renderer def get_systemd_service_invocation_command(props, name): if props.hasProperty("branch") and len(props["branch"]) > 0: service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"]) if service is not None: return "ssh -o UserKnownHostsFile={0} -i {1} buildbot@eldiron systemctl show -p InvocationID --value {2}.service".format(E.SSH_HOST_PATH, E.SSH_KEY_PATH, service) @util.renderer def follow_systemd_command(props, name, invocation_id): if props.hasProperty("branch") and len(props["branch"]) > 0: service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"]) if service is not None: return ["follow-systemd-unit", service, util.Property("service_invocation_id")] def configure_gsm_cells(c, worker_name): builder_name = "GSMCells_build" c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly", builderNames=[builder_name], dayOfWeek=6, hour=3)) c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=gsm_cells_build_factory())) return builder_name def _configure(name, factory, c, worker_name): if name == "bip39": capitalized = "BIP39" else: capitalized = "".join([n.capitalize() for n in name.split('_')]) builder_name = "{}_build".format(capitalized) c['schedulers'].append(git_hook_scheduler(capitalized, [builder_name])) c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=factory())) return builder_name def configure_symfony_isabelle_aten(c, worker_name): return _configure_symfony("IsabelleAten", c, worker_name, "gitolite@git.immae.eu:perso/Immae/Sites/Aten", parameters_path=None, other_steps=lambda path_env : [ NixShellCommand(name="Install yarn", logEnviron=False, haltOnFailure=True, env=path_env, command="yarn install"), NixShellCommand(name="Build frontend", logEnviron=False, haltOnFailure=True, env=path_env, command="yarn run encore production") ] ) def _configure_symfony(name, c, worker_name, *args, **kwargs): builder_name = "Symfony_{}_build".format(name) c['schedulers'].append(git_hook_scheduler("Symfony_{}".format(name), [builder_name])) c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=symfony_project_factory(name, *args, **kwargs))) return builder_name def history_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, env=path_env, command="jekyll build")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="_site", masterdest=E.HISTORY_RELEASE_PATH, url="https://www.immae.eu/history")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.HISTORY_RELEASE_PATH))) return factory def docs_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, env=path_env, command="make html")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="_build/html", masterdest=E.DOCS_RELEASE_PATH, url="https://www.immae.eu/docs")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.DOCS_RELEASE_PATH))) return factory def recettes_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, env=path_env, command="jekyll build --trace --baseurl /recettes")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RECETTES_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="_site", masterdest=E.RECETTES_RELEASE_PATH, url="https://www.immae.eu/recettes")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RECETTES_RELEASE_PATH))) return factory def jardin_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.JARDIN_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, env=path_env, command="jekyll build --trace --baseurl /jardin")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.JARDIN_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="_site", masterdest=E.JARDIN_RELEASE_PATH, url="https://www.immae.eu/jardin")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.JARDIN_RELEASE_PATH))) return factory def bip39_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin", "NIX_PATH": "nixpkgs=channel:nixos-unstable", } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL, submodules=True, mode="full", method="fresh")) factory.addStep(NixShellCommand(name="build file", nixPackages=["python3"], logEnviron=False, haltOnFailure=True, env=path_env, command="python compile.py")) factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html", masterdest=E.BIP39_RELEASE_PATH + "/index.html", url="https://tools.immae.eu/BIP39", mode=0o644)) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.BIP39_RELEASE_PATH))) return factory def immae_eu_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin", } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, pure=False, env=path_env, command="make html")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="output", masterdest=E.IMMAE_EU_RELEASE_PATH, url="https://www.immae.eu")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH))) return factory def cours_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin", "CI": "yes" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="incremental")) factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, pure=True, command="make build", env=path_env)) factory.addStep(NixShellCommand(name="progress report", logEnviron=False, haltOnFailure=True, pure=True, command="make report", env=path_env)) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="build", masterdest=E.COURS_RELEASE_PATH, url="https://www.immae.eu/cours")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) factory.addStep(NixShellCommand(name="build pdfs", logEnviron=False, haltOnFailure=True, pure=True, command="make pdfs", env=path_env)) package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch) package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch) factory.addStep(steps.ShellCommand(name="build pdf tarball", logEnviron=False, haltOnFailure=True, command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env)) factory.addStep(steps.FileUpload(name="upload package", workersrc=package, masterdest=package_dest, url=package_url, mode=0o644)) return factory def normalesup_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin", "GIT_SSH_COMMAND": "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH), } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="incremental")) factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, pure=False, command="make build", env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", logEnviron=False, haltOnFailure=True, command="chmod -R a+rX build", env=path_env)) factory.addStep(steps.ShellCommand(name="synchronize with phare", logEnviron=False, haltOnFailure=True, env=path_env, command=[ "rsync", "-av", "--delete", "-e", "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH), "build/", os.environ["BUILDBOT_NORMALESUP_HOST"] ])) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="build", masterdest=E.NORMALESUP_RELEASE_PATH, url="https://www.immae.eu/recherche")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH))) return factory def gsm_cells_build_factory(): path_env = { "PATH": "/run/current-system/sw/bin", "NIX_PATH": "nixpkgs=channel:nixos-unstable", "IN_BUILDBOT": "yes", } master_env = { "HTACCESS": ''' Options +FollowSymLinks IndexIgnore * ''' } for k, v in os.environ.items(): if k.startswith("BUILDBOT_GSM_CELLS_"): path_env[k[len("BUILDBOT_GSM_CELLS_"):]] = v script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" factory = util.BuildFactory() factory.addStep(steps.FileDownload(mastersrc=script, workerdest="lacells_download", mode=0o755)) factory.addStep(NixShellCommand(name="download files", logEnviron=False, haltOnFailure=True, command="./lacells_download", nixPackages=["sqlite", "wget", "gzip"], pure=False, env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", logEnviron=False, haltOnFailure=True, command="chmod a+r lacells.db", env=path_env)) factory.addStep(steps.FileUpload(workersrc="lacells.db", masterdest=(E.GSMCELLS_RELEASE_PATH+"/lacells.db"), url=(E.GSMCELLS_RELEASE_URL+"/lacells.db"))) factory.addStep(steps.MasterShellCommand(command="touch {}/.duplicity-ignore".format(E.GSMCELLS_RELEASE_PATH))) factory.addStep(steps.MasterShellCommand(command='echo "$HTACCESS" > {}/.htaccess'.format(E.GSMCELLS_RELEASE_PATH), env=master_env)) factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH))) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH))) return factory def symfony_project_factory(name, repourl, parameters_path="app/config/parameters.yml", other_steps=lambda a : []): master_path_env = { "PATH": os.environ["BUILDBOT_SYMFONY_MASTER_PATH"] + ":${PATH}" } path_env = { "PATH": "/run/current-system/sw/bin" } for k, v in os.environ.items(): if k.startswith("BUILDBOT_SYMFONY_{}_".format(name)): path_env[k[len("BUILDBOT_SYMFONY_{}_".format(name)):]] = v factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=repourl, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) if parameters_path is not None: factory.addStep(steps.FileDownload(mastersrc=get_parameters_file.withArgs(name), workerdest=parameters_path)) factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, env=path_env, command=get_composer_install_command.withArgs(name))) if name in BRANCH_TO_POST_STEP: factory.addStep(NixShellCommand(name="build website post", logEnviron=False, haltOnFailure=True, doStepIf=partial(need_post_step, name), env=path_env, command=get_post_step_command.withArgs(name))) if parameters_path is not None: factory.addStep(steps.ShellCommand(name="Remove parameters.yml", logEnviron=False, haltOnFailure=True, command="rm -f {}".format(parameters_path))) for step in other_steps(path_env): factory.addStep(step) package = util.Interpolate("{}_%(kw:clean_branch)s.tar.gz".format(name), clean_branch=clean_branch) release_file = "{0}/{1}/%(kw:clean_branch)s.tar.gz" package_dest = util.Interpolate(release_file.format("/var/lib/ftp/release.immae.eu/buildbot", name), clean_branch=clean_branch) # Tar doesn’t like creating the tarball in the same directory factory.addStep(steps.ShellCommand(name="Make a tarball 1/2", logEnviron=False, haltOnFailure=True, env=path_env, command=["touch", package])) factory.addStep(steps.ShellCommand(name="Make a tarball 2/2", logEnviron=False, haltOnFailure=True, env=path_env, command=["tar", "--exclude", package, "-czf", package, "."])) factory.addStep(steps.SetPropertyFromCommand(command=get_systemd_service_invocation_command.withArgs(name), property="service_invocation_id", doStepIf=partial(need_follow_systemd, name))) factory.addStep(steps.FileUpload(name="upload package", workersrc=package, masterdest=package_dest, mode=0o644)) factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=master_path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name))) return factory