X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=modules%2Fprivate%2Fbuildbot%2Fprojects%2FimmaeEu%2F__init__.py;h=e817ad0ac0a93a1ddd8abee7d748ec5ffdb50e44;hb=bd0cb07b13aecd16a0782492655843a1b699611d;hp=3a2c0042ccaca129c15ba5bf75a4e238c9a5c620;hpb=f6e48e3a41e2d02299e0e1fcb295dcd82784f4e7;p=perso%2FImmae%2FConfig%2FNix.git diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py index 3a2c004..e817ad0 100644 --- a/modules/private/buildbot/projects/immaeEu/__init__.py +++ b/modules/private/buildbot/projects/immaeEu/__init__.py @@ -13,6 +13,7 @@ class E(): SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" + SSH_HOST_PATH = "/var/lib/buildbot/buildbot_hosts" SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") @@ -166,7 +167,7 @@ def get_systemd_service_invocation_command(props, name): if props.hasProperty("branch") and len(props["branch"]) > 0: service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"]) if service is not None: - return "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no -i {} buildbot@eldiron systemctl show -p InvocationID --value {}.service".format(E.SSH_KEY_PATH, service) + return "ssh -o UserKnownHostsFile={0} -i {1} buildbot@eldiron systemctl show -p InvocationID --value {2}.service".format(E.SSH_HOST_PATH, E.SSH_KEY_PATH, service) @util.renderer def follow_systemd_command(props, name, invocation_id): @@ -218,15 +219,15 @@ def _configure_symfony(name, c, worker_name, *args, **kwargs): def history_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_History"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build website", + factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, - env=path_env, command=["jekyll", "build"])) + env=path_env, command="jekyll build")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="_site", masterdest=E.HISTORY_RELEASE_PATH, @@ -237,15 +238,15 @@ def history_build_factory(): def docs_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Docs"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build website", + factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, - env=path_env, command=["make", "html"])) + env=path_env, command="make html")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="_build/html", masterdest=E.DOCS_RELEASE_PATH, @@ -256,7 +257,7 @@ def docs_build_factory(): def recettes_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL, @@ -275,14 +276,14 @@ def recettes_build_factory(): def bip39_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_BIP39"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin", + "NIX_PATH": "nixpkgs=channel:nixos-unstable", } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL, submodules=True, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build file", - logEnviron=False, haltOnFailure=True, - env=path_env, command=["python", "compile.py"])) + factory.addStep(NixShellCommand(name="build file", nixPackages=["python3"], + logEnviron=False, haltOnFailure=True, env=path_env, command="python compile.py")) factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html", masterdest=E.BIP39_RELEASE_PATH + "/index.html", url="https://tools.immae.eu/BIP39", mode=0o644)) @@ -292,15 +293,15 @@ def bip39_build_factory(): def immae_eu_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin", } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, - env=path_env, command=["make", "html"])) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=False, + env=path_env, command="make html")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="output", masterdest=E.IMMAE_EU_RELEASE_PATH, @@ -311,25 +312,25 @@ def immae_eu_build_factory(): def cours_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}", + "PATH": "/run/current-system/sw/bin", "CI": "yes" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="incremental")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, - command=["make", "build"], env=path_env)) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=True, + command="make build", env=path_env)) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="build", masterdest=E.COURS_RELEASE_PATH, url="https://www.immae.eu/cours")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) - factory.addStep(steps.ShellCommand(name="build pdfs", - logEnviron=False, haltOnFailure=True, - command=["make", "pdfs"], env=path_env)) + factory.addStep(NixShellCommand(name="build pdfs", + logEnviron=False, haltOnFailure=True, pure=True, + command="make pdfs", env=path_env)) package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" @@ -345,15 +346,16 @@ def cours_build_factory(): def normalesup_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin", + "GIT_SSH_COMMAND": "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH), } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="incremental")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, - command=["make", "build"], env=path_env)) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=False, + command="make build", env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", logEnviron=False, haltOnFailure=True, command="chmod -R a+rX build", env=path_env)) @@ -361,7 +363,7 @@ def normalesup_build_factory(): logEnviron=False, haltOnFailure=True, env=path_env, command=[ "rsync", "-av", "--delete", - "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH), + "-e", "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH), "build/", os.environ["BUILDBOT_NORMALESUP_HOST"] ])) @@ -374,7 +376,8 @@ def normalesup_build_factory(): def gsm_cells_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}", + "PATH": "/run/current-system/sw/bin", + "NIX_PATH": "nixpkgs=channel:nixos-unstable", "IN_BUILDBOT": "yes", } master_env = { @@ -389,8 +392,10 @@ def gsm_cells_build_factory(): script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" factory = util.BuildFactory() - factory.addStep(steps.ShellCommand(name="download files", - logEnviron=False, haltOnFailure=True, command=[script], env=path_env)) + factory.addStep(steps.FileDownload(mastersrc=script, workerdest="lacells_download", mode=0o755)) + factory.addStep(NixShellCommand(name="download files", + logEnviron=False, haltOnFailure=True, command="./lacells_download", + nixPackages=["sqlite", "wget", "gzip"], pure=False, env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", logEnviron=False, haltOnFailure=True, command="chmod a+r lacells.db", env=path_env)) @@ -405,14 +410,12 @@ def gsm_cells_build_factory(): return factory def symfony_project_factory(name, repourl, parameters_path="app/config/parameters.yml", other_steps=lambda a : []): - if "BUILDBOT_PATH_SYMFONY_{}".format(name) in os.environ: - path_env = { - "PATH": os.environ["BUILDBOT_PATH_SYMFONY_{}".format(name)] + ":${PATH}" - } - else: - path_env = { - "PATH": "${PATH}" - } + master_path_env = { + "PATH": os.environ["BUILDBOT_SYMFONY_MASTER_PATH"] + ":${PATH}" + } + path_env = { + "PATH": "/run/current-system/sw/bin" + } for k, v in os.environ.items(): if k.startswith("BUILDBOT_SYMFONY_{}_".format(name)): @@ -452,5 +455,5 @@ def symfony_project_factory(name, repourl, parameters_path="app/config/parameter property="service_invocation_id", doStepIf=partial(need_follow_systemd, name))) factory.addStep(steps.FileUpload(name="upload package", workersrc=package, masterdest=package_dest, mode=0o644)) - factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name))) + factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=master_path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name))) return factory