From bd0cb07b13aecd16a0782492655843a1b699611d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Sun, 17 Apr 2022 20:33:48 +0200 Subject: [PATCH] Rework buildbot: Move towards independent builds --- .../private/buildbot/common/build_helpers.py | 59 ++++--------- modules/private/buildbot/default.nix | 12 ++- .../buildbot/projects/caldance/__init__.py | 9 -- .../projects/cryptoportfolio/__init__.py | 9 -- .../buildbot/projects/denise/__init__.py | 6 +- .../buildbot/projects/immaeEu/__init__.py | 83 ++++++++++--------- .../buildbot/projects/nicecoop/__init__.py | 4 +- .../buildbot/projects/test/__init__.py | 9 -- modules/private/environment.nix | 9 -- nix/sources.json | 63 -------------- overlays/default.nix | 1 - overlays/pelican/default.nix | 7 -- 12 files changed, 73 insertions(+), 198 deletions(-) delete mode 100644 overlays/pelican/default.nix diff --git a/modules/private/buildbot/common/build_helpers.py b/modules/private/buildbot/common/build_helpers.py index 55b8b98..ebd49ae 100644 --- a/modules/private/buildbot/common/build_helpers.py +++ b/modules/private/buildbot/common/build_helpers.py @@ -5,7 +5,7 @@ from shutil import which __all__ = [ "force_scheduler", "deploy_scheduler", "git_hook_scheduler", "clean_branch", "package_and_upload", "SlackStatusPush", - "XMPPStatusPush", "LdapEdit", "NixShellCommand", + "XMPPStatusPush", "NixShellCommand", "all_builder_names", "compute_build_infos", "deploy_ssh_command", "configure_slack_push", "configure_xmpp_push", "deploy_hook_scheduler", ] @@ -35,7 +35,7 @@ def package_and_upload(package, package_dest, package_url): # Steps class NixShellCommand(steps.ShellCommand): - def __init__(self, command=None, pure=True, nixfile=None, **kwargs): + def __init__(self, command=None, nixPackages=[], pure=True, nixFile=None, nixIncludes={}, nixArgs={}, **kwargs): oldpath = kwargs.get("env", {}).get("PATH", None) if which("nix-shell", path=oldpath) is None: kwargs["env"] = kwargs.get("env", {}) @@ -44,12 +44,22 @@ class NixShellCommand(steps.ShellCommand): elif isinstance(oldpath, list): kwargs["env"]["PATH"] = ["/run/current-system/sw/bin"] + oldpath nixcommand = ["nix-shell"] + for k, v in nixArgs.items(): + nixcommand.append("--arg") + nixcommand.append(k) + nixcommand.append(v) if pure: nixcommand.append("--pure") + for k, v in nixIncludes.items(): + nixcommand.append("-I") + nixcommand.append("{}={}".format(k, v)) nixcommand.append("--run") nixcommand.append(command) - if nixfile is not None: - nixcommand.append(nixfile) + if len(nixPackages) > 0: + nixcommand.append("-p") + nixcommand += nixPackages + elif nixFile is not None: + nixcommand.append(nixFile) super().__init__(command=nixcommand, **kwargs) # Schedulers @@ -307,47 +317,6 @@ from buildbot.process.buildstep import FAILURE from buildbot.process.buildstep import SUCCESS from buildbot.process.buildstep import BuildStep -class LdapEdit(BuildStep): - name = "LdapEdit" - renderables = ["environment", "build_version", "build_hash", "ldap_password"] - - def __init__(self, **kwargs): - self.environment = kwargs.pop("environment") - self.build_version = kwargs.pop("build_version") - self.build_hash = kwargs.pop("build_hash") - self.ldap_password = kwargs.pop("ldap_password") - self.ldap_host = kwargs.pop("ldap_host") - self.ldap_dn = kwargs.pop("ldap_dn") - self.ldap_roles_base = kwargs.pop("ldap_roles_base") - self.ldap_cn_template = kwargs.pop("ldap_cn_template") - self.config_key = kwargs.pop("config_key") - super().__init__(**kwargs) - - def run(self): - import json - from ldap3 import Reader, Writer, Server, Connection, ObjectDef - server = Server(self.ldap_host) - conn = Connection(server, - user=self.ldap_dn, - password=self.ldap_password) - conn.bind() - obj = ObjectDef("immaePuppetClass", conn) - r = Reader(conn, obj, - "cn={},{}".format(self.ldap_cn_template.format(self.environment), self.ldap_roles_base)) - r.search() - if len(r) > 0: - w = Writer.from_cursor(r) - for value in w[0].immaePuppetJson.values: - config = json.loads(value) - if "{}_version".format(self.config_key) in config: - config["{}_version".format(self.config_key)] = self.build_version - config["{}_sha256".format(self.config_key)] = self.build_hash - w[0].immaePuppetJson -= value - w[0].immaePuppetJson += json.dumps(config, indent=" ") - w.commit() - return defer.succeed(SUCCESS) - return defer.succeed(FAILURE) - def compute_build_infos(prefix, release_path): @util.renderer def compute(props): diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix index e8d656b..3ee1f8b 100644 --- a/modules/private/buildbot/default.nix +++ b/modules/private/buildbot/default.nix @@ -124,7 +124,6 @@ in text = let project_env = with lib.attrsets; mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" (if builtins.isFunction v then v pkgs else v)) project.environment // - mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) // { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; BUILDBOT_WORKER_PORT = builtins.toString project.workerPort; @@ -156,6 +155,16 @@ in group = "buildbot"; text = config.myEnv.buildbot.ssh_key.private; }; + "buildbot/ssh_known_hosts" = { + permissions = "0644"; + user = "buildbot"; + group = "buildbot"; + text = '' + git.immae.eu ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF + eldiron ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF + phare.normalesup.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN2GomItXICXpCtCFRMT2xuerqx2nLMO/3mNUuWyzFr1 + ''; + }; }; services.filesWatcher = lib.attrsets.mapAttrs' (k: project: lib.attrsets.nameValuePair "buildbot-${project.name}" { @@ -222,6 +231,7 @@ in ln -sf ${tac_file} ${varDir}/${project.name}/buildbot.tac # different buildbots may be trying that simultaneously, add the || true to avoid complaining in case of race install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ssh_key"} ${varDir}/buildbot_key || true + install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ssh_known_hosts"} ${varDir}/buildbot_hosts || true buildbot_secrets=${varDir}/${project.name}/secrets install -m 0700 -o buildbot -g buildbot -d $buildbot_secrets install -Dm600 -o buildbot -g buildbot -T ${config.secrets.fullPaths."buildbot/ldap"} $buildbot_secrets/ldap diff --git a/modules/private/buildbot/projects/caldance/__init__.py b/modules/private/buildbot/projects/caldance/__init__.py index 87d30d5..747e86e 100644 --- a/modules/private/buildbot/projects/caldance/__init__.py +++ b/modules/private/buildbot/projects/caldance/__init__.py @@ -16,9 +16,6 @@ class E(): GIT_URL = "gitolite@git.immae.eu:perso/simon_descarpentries/www.cal-dance.com" SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" - LDAP_HOST = "ldap.immae.eu" - LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" - LDAP_ROLES_BASE = "ou=roles,ou=hosts,dc=immae,dc=eu" XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") DEPLOY_HOSTS = { @@ -116,11 +113,5 @@ def deploy_factory(): factory = util.BuildFactory() factory.addStep(steps.MasterShellCommand(command=["test", "-f", package_dest])) factory.addStep(steps.SetProperties(properties=compute_build_infos(project, E.RELEASE_PATH))) - factory.addStep(LdapEdit(environment=util.Property("environment"), - build_version=util.Property("build_version"), - build_hash=util.Property("build_hash"), - config_key="role::caldance::{}".format(project), - ldap_host=E.LDAP_HOST, ldap_roles_base=E.LDAP_ROLES_BASE, ldap_dn=E.LDAP_DN, - ldap_cn_template="caldance.{}", ldap_password=util.Secret("ldap"))) factory.addStep(steps.MasterShellCommand(command=deploy_ssh_command(E.SSH_KEY_PATH, E.DEPLOY_HOSTS))) return factory diff --git a/modules/private/buildbot/projects/cryptoportfolio/__init__.py b/modules/private/buildbot/projects/cryptoportfolio/__init__.py index 1e94d08..b99ebb5 100644 --- a/modules/private/buildbot/projects/cryptoportfolio/__init__.py +++ b/modules/private/buildbot/projects/cryptoportfolio/__init__.py @@ -13,9 +13,6 @@ class E(): RELEASE_URL = "https://release.immae.eu/{}".format(PROJECT) GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/Cryptoportfolio/{0}.git" SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" - LDAP_HOST = "ldap.immae.eu" - LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" - LDAP_ROLES_BASE = "ou=roles,ou=hosts,dc=immae,dc=eu" DEPLOY_HOSTS = { "production": "root@cryptoportfolio.immae.eu", @@ -105,11 +102,5 @@ def deploy_factory(project): factory = util.BuildFactory() factory.addStep(steps.MasterShellCommand(command=["test", "-f", package_dest])) factory.addStep(steps.SetProperties(properties=compute_build_infos(project, "{}/{}".format(E.RELEASE_PATH, project)))) - factory.addStep(LdapEdit(environment=util.Property("environment"), - build_version=util.Property("build_version"), - build_hash=util.Property("build_hash"), - config_key="role::cryptoportfolio::{}".format(project), - ldap_host=E.LDAP_HOST, ldap_roles_base=E.LDAP_ROLES_BASE, ldap_dn=E.LDAP_DN, - ldap_cn_template="cryptoportfolio.{}", ldap_password=util.Secret("ldap"))) factory.addStep(steps.MasterShellCommand(command=deploy_ssh_command(E.SSH_KEY_PATH, E.DEPLOY_HOSTS))) return factory diff --git a/modules/private/buildbot/projects/denise/__init__.py b/modules/private/buildbot/projects/denise/__init__.py index 96b2fcf..12a72c3 100644 --- a/modules/private/buildbot/projects/denise/__init__.py +++ b/modules/private/buildbot/projects/denise/__init__.py @@ -148,16 +148,16 @@ def oms_build_factory(): def aventuriers_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Aventuriers"] + ":${PATH}", + "PATH": "/run/current-system/sw/bin", "TZ": "Europe/Paris", } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.AVENTURIERS_GIT_URL, submodules=True, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build files", + factory.addStep(NixShellCommand(name="build files", logEnviron=False, haltOnFailure=True, - env=path_env, command=["make", "tout", "encyclo"])) + env=path_env, command=["make tout encyclo"])) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.AVENTURIERS_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="html", masterdest=E.AVENTURIERS_RELEASE_PATH, diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py index 3a2c004..e817ad0 100644 --- a/modules/private/buildbot/projects/immaeEu/__init__.py +++ b/modules/private/buildbot/projects/immaeEu/__init__.py @@ -13,6 +13,7 @@ class E(): SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" + SSH_HOST_PATH = "/var/lib/buildbot/buildbot_hosts" SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") @@ -166,7 +167,7 @@ def get_systemd_service_invocation_command(props, name): if props.hasProperty("branch") and len(props["branch"]) > 0: service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"]) if service is not None: - return "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no -i {} buildbot@eldiron systemctl show -p InvocationID --value {}.service".format(E.SSH_KEY_PATH, service) + return "ssh -o UserKnownHostsFile={0} -i {1} buildbot@eldiron systemctl show -p InvocationID --value {2}.service".format(E.SSH_HOST_PATH, E.SSH_KEY_PATH, service) @util.renderer def follow_systemd_command(props, name, invocation_id): @@ -218,15 +219,15 @@ def _configure_symfony(name, c, worker_name, *args, **kwargs): def history_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_History"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build website", + factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, - env=path_env, command=["jekyll", "build"])) + env=path_env, command="jekyll build")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="_site", masterdest=E.HISTORY_RELEASE_PATH, @@ -237,15 +238,15 @@ def history_build_factory(): def docs_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Docs"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build website", + factory.addStep(NixShellCommand(name="build website", logEnviron=False, haltOnFailure=True, - env=path_env, command=["make", "html"])) + env=path_env, command="make html")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="_build/html", masterdest=E.DOCS_RELEASE_PATH, @@ -256,7 +257,7 @@ def docs_build_factory(): def recettes_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL, @@ -275,14 +276,14 @@ def recettes_build_factory(): def bip39_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_BIP39"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin", + "NIX_PATH": "nixpkgs=channel:nixos-unstable", } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL, submodules=True, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build file", - logEnviron=False, haltOnFailure=True, - env=path_env, command=["python", "compile.py"])) + factory.addStep(NixShellCommand(name="build file", nixPackages=["python3"], + logEnviron=False, haltOnFailure=True, env=path_env, command="python compile.py")) factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html", masterdest=E.BIP39_RELEASE_PATH + "/index.html", url="https://tools.immae.eu/BIP39", mode=0o644)) @@ -292,15 +293,15 @@ def bip39_build_factory(): def immae_eu_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin", } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, - env=path_env, command=["make", "html"])) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=False, + env=path_env, command="make html")) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="output", masterdest=E.IMMAE_EU_RELEASE_PATH, @@ -311,25 +312,25 @@ def immae_eu_build_factory(): def cours_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}", + "PATH": "/run/current-system/sw/bin", "CI": "yes" } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="incremental")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, - command=["make", "build"], env=path_env)) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=True, + command="make build", env=path_env)) factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="build", masterdest=E.COURS_RELEASE_PATH, url="https://www.immae.eu/cours")) factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) - factory.addStep(steps.ShellCommand(name="build pdfs", - logEnviron=False, haltOnFailure=True, - command=["make", "pdfs"], env=path_env)) + factory.addStep(NixShellCommand(name="build pdfs", + logEnviron=False, haltOnFailure=True, pure=True, + command="make pdfs", env=path_env)) package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" @@ -345,15 +346,16 @@ def cours_build_factory(): def normalesup_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}" + "PATH": "/run/current-system/sw/bin", + "GIT_SSH_COMMAND": "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH), } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="incremental")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, - command=["make", "build"], env=path_env)) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, pure=False, + command="make build", env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", logEnviron=False, haltOnFailure=True, command="chmod -R a+rX build", env=path_env)) @@ -361,7 +363,7 @@ def normalesup_build_factory(): logEnviron=False, haltOnFailure=True, env=path_env, command=[ "rsync", "-av", "--delete", - "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH), + "-e", "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH), "build/", os.environ["BUILDBOT_NORMALESUP_HOST"] ])) @@ -374,7 +376,8 @@ def normalesup_build_factory(): def gsm_cells_build_factory(): path_env = { - "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}", + "PATH": "/run/current-system/sw/bin", + "NIX_PATH": "nixpkgs=channel:nixos-unstable", "IN_BUILDBOT": "yes", } master_env = { @@ -389,8 +392,10 @@ def gsm_cells_build_factory(): script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" factory = util.BuildFactory() - factory.addStep(steps.ShellCommand(name="download files", - logEnviron=False, haltOnFailure=True, command=[script], env=path_env)) + factory.addStep(steps.FileDownload(mastersrc=script, workerdest="lacells_download", mode=0o755)) + factory.addStep(NixShellCommand(name="download files", + logEnviron=False, haltOnFailure=True, command="./lacells_download", + nixPackages=["sqlite", "wget", "gzip"], pure=False, env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", logEnviron=False, haltOnFailure=True, command="chmod a+r lacells.db", env=path_env)) @@ -405,14 +410,12 @@ def gsm_cells_build_factory(): return factory def symfony_project_factory(name, repourl, parameters_path="app/config/parameters.yml", other_steps=lambda a : []): - if "BUILDBOT_PATH_SYMFONY_{}".format(name) in os.environ: - path_env = { - "PATH": os.environ["BUILDBOT_PATH_SYMFONY_{}".format(name)] + ":${PATH}" - } - else: - path_env = { - "PATH": "${PATH}" - } + master_path_env = { + "PATH": os.environ["BUILDBOT_SYMFONY_MASTER_PATH"] + ":${PATH}" + } + path_env = { + "PATH": "/run/current-system/sw/bin" + } for k, v in os.environ.items(): if k.startswith("BUILDBOT_SYMFONY_{}_".format(name)): @@ -452,5 +455,5 @@ def symfony_project_factory(name, repourl, parameters_path="app/config/parameter property="service_invocation_id", doStepIf=partial(need_follow_systemd, name))) factory.addStep(steps.FileUpload(name="upload package", workersrc=package, masterdest=package_dest, mode=0o644)) - factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name))) + factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=master_path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name))) return factory diff --git a/modules/private/buildbot/projects/nicecoop/__init__.py b/modules/private/buildbot/projects/nicecoop/__init__.py index b2c02ee..536b424 100644 --- a/modules/private/buildbot/projects/nicecoop/__init__.py +++ b/modules/private/buildbot/projects/nicecoop/__init__.py @@ -42,7 +42,7 @@ def configure(c): def gestion_factory(env): path_env = { - "PATH": os.environ["BUILDBOT_PATH_Gestion"] + ":${PATH}", + "PATH": "/run/current-system/sw/bin" } factory = util.BuildFactory() @@ -59,7 +59,7 @@ def gestion_factory(env): factory.addStep(steps.ShellCommand(name="remove symlinks", logEnviron=False, haltOnFailure=True, workdir="{}_app".format(env), command="rm var app/config/parameters.yml")) - factory.addStep(steps.ShellCommand(name="copy parameters", + factory.addStep(NixShellCommand(name="copy parameters", logEnviron=False, haltOnFailure=True, env=path_env, command="cat {0}/parameters.yml | gucci -f /var/secrets/buildbot/nicecoop/{0}.yml > ../{0}_app/app/config/parameters.yml".format(env))) factory.addStep(steps.ShellCommand(name="test configuration", diff --git a/modules/private/buildbot/projects/test/__init__.py b/modules/private/buildbot/projects/test/__init__.py index 2898bbc..926a6bb 100644 --- a/modules/private/buildbot/projects/test/__init__.py +++ b/modules/private/buildbot/projects/test/__init__.py @@ -18,9 +18,6 @@ class E(): GIT_URL = "https://git.immae.eu/perso/Immae/TestProject.git" SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" LIBVIRT_URL = os.environ["BUILDBOT_VIRT_URL"] + "?keyfile=" + SSH_KEY_PATH - LDAP_HOST = "ldap.immae.eu" - LDAP_DN = "cn=buildbot,ou=services,dc=immae,dc=eu" - LDAP_ROLES_BASE = "ou=roles,ou=hosts,dc=immae,dc=eu" XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") # master.cfg @@ -120,11 +117,5 @@ def deploy_factory(): factory = util.BuildFactory() factory.addStep(steps.MasterShellCommand(command=["test", "-f", package_dest])) factory.addStep(steps.SetProperties(properties=compute_build_infos("test", E.RELEASE_PATH))) - factory.addStep(LdapEdit(environment=util.Property("environment"), - build_version=util.Property("build_version"), - build_hash=util.Property("build_hash"), - config_key="test", - ldap_host=E.LDAP_HOST, ldap_roles_base=E.LDAP_ROLES_BASE, ldap_dn=E.LDAP_DN, - ldap_cn_template="test.{}", ldap_password=util.Secret("ldap"))) factory.addStep(steps.MasterShellCommand(command=deploy_ssh_command(E.SSH_KEY_PATH, {}))) return factory diff --git a/modules/private/environment.nix b/modules/private/environment.nix index 7b13870..b321e03 100644 --- a/modules/private/environment.nix +++ b/modules/private/environment.nix @@ -887,15 +887,6 @@ in Activation script to run during deployment ''; }; - builderPaths = mkOption { - type = attrsOf unspecified; - default = {}; - description = '' - Attrs of functions to make accessible specifically per builder. - Takes pkgs as argument and should return a single path containing binaries. - This path will be accessible as BUILDBOT_PATH_ - ''; - }; webhookTokens = mkOption { type = nullOr (listOf str); default = null; diff --git a/nix/sources.json b/nix/sources.json index f73ea55..514ddcd 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,55 +1,4 @@ { - "buildbot-cours-deps": { - "ref": "master", - "repo": "ssh://gitolite@git.immae.eu/perso/Immae/Sites/Cours", - "rev": "33f139f2e2ea0a16b8ed813c5436e5a5bb506f6d", - "type": "git" - }, - "buildbot-cours-nixpkgs": { - "branch": "nixos-unstable", - "description": "Frozen nixpkgs snapshot for texlive", - "homepage": "https://github.com/NixOS/nixpkgs", - "owner": "NixOS", - "repo": "nixpkgs-channels", - "rev": "fce7562cf46727fdaf801b232116bc9ce0512049", - "sha256": "14rvi69ji61x3z88vbn17rg5vxrnw2wbnanxb7y0qzyqrj7spapx", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs-channels/archive/fce7562cf46727fdaf801b232116bc9ce0512049.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "buildbot-denise-aventuriers-nixpkgs": { - "branch": "nixos-unstable", - "description": "Frozen nixpkgs snapshot for texlive", - "homepage": "https://github.com/NixOS/nixpkgs", - "owner": "NixOS", - "repo": "nixpkgs-channels", - "rev": "8e2b14aceb1d40c7e8b84c03a7c78955359872bb", - "sha256": "0zzjpd9smr7rxzrdf6raw9kbj42fbvafxb5bz36lcxgv290pgsm8", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs-channels/archive/8e2b14aceb1d40c7e8b84c03a7c78955359872bb.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "buildbot-history-deps": { - "ref": "master", - "repo": "ssh://gitolite@git.immae.eu/perso/Immae/Sites/History", - "rev": "5f2aae4f9c92107accfc6fb6db6cb3c1f97ed9dc", - "type": "git" - }, - "buildbot-normalesup-deps": { - "ref": "master", - "repo": "ssh://gitolite@git.immae.eu/perso/Immae/Projets/Sites/Normalesup", - "rev": "de294da4027e80a7b8bf7ec8c393ca0b60218575", - "type": "git" - }, - "buildbot-normalesup-nixpkgs": { - "description": "Frozen nixpkgs snapshot for texlive", - "revision": "173017.85f820d6e41", - "sha256": "0fs390gvz8d3n7v4zj538gf3n1b14sikbf0ijrxgxib5i0704mdb", - "type": "tarball", - "url": "https://releases.nixos.org/nixos/19.03/nixos-19.03.173017.85f820d6e41/nixexprs.tar.xz", - "url_template": "https://releases.nixos.org/nixos//nixos-./nixexprs.tar.xz", - "version": "19.03" - }, "home-manager": { "branch": "master", "description": "Manage a user environment using Nix", @@ -62,18 +11,6 @@ "url": "https://github.com/rycee/home-manager/archive/6cf6b587b575493e7718bf08b209013d7dcf4d58.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "niv": { - "branch": "master", - "description": "Easy dependency management for Nix projects", - "homepage": "https://github.com/nmattia/niv", - "owner": "nmattia", - "repo": "niv", - "rev": "372f96bff217a7a019de27667d04118cffa9841b", - "sha256": "1l0z6162zw60pdcdj03aq64qgf1vyzmf24i9pxca64i4sprl1b7p", - "type": "tarball", - "url": "https://github.com/nmattia/niv/archive/372f96bff217a7a019de27667d04118cffa9841b.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, "nixpkgs": { "branch": "nixos-unstable", "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", diff --git a/overlays/default.nix b/overlays/default.nix index ccdaa3c..f4d28bc 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -34,7 +34,6 @@ in flakes // { neomutt = import ./neomutt; nixops = import ./nixops; pass = import ./pass; - pelican = import ./pelican; php-packages = import ./php-packages; postfix = import ./postfix; postgresql = import ./databases/postgresql; diff --git a/overlays/pelican/default.nix b/overlays/pelican/default.nix deleted file mode 100644 index 4f8aece..0000000 --- a/overlays/pelican/default.nix +++ /dev/null @@ -1,7 +0,0 @@ -self: super: { - pelican = with self.python3Packages; - pelican.overrideAttrs(old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pyyaml markdown ]; - doInstallCheck = false; - }); -} -- 2.41.0