From 0012da0ff3d45df9f68412b90be4f7c24d46a777 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Sun, 1 Dec 2019 15:14:07 +0100 Subject: [PATCH] Add cours to buildbot --- modules/private/buildbot/default.nix | 4 +- .../buildbot/projects/immaeEu/__init__.py | 78 +++++++++++++++---- modules/private/websites/immae/production.nix | 20 ++++- 3 files changed, 87 insertions(+), 15 deletions(-) diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix index 8a09139..273ad70 100644 --- a/modules/private/buildbot/default.nix +++ b/modules/private/buildbot/default.nix @@ -182,7 +182,9 @@ in )} ''; environment = let - project_env = lib.attrsets.mapAttrs' (k: v: lib.attrsets.nameValuePair "BUILDBOT_${k}" v) project.environment; + project_env = with lib.attrsets; + mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment // + mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project); buildbot_config = pkgs.python3Packages.buildPythonPackage (rec { name = "buildbot_config-${project.name}"; src = ./projects + "/${project.name}"; diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py index 437af72..f29eb14 100644 --- a/modules/private/buildbot/projects/immaeEu/__init__.py +++ b/modules/private/buildbot/projects/immaeEu/__init__.py @@ -11,14 +11,20 @@ class E(): BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) - RELEASE_PATH = "/var/lib/buildbot/outputs/immaeEu" SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") - IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" + IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" + COURS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Cours" NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup" + COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/cours" + COURS_TARBALL_PATH = "/var/lib/ftp/release.immae.eu/cours" + COURS_TARBALL_URL = "https://release.immae.eu/cours" + IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immaeEu" + NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/recherche" + # master.cfg SECRETS_FILE = os.getcwd() + "/secrets" LDAP_URL = "ldaps://ldap.immae.eu:636" @@ -54,55 +60,101 @@ def configure(c): c['schedulers'].append(hook_scheduler("ImmaeEu", timer=1)) c['schedulers'].append(hook_scheduler("Normalesup", timer=1)) - c['schedulers'].append(force_scheduler("force_immae_eu", ["ImmaeEu_build", "Normalesup_build"])) + c['schedulers'].append(hook_scheduler("Cours", timer=1)) + c['schedulers'].append(force_scheduler("force_immae_eu", ["ImmaeEu_build", "Normalesup_build", "Cours_build"])) c['builders'].append(immae_eu_factory()) c['builders'].append(normalesup_factory()) + c['builders'].append(cours_factory()) c['services'].append(SlackStatusPush( name="slack_status_immae_eu_project", - builders=["ImmaeEu_build", "Normalesup_build"], + builders=["ImmaeEu_build", "Normalesup_build", "Cours_build"], serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) c['services'].append(XMPPStatusPush( name="xmpp_status_immae_eu_project", - builders=["ImmaeEu_build", "Normalesup_build"], + builders=["ImmaeEu_build", "Normalesup_build", "Cours_build"], recipients=E.XMPP_RECIPIENTS, password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip())) def immae_eu_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}" + } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) factory.addStep(steps.ShellCommand(name="build website", logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "html"])) - factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RELEASE_PATH))) + env=path_env, command=["make", "html"])) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="../source/output", - masterdest=E.RELEASE_PATH, + masterdest=E.IMMAE_EU_RELEASE_PATH, url="https://www.immae.eu")) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RELEASE_PATH))) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH))) return util.BuilderConfig(name="ImmaeEu_build", workernames=["generic-worker-immae-eu"], factory=factory) +def cours_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}" + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) + factory.addStep(steps.ShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, workdir="source", + command=["make", "build"], env=path_env)) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) + factory.addStep(steps.DirectoryUpload(workersrc="../source/build", + masterdest=E.COURS_RELEASE_PATH, + url="https://www.immae.eu/cours")) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) + + factory.addStep(steps.ShellCommand(name="build pdfs", + logEnviron=False, haltOnFailure=True, workdir="source", + command=["make", "pdfs"], env=path_env)) + + package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) + release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" + package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch) + package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch) + factory.addStep(steps.ShellCommand(name="build pdf tarball", + logEnviron=False, haltOnFailure=True, workdir="source", + command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env)) + factory.addStep(steps.FileUpload(name="upload package", workersrc=package, + workdir="source", masterdest=package_dest, + url=package_url, mode=0o644)) + + return util.BuilderConfig(name="Cours_build", workernames=["generic-worker-immae-eu"], factory=factory) + def normalesup_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}" + } factory = util.BuildFactory() factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) factory.addStep(steps.ShellCommand(name="build website", logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "build"])) + command=["make", "build"], env=path_env)) factory.addStep(steps.ShellCommand(name="give read access to all files", logEnviron=False, haltOnFailure=True, workdir="source", - command="chmod -R a+rX build")) + command="chmod -R a+rX build", env=path_env)) factory.addStep(steps.ShellCommand(name="synchronize with phare", logEnviron=False, haltOnFailure=True, workdir="source", - command=[ + env=path_env, command=[ "rsync", "-av", "--delete", "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH), "build/", - "bouya@phare.normalesup.org:" + os.environ["BUILDBOT_NORMALESUP_HOST"] ])) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH))) + factory.addStep(steps.DirectoryUpload(workersrc="../source/build", masterdest=E.NORMALESUP_RELEASE_PATH, + url="https://www.immae.eu/recherche")) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH))) return util.BuilderConfig(name="Normalesup_build", workernames=["generic-worker-immae-eu"], factory=factory) diff --git a/modules/private/websites/immae/production.nix b/modules/private/websites/immae/production.nix index e02d026..3b2301a 100644 --- a/modules/private/websites/immae/production.nix +++ b/modules/private/websites/immae/production.nix @@ -2,6 +2,8 @@ let cfg = config.myServices.websites.immae.production; varDir = "/var/lib/buildbot/outputs/immaeEu"; + coursDir = "/var/lib/buildbot/outputs/cours"; + rechercheDir = "/var/lib/buildbot/outputs/recherche"; env = myconfig.env.websites.immae; in { options.myServices.websites.immae.production.enable = lib.mkEnableOption "enable Immae's website"; @@ -19,7 +21,23 @@ in { Use Stats www.immae.eu - DirectoryIndex index.php index.htm index.html + DirectoryIndex index.htm index.html + Options Indexes FollowSymLinks MultiViews Includes + AllowOverride All + Require all granted + + + Alias /cours ${coursDir} + + DirectoryIndex index.htm index.html + Options Indexes FollowSymLinks MultiViews Includes + AllowOverride All + Require all granted + + + Alias /recherche ${rechercheDir} + + DirectoryIndex index.htm index.html Options Indexes FollowSymLinks MultiViews Includes AllowOverride All Require all granted -- 2.41.0