Add report command to cours

[perso/Immae/Config/Nix.git] / modules / private / buildbot / projects / immaeEu / __init__.py

diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py
index 3a2c0042ccaca129c15ba5bf75a4e238c9a5c620..f55250ded79ed26534fb236200ed24d9b2d78bed 100644 (file)
--- a/modules/private/buildbot/projects/immaeEu/__init__.py
+++ b/modules/private/buildbot/projects/immaeEu/__init__.py
@@ -13,6 +13,7 @@ class E():
     SOCKET        = "unix:/run/buildbot/{}.sock".format(PROJECT)
     PB_SOCKET     = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT)
     SSH_KEY_PATH  = "/var/lib/buildbot/buildbot_key"
+    SSH_HOST_PATH = "/var/lib/buildbot/buildbot_hosts"
     SSH_HOST_KEY  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF"
     XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ")
 
@@ -81,7 +82,7 @@ def configure(c):
     c['schedulers'].append(force_scheduler("force_immae_eu", withbranch))
     c['schedulers'].append(force_scheduler("force_no_branch_immae_eu", withoutbranch, nobranch=True))
 
-    configure_slack_push(c, E.SECRETS_FILE, all_builder_names(c))
+    configure_apprise_push(c, E.SECRETS_FILE, all_builder_names(c))
     configure_xmpp_push(c, E.SECRETS_FILE, all_builder_names(c), E.XMPP_RECIPIENTS)
 
 BRANCH_TO_SYMFONY_ENV = {
@@ -166,7 +167,7 @@ def get_systemd_service_invocation_command(props, name):
     if props.hasProperty("branch") and len(props["branch"]) > 0:
         service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"])
         if service is not None:
-            return "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no -i {} buildbot@eldiron systemctl show -p InvocationID --value {}.service".format(E.SSH_KEY_PATH, service)
+            return "ssh -o UserKnownHostsFile={0} -i {1} buildbot@eldiron systemctl show -p InvocationID --value {2}.service".format(E.SSH_HOST_PATH, E.SSH_KEY_PATH, service)
 
 @util.renderer
 def follow_systemd_command(props, name, invocation_id):
@@ -218,15 +219,15 @@ def _configure_symfony(name, c, worker_name, *args, **kwargs):
 
 def history_build_factory():
     path_env = {
-            "PATH": os.environ["BUILDBOT_PATH_History"] + ":${PATH}"
+            "PATH": "/run/current-system/sw/bin"
             }
     factory = util.BuildFactory()
     factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL,
         submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
         sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh"))
-    factory.addStep(steps.ShellCommand(name="build website",
+    factory.addStep(NixShellCommand(name="build website",
         logEnviron=False, haltOnFailure=True,
-        env=path_env, command=["jekyll", "build"]))
+        env=path_env, command="jekyll build"))
     factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH)))
     factory.addStep(steps.DirectoryUpload(workersrc="_site",
         masterdest=E.HISTORY_RELEASE_PATH,
@@ -237,15 +238,15 @@ def history_build_factory():
 
 def docs_build_factory():
     path_env = {
-            "PATH": os.environ["BUILDBOT_PATH_Docs"] + ":${PATH}"
+            "PATH": "/run/current-system/sw/bin"
             }
     factory = util.BuildFactory()
     factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL,
         submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
         sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh"))
-    factory.addStep(steps.ShellCommand(name="build website",
+    factory.addStep(NixShellCommand(name="build website",
         logEnviron=False, haltOnFailure=True,
-        env=path_env, command=["make", "html"]))
+        env=path_env, command="make html"))
     factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH)))
     factory.addStep(steps.DirectoryUpload(workersrc="_build/html",
         masterdest=E.DOCS_RELEASE_PATH,
@@ -256,7 +257,7 @@ def docs_build_factory():
 
 def recettes_build_factory():
     path_env = {
-            "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}"
+            "PATH": "/run/current-system/sw/bin"
             }
     factory = util.BuildFactory()
     factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL,
@@ -275,14 +276,14 @@ def recettes_build_factory():
 
 def bip39_build_factory():
     path_env = {
-            "PATH": os.environ["BUILDBOT_PATH_BIP39"] + ":${PATH}"
+            "PATH": "/run/current-system/sw/bin",
+            "NIX_PATH": "nixpkgs=channel:nixos-unstable",
             }
     factory = util.BuildFactory()
     factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL,
         submodules=True, mode="full", method="fresh"))
-    factory.addStep(steps.ShellCommand(name="build file",
-        logEnviron=False, haltOnFailure=True,
-        env=path_env, command=["python", "compile.py"]))
+    factory.addStep(NixShellCommand(name="build file", nixPackages=["python3"],
+        logEnviron=False, haltOnFailure=True, env=path_env, command="python compile.py"))
     factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html",
         masterdest=E.BIP39_RELEASE_PATH + "/index.html",
         url="https://tools.immae.eu/BIP39", mode=0o644))
@@ -292,15 +293,15 @@ def bip39_build_factory():
 
 def immae_eu_build_factory():
     path_env = {
-            "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}"
+            "PATH": "/run/current-system/sw/bin",
             }
     factory = util.BuildFactory()
     factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL,
         submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
         sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh"))
-    factory.addStep(steps.ShellCommand(name="build website",
-        logEnviron=False, haltOnFailure=True,
-        env=path_env, command=["make", "html"]))
+    factory.addStep(NixShellCommand(name="build website",
+        logEnviron=False, haltOnFailure=True, pure=False,
+        env=path_env, command="make html"))
     factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH)))
     factory.addStep(steps.DirectoryUpload(workersrc="output",
         masterdest=E.IMMAE_EU_RELEASE_PATH,
@@ -311,25 +312,28 @@ def immae_eu_build_factory():
 
 def cours_build_factory():
     path_env = {
-            "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}",
+            "PATH": "/run/current-system/sw/bin",
             "CI": "yes"
             }
     factory = util.BuildFactory()
     factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL,
         submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
         sshHostKey=E.SSH_HOST_KEY, mode="incremental"))
-    factory.addStep(steps.ShellCommand(name="build website",
-        logEnviron=False, haltOnFailure=True,
-        command=["make", "build"], env=path_env))
+    factory.addStep(NixShellCommand(name="build website",
+        logEnviron=False, haltOnFailure=True, pure=True,
+        command="make build", env=path_env))
+    factory.addStep(NixShellCommand(name="progress report",
+        logEnviron=False, haltOnFailure=True, pure=True,
+        command="make report", env=path_env))
     factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH)))
     factory.addStep(steps.DirectoryUpload(workersrc="build",
         masterdest=E.COURS_RELEASE_PATH,
         url="https://www.immae.eu/cours"))
     factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH)))
 
-    factory.addStep(steps.ShellCommand(name="build pdfs",
-        logEnviron=False, haltOnFailure=True,
-        command=["make", "pdfs"], env=path_env))
+    factory.addStep(NixShellCommand(name="build pdfs",
+        logEnviron=False, haltOnFailure=True, pure=True,
+        command="make pdfs", env=path_env))
 
     package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch)
     release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz"
@@ -345,15 +349,16 @@ def cours_build_factory():
 
 def normalesup_build_factory():
     path_env = {
-            "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}"
+            "PATH": "/run/current-system/sw/bin",
+            "GIT_SSH_COMMAND": "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH),
             }
     factory = util.BuildFactory()
     factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL,
         submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
         sshHostKey=E.SSH_HOST_KEY, mode="incremental"))
-    factory.addStep(steps.ShellCommand(name="build website",
-        logEnviron=False, haltOnFailure=True,
-        command=["make", "build"], env=path_env))
+    factory.addStep(NixShellCommand(name="build website",
+        logEnviron=False, haltOnFailure=True, pure=False,
+        command="make build", env=path_env))
     factory.addStep(steps.ShellCommand(name="give read access to all files",
         logEnviron=False, haltOnFailure=True,
         command="chmod -R a+rX build", env=path_env))
@@ -361,7 +366,7 @@ def normalesup_build_factory():
         logEnviron=False, haltOnFailure=True,
         env=path_env, command=[
             "rsync", "-av", "--delete",
-            "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH),
+            "-e", "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH),
             "build/",
             os.environ["BUILDBOT_NORMALESUP_HOST"]
             ]))
@@ -374,7 +379,8 @@ def normalesup_build_factory():
 
 def gsm_cells_build_factory():
     path_env = {
-            "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}",
+            "PATH": "/run/current-system/sw/bin",
+            "NIX_PATH": "nixpkgs=channel:nixos-unstable",
             "IN_BUILDBOT": "yes",
             }
     master_env = {
@@ -389,8 +395,10 @@ def gsm_cells_build_factory():
 
     script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download"
     factory = util.BuildFactory()
-    factory.addStep(steps.ShellCommand(name="download files",
-        logEnviron=False, haltOnFailure=True, command=[script], env=path_env))
+    factory.addStep(steps.FileDownload(mastersrc=script, workerdest="lacells_download", mode=0o755))
+    factory.addStep(NixShellCommand(name="download files",
+        logEnviron=False, haltOnFailure=True, command="./lacells_download",
+        nixPackages=["sqlite", "wget", "gzip"], pure=False, env=path_env))
     factory.addStep(steps.ShellCommand(name="give read access to all files",
         logEnviron=False, haltOnFailure=True,
         command="chmod a+r lacells.db", env=path_env))
@@ -405,14 +413,12 @@ def gsm_cells_build_factory():
     return factory
 
 def symfony_project_factory(name, repourl, parameters_path="app/config/parameters.yml", other_steps=lambda a : []):
-    if "BUILDBOT_PATH_SYMFONY_{}".format(name) in os.environ:
-        path_env = {
-                "PATH": os.environ["BUILDBOT_PATH_SYMFONY_{}".format(name)] + ":${PATH}"
-                }
-    else:
-        path_env = {
-                "PATH": "${PATH}"
-                }
+    master_path_env = {
+            "PATH": os.environ["BUILDBOT_SYMFONY_MASTER_PATH"] + ":${PATH}"
+            }
+    path_env = {
+            "PATH": "/run/current-system/sw/bin"
+            }
 
     for k, v in os.environ.items():
         if k.startswith("BUILDBOT_SYMFONY_{}_".format(name)):
@@ -452,5 +458,5 @@ def symfony_project_factory(name, repourl, parameters_path="app/config/parameter
         property="service_invocation_id", doStepIf=partial(need_follow_systemd, name)))
     factory.addStep(steps.FileUpload(name="upload package",
         workersrc=package, masterdest=package_dest, mode=0o644))
-    factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name)))
+    factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=master_path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name)))
     return factory