X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=modules%2Fprivate%2Fbuildbot%2Fprojects%2FimmaeEu%2F__init__.py;h=f119b4f2d2a49f3166a4399b0c1eaff1c755aec8;hb=6b017278cb0dff04fc6cde863a1bffa012ed5f59;hp=23ac970639787578e277f39c2b6392c656667de6;hpb=8fa7ff2c63fb0722144bc90837512d9f8b8c929d;p=perso%2FImmae%2FConfig%2FNix.git diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py index 23ac970..f119b4f 100644 --- a/modules/private/buildbot/projects/immaeEu/__init__.py +++ b/modules/private/buildbot/projects/immaeEu/__init__.py @@ -11,12 +11,28 @@ class E(): BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) - RELEASE_PATH = "/var/lib/buildbot/outputs/immaeEu" - GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") + BIP39_GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git" + IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" + HISTORY_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/History" + RECETTES_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Recettes" + COURS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Cours" + NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup" + + COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/cours" + COURS_TARBALL_PATH = "/var/lib/ftp/release.immae.eu/cours" + COURS_TARBALL_URL = "https://release.immae.eu/cours" + BIP39_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/bip39" + HISTORY_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/history" + IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/blog" + RECETTES_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recettes" + NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recherche" + GSMCELLS_RELEASE_PATH = "/var/lib/ftp/release.immae.eu/gsm_cells" + GSMCELLS_RELEASE_URL = "https://release.immae.eu/gsm_cells" + # master.cfg SECRETS_FILE = os.getcwd() + "/secrets" LDAP_URL = "ldaps://ldap.immae.eu:636" @@ -51,32 +67,225 @@ def configure(c): c['workers'].append(worker.LocalWorker("generic-worker-immae-eu")) c['schedulers'].append(hook_scheduler("ImmaeEu", timer=1)) - c['schedulers'].append(force_scheduler("force_immae_eu", ["ImmaeEu_build"])) + c['schedulers'].append(hook_scheduler("Normalesup", timer=1)) + c['schedulers'].append(hook_scheduler("Cours", timer=1)) + c['schedulers'].append(hook_scheduler("Recettes", timer=1)) + c['schedulers'].append(hook_scheduler("History", timer=1)) + c['schedulers'].append(hook_scheduler("BIP39", timer=1)) + c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly", + builderNames=["GSMCells_build"], dayOfWeek=6, hour=3)) + c['schedulers'].append(force_scheduler("force_immae_eu", [ + "ImmaeEu_build", "Normalesup_build", "Cours_build", + "Recettes_build", "History_build", "BIP39_build" + ])) + c['schedulers'].append(schedulers.ForceScheduler( + name="GSMCells-force", label="Force build", + buttonName="Force build", + reason=util.StringParameter(name="reason", label="Reason", default="Force build"), + codebases=[ + util.CodebaseParameter("", + branch=util.FixedParameter(name="branch", default=""), + revision=util.FixedParameter(name="revision", default=""), + repository=util.FixedParameter(name="repository", default=""), + project=util.FixedParameter(name="project", default=""), + ), + ], + username=util.FixedParameter(name="username", default="Web button"), + builderNames=["GSMCells_build"] + )) - c['builders'].append(factory()) + c['builders'].append(immae_eu_factory()) + c['builders'].append(normalesup_factory()) + c['builders'].append(cours_factory()) + c['builders'].append(gsm_cells_factory()) + c['builders'].append(recettes_factory()) + c['builders'].append(history_factory()) + c['builders'].append(bip39_factory()) c['services'].append(SlackStatusPush( name="slack_status_immae_eu_project", - builders=["ImmaeEu_build"], + builders=[ + "ImmaeEu_build", "Normalesup_build", "Cours_build", + "GSMCells_build", "Recettes_build", "History_build", + "BIP39_build" + ], serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) c['services'].append(XMPPStatusPush( name="xmpp_status_immae_eu_project", - builders=["ImmaeEu_build"], + builders=[ + "ImmaeEu_build", "Normalesup_build", "Cours_build", + "GSMCells_build", "Recettes_build", "History_build", + "BIP39_build" + ], recipients=E.XMPP_RECIPIENTS, password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip())) -def factory(): +def history_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_History"] + ":${PATH}" + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) + factory.addStep(steps.ShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, workdir="source", + env=path_env, command=["jekyll", "build"])) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH))) + factory.addStep(steps.DirectoryUpload(workersrc="../source/_site", + masterdest=E.HISTORY_RELEASE_PATH, + url="https://www.immae.eu/history")) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.HISTORY_RELEASE_PATH))) + + return util.BuilderConfig(name="History_build", workernames=["generic-worker-immae-eu"], factory=factory) + +def recettes_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}" + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) + factory.addStep(NixShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, workdir="source", + env=path_env, command="jekyll build --trace --baseurl /recettes")) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RECETTES_RELEASE_PATH))) + factory.addStep(steps.DirectoryUpload(workersrc="../source/_site", + masterdest=E.RECETTES_RELEASE_PATH, + url="https://www.immae.eu/recettes")) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RECETTES_RELEASE_PATH))) + + return util.BuilderConfig(name="Recettes_build", workernames=["generic-worker-immae-eu"], factory=factory) + +def bip39_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_BIP39"] + ":${PATH}" + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL, + submodules=True, mode="full", method="copy")) + factory.addStep(steps.ShellCommand(name="build file", + logEnviron=False, haltOnFailure=True, workdir="source", + env=path_env, command=["python", "compile.py"])) + factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html", + workdir="source", masterdest=E.BIP39_RELEASE_PATH + "/index.html", + url="https://tools.immae.eu/BIP39", mode=0o644)) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.BIP39_RELEASE_PATH))) + + return util.BuilderConfig(name="BIP39_build", workernames=["generic-worker-immae-eu"], factory=factory) + +def immae_eu_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}" + } factory = util.BuildFactory() - factory.addStep(steps.Git(logEnviron=False, repourl=E.GIT_URL, - sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) factory.addStep(steps.ShellCommand(name="build website", logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "html"])) - factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RELEASE_PATH))) + env=path_env, command=["make", "html"])) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) factory.addStep(steps.DirectoryUpload(workersrc="../source/output", - masterdest=E.RELEASE_PATH, + masterdest=E.IMMAE_EU_RELEASE_PATH, url="https://www.immae.eu")) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RELEASE_PATH))) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH))) return util.BuilderConfig(name="ImmaeEu_build", workernames=["generic-worker-immae-eu"], factory=factory) + +def cours_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}", + "CI": "yes" + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) + factory.addStep(steps.ShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, workdir="source", + command=["make", "build"], env=path_env)) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) + factory.addStep(steps.DirectoryUpload(workersrc="../source/build", + masterdest=E.COURS_RELEASE_PATH, + url="https://www.immae.eu/cours")) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) + + factory.addStep(steps.ShellCommand(name="build pdfs", + logEnviron=False, haltOnFailure=True, workdir="source", + command=["make", "pdfs"], env=path_env)) + + package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) + release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" + package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch) + package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch) + factory.addStep(steps.ShellCommand(name="build pdf tarball", + logEnviron=False, haltOnFailure=True, workdir="source", + command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env)) + factory.addStep(steps.FileUpload(name="upload package", workersrc=package, + workdir="source", masterdest=package_dest, + url=package_url, mode=0o644)) + + return util.BuilderConfig(name="Cours_build", workernames=["generic-worker-immae-eu"], factory=factory) + +def normalesup_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}" + } + factory = util.BuildFactory() + factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, + submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), + sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) + factory.addStep(steps.ShellCommand(name="build website", + logEnviron=False, haltOnFailure=True, workdir="source", + command=["make", "build"], env=path_env)) + factory.addStep(steps.ShellCommand(name="give read access to all files", + logEnviron=False, haltOnFailure=True, workdir="source", + command="chmod -R a+rX build", env=path_env)) + factory.addStep(steps.ShellCommand(name="synchronize with phare", + logEnviron=False, haltOnFailure=True, workdir="source", + env=path_env, command=[ + "rsync", "-av", "--delete", + "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH), + "build/", + os.environ["BUILDBOT_NORMALESUP_HOST"] + ])) + factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH))) + factory.addStep(steps.DirectoryUpload(workersrc="../source/build", masterdest=E.NORMALESUP_RELEASE_PATH, + url="https://www.immae.eu/recherche")) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH))) + + return util.BuilderConfig(name="Normalesup_build", workernames=["generic-worker-immae-eu"], factory=factory) + +def gsm_cells_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}", + "IN_BUILDBOT": "yes", + } + master_env = { + "HTACCESS": ''' + Options +FollowSymLinks + IndexIgnore * + ''' + } + for k, v in os.environ.items(): + if k.startswith("BUILDBOT_GSM_CELLS_"): + path_env[k[len("BUILDBOT_GSM_CELLS_"):]] = v + + script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" + factory = util.BuildFactory() + factory.addStep(steps.ShellCommand(name="download files", + logEnviron=False, haltOnFailure=True, command=[script], env=path_env)) + factory.addStep(steps.ShellCommand(name="give read access to all files", + logEnviron=False, haltOnFailure=True, + command="chmod a+r lacells.db", env=path_env)) + factory.addStep(steps.FileUpload(workersrc="lacells.db", + masterdest=(E.GSMCELLS_RELEASE_PATH+"/lacells.db"), url=(E.GSMCELLS_RELEASE_URL+"/lacells.db"))) + factory.addStep(steps.MasterShellCommand(command="touch {}/.duplicity-ignore".format(E.GSMCELLS_RELEASE_PATH))) + factory.addStep(steps.MasterShellCommand(command='echo "$HTACCESS" > {}/.htaccess'.format(E.GSMCELLS_RELEASE_PATH), + env=master_env)) + factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH))) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH))) + + return util.BuilderConfig(name="GSMCells_build", workernames=["generic-worker-immae-eu"], factory=factory)