X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;ds=sidebyside;f=modules%2Fprivate%2Fbuildbot%2Fprojects%2FimmaeEu%2F__init__.py;fp=modules%2Fprivate%2Fbuildbot%2Fprojects%2FimmaeEu%2F__init__.py;h=0000000000000000000000000000000000000000;hb=1a64deeb894dc95e2645a75771732c6cc53a79ad;hp=83265cdfe24c5e6c02f4287b766c61a46a06c4ec;hpb=fa25ffd4583cc362075cd5e1b4130f33306103f0;p=perso%2FImmae%2FConfig%2FNix.git diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py deleted file mode 100644 index 83265cd..0000000 --- a/modules/private/buildbot/projects/immaeEu/__init__.py +++ /dev/null @@ -1,314 +0,0 @@ -from buildbot.plugins import * -from buildbot_common.build_helpers import * -import os -from buildbot.util import bytes2unicode -import json - -__all__ = [ "configure", "E" ] - -class E(): - PROJECT = "immaeEu" - BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) - SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) - PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) - SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" - SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" - XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") - - BIP39_GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git" - IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" - HISTORY_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/History" - RECETTES_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Recettes" - COURS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Cours" - DOCS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Docs" - NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup" - - COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/cours" - COURS_TARBALL_PATH = "/var/lib/ftp/release.immae.eu/cours" - COURS_TARBALL_URL = "https://release.immae.eu/cours" - BIP39_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/bip39" - HISTORY_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/history" - IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/blog" - DOCS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/docs" - RECETTES_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recettes" - NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recherche" - GSMCELLS_RELEASE_PATH = "/var/lib/ftp/release.immae.eu/gsm_cells" - GSMCELLS_RELEASE_URL = "https://release.immae.eu/gsm_cells" - - # master.cfg - SECRETS_FILE = os.getcwd() + "/secrets" - LDAP_URL = "ldaps://ldap.immae.eu:636" - LDAP_ADMIN_USER = "cn=buildbot,ou=services,dc=immae,dc=eu" - LDAP_BASE = "dc=immae,dc=eu" - LDAP_PATTERN = "(uid=%(username)s)" - LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=immaeEu,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))" - TITLE_URL = "https://www.immae.eu" - TITLE = "Immae website" - -class CustomBase(webhooks.base): - def getChanges(self, request): - try: - content = request.content.read() - args = json.loads(bytes2unicode(content)) - except Exception as e: - raise ValueError("Error loading JSON: " + str(e)) - - args.setdefault("comments", "") - args.setdefault("repository", "") - args.setdefault("author", args.get("who", "unknown")) - - return ([args], None) - -def configure(c): - c["buildbotURL"] = E.BUILDBOT_URL - c["www"]["port"] = E.SOCKET - - c["www"]["change_hook_dialects"]["base"] = { - "custom_class": CustomBase - } - c['workers'].append(worker.LocalWorker("generic-worker-immae-eu")) - - c['schedulers'].append(hook_scheduler("ImmaeEu", timer=1)) - c['schedulers'].append(hook_scheduler("Normalesup", timer=1)) - c['schedulers'].append(hook_scheduler("Cours", timer=1)) - c['schedulers'].append(hook_scheduler("Recettes", timer=1)) - c['schedulers'].append(hook_scheduler("Docs", timer=1)) - c['schedulers'].append(hook_scheduler("History", timer=1)) - c['schedulers'].append(hook_scheduler("BIP39", timer=1)) - c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly", - builderNames=["GSMCells_build"], dayOfWeek=6, hour=3)) - c['schedulers'].append(force_scheduler("force_immae_eu", [ - "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build", - "Recettes_build", "History_build", "BIP39_build" - ])) - c['schedulers'].append(schedulers.ForceScheduler( - name="GSMCells-force", label="Force build", - buttonName="Force build", - reason=util.StringParameter(name="reason", label="Reason", default="Force build"), - codebases=[ - util.CodebaseParameter("", - branch=util.FixedParameter(name="branch", default=""), - revision=util.FixedParameter(name="revision", default=""), - repository=util.FixedParameter(name="repository", default=""), - project=util.FixedParameter(name="project", default=""), - ), - ], - username=util.FixedParameter(name="username", default="Web button"), - builderNames=["GSMCells_build"] - )) - - c['builders'].append(immae_eu_factory()) - c['builders'].append(normalesup_factory()) - c['builders'].append(cours_factory()) - c['builders'].append(gsm_cells_factory()) - c['builders'].append(recettes_factory()) - c['builders'].append(docs_factory()) - c['builders'].append(history_factory()) - c['builders'].append(bip39_factory()) - - c['services'].append(SlackStatusPush( - name="slack_status_immae_eu_project", - builders=[ - "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build", - "GSMCells_build", "Recettes_build", "History_build", - "BIP39_build" - ], - serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) - c['services'].append(XMPPStatusPush( - name="xmpp_status_immae_eu_project", - builders=[ - "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build", - "GSMCells_build", "Recettes_build", "History_build", - "BIP39_build" - ], - recipients=E.XMPP_RECIPIENTS, - password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip())) - -def history_factory(): - path_env = { - "PATH": os.environ["BUILDBOT_PATH_History"] + ":${PATH}" - } - factory = util.BuildFactory() - factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL, - submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - env=path_env, command=["jekyll", "build"])) - factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/_site", - masterdest=E.HISTORY_RELEASE_PATH, - url="https://www.immae.eu/history")) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.HISTORY_RELEASE_PATH))) - - return util.BuilderConfig(name="History_build", workernames=["generic-worker-immae-eu"], factory=factory) - -def docs_factory(): - path_env = { - "PATH": os.environ["BUILDBOT_PATH_Docs"] + ":${PATH}" - } - factory = util.BuildFactory() - factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL, - submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - env=path_env, command=["make", "clean", "html"])) - factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/_build/html", - masterdest=E.DOCS_RELEASE_PATH, - url="https://www.immae.eu/docs")) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.DOCS_RELEASE_PATH))) - - return util.BuilderConfig(name="Docs_build", workernames=["generic-worker-immae-eu"], factory=factory) - -def recettes_factory(): - path_env = { - "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}" - } - factory = util.BuildFactory() - factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL, - submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(NixShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - env=path_env, command="jekyll build --trace --baseurl /recettes")) - factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RECETTES_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/_site", - masterdest=E.RECETTES_RELEASE_PATH, - url="https://www.immae.eu/recettes")) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RECETTES_RELEASE_PATH))) - - return util.BuilderConfig(name="Recettes_build", workernames=["generic-worker-immae-eu"], factory=factory) - -def bip39_factory(): - path_env = { - "PATH": os.environ["BUILDBOT_PATH_BIP39"] + ":${PATH}" - } - factory = util.BuildFactory() - factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL, - submodules=True, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build file", - logEnviron=False, haltOnFailure=True, workdir="source", - env=path_env, command=["python", "compile.py"])) - factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html", - workdir="source", masterdest=E.BIP39_RELEASE_PATH + "/index.html", - url="https://tools.immae.eu/BIP39", mode=0o644)) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.BIP39_RELEASE_PATH))) - - return util.BuilderConfig(name="BIP39_build", workernames=["generic-worker-immae-eu"], factory=factory) - -def immae_eu_factory(): - path_env = { - "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}" - } - factory = util.BuildFactory() - factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, - submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - env=path_env, command=["make", "html"])) - factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/output", - masterdest=E.IMMAE_EU_RELEASE_PATH, - url="https://www.immae.eu")) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH))) - - return util.BuilderConfig(name="ImmaeEu_build", workernames=["generic-worker-immae-eu"], factory=factory) - -def cours_factory(): - path_env = { - "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}", - "CI": "yes" - } - factory = util.BuildFactory() - factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, - submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "build"], env=path_env)) - factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/build", - masterdest=E.COURS_RELEASE_PATH, - url="https://www.immae.eu/cours")) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) - - factory.addStep(steps.ShellCommand(name="build pdfs", - logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "pdfs"], env=path_env)) - - package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) - release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" - package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch) - package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch) - factory.addStep(steps.ShellCommand(name="build pdf tarball", - logEnviron=False, haltOnFailure=True, workdir="source", - command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env)) - factory.addStep(steps.FileUpload(name="upload package", workersrc=package, - workdir="source", masterdest=package_dest, - url=package_url, mode=0o644)) - - return util.BuilderConfig(name="Cours_build", workernames=["generic-worker-immae-eu"], factory=factory) - -def normalesup_factory(): - path_env = { - "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}" - } - factory = util.BuildFactory() - factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, - submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), - sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) - factory.addStep(steps.ShellCommand(name="build website", - logEnviron=False, haltOnFailure=True, workdir="source", - command=["make", "build"], env=path_env)) - factory.addStep(steps.ShellCommand(name="give read access to all files", - logEnviron=False, haltOnFailure=True, workdir="source", - command="chmod -R a+rX build", env=path_env)) - factory.addStep(steps.ShellCommand(name="synchronize with phare", - logEnviron=False, haltOnFailure=True, workdir="source", - env=path_env, command=[ - "rsync", "-av", "--delete", - "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH), - "build/", - os.environ["BUILDBOT_NORMALESUP_HOST"] - ])) - factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH))) - factory.addStep(steps.DirectoryUpload(workersrc="../source/build", masterdest=E.NORMALESUP_RELEASE_PATH, - url="https://www.immae.eu/recherche")) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH))) - - return util.BuilderConfig(name="Normalesup_build", workernames=["generic-worker-immae-eu"], factory=factory) - -def gsm_cells_factory(): - path_env = { - "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}", - "IN_BUILDBOT": "yes", - } - master_env = { - "HTACCESS": ''' - Options +FollowSymLinks - IndexIgnore * - ''' - } - for k, v in os.environ.items(): - if k.startswith("BUILDBOT_GSM_CELLS_"): - path_env[k[len("BUILDBOT_GSM_CELLS_"):]] = v - - script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" - factory = util.BuildFactory() - factory.addStep(steps.ShellCommand(name="download files", - logEnviron=False, haltOnFailure=True, command=[script], env=path_env)) - factory.addStep(steps.ShellCommand(name="give read access to all files", - logEnviron=False, haltOnFailure=True, - command="chmod a+r lacells.db", env=path_env)) - factory.addStep(steps.FileUpload(workersrc="lacells.db", - masterdest=(E.GSMCELLS_RELEASE_PATH+"/lacells.db"), url=(E.GSMCELLS_RELEASE_URL+"/lacells.db"))) - factory.addStep(steps.MasterShellCommand(command="touch {}/.duplicity-ignore".format(E.GSMCELLS_RELEASE_PATH))) - factory.addStep(steps.MasterShellCommand(command='echo "$HTACCESS" > {}/.htaccess'.format(E.GSMCELLS_RELEASE_PATH), - env=master_env)) - factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH))) - factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH))) - - return util.BuilderConfig(name="GSMCells_build", workernames=["generic-worker-immae-eu"], factory=factory)