[perso/Immae/Config/Nix.git] / modules / private / buildbot / projects / immaeEu / __init__.py

from buildbot.plugins import *
from buildbot_common.build_helpers import *
import os
from buildbot.util import bytes2unicode
import json
from functools import partial

__all__ = [ "configure", "E" ]

class E():
    PROJECT       = "immaeEu"
    BUILDBOT_URL  = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
    SOCKET        = "unix:/run/buildbot/{}.sock".format(PROJECT)
    PB_SOCKET     = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT)
    SSH_KEY_PATH  = "/var/lib/buildbot/buildbot_key"
    SSH_HOST_PATH = "/var/lib/buildbot/buildbot_hosts"
    SSH_HOST_KEY  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF"
    XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ")

    BIP39_GIT_URL      = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git"
    IMMAE_EU_GIT_URL   = "gitolite@git.immae.eu:perso/Immae/Sites/Blog"
    HISTORY_GIT_URL    = "gitolite@git.immae.eu:perso/Immae/Sites/History"
    RECETTES_GIT_URL   = "gitolite@git.immae.eu:perso/Immae/Sites/Recettes"
    COURS_GIT_URL      = "gitolite@git.immae.eu:perso/Immae/Sites/Cours"
    DOCS_GIT_URL       = "gitolite@git.immae.eu:perso/Immae/Sites/Docs"
    NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup"

    COURS_RELEASE_PATH      = "/var/lib/buildbot/outputs/immae/cours"
    COURS_TARBALL_PATH      = "/var/lib/ftp/release.immae.eu/cours"
    COURS_TARBALL_URL       = "https://release.immae.eu/cours"
    BIP39_RELEASE_PATH      = "/var/lib/buildbot/outputs/immae/bip39"
    HISTORY_RELEASE_PATH    = "/var/lib/buildbot/outputs/immae/history"
    IMMAE_EU_RELEASE_PATH   = "/var/lib/buildbot/outputs/immae/blog"
    DOCS_RELEASE_PATH       = "/var/lib/buildbot/outputs/immae/docs"
    RECETTES_RELEASE_PATH   = "/var/lib/buildbot/outputs/immae/recettes"
    NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recherche"
    GSMCELLS_RELEASE_PATH   = "/var/lib/ftp/release.immae.eu/gsm_cells"
    GSMCELLS_RELEASE_URL    = "https://release.immae.eu/gsm_cells"

    # master.cfg
    SECRETS_FILE       = os.getcwd() + "/secrets"
    LDAP_URL           = "ldaps://ldap.immae.eu:636"
    LDAP_ADMIN_USER    = "cn=buildbot,ou=services,dc=immae,dc=eu"
    LDAP_BASE          = "dc=immae,dc=eu"
    LDAP_PATTERN       = "(uid=%(username)s)"
    LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=immaeEu,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))"
    TITLE_URL          = "https://www.immae.eu"
    TITLE              = "Immae website"

def configure(c):
    c["buildbotURL"] = E.BUILDBOT_URL
    c["www"]["port"] = E.SOCKET

    worker_name = "generic-worker-immae-eu"
    c['workers'].append(worker.LocalWorker(worker_name))

    withbranch = []
    withoutbranch = []

    withoutbranch.append(configure_gsm_cells(c, worker_name))

    withbranch.append(_configure("immae_eu", immae_eu_build_factory, c, worker_name))
    withbranch.append(_configure("normalesup", normalesup_build_factory, c, worker_name))
    withbranch.append(_configure("cours", cours_build_factory, c, worker_name))
    withbranch.append(_configure("recettes", recettes_build_factory, c, worker_name))
    withbranch.append(_configure("docs", docs_build_factory, c, worker_name))
    withbranch.append(_configure("history", history_build_factory, c, worker_name))
    withbranch.append(_configure("bip39", bip39_build_factory, c, worker_name))

    withbranch.append(_configure_symfony("Chloe", c, worker_name,
        "gitolite@git.immae.eu:perso/Immae/Sites/Chloe/New"))
    withbranch.append(_configure_symfony("Florian", c, worker_name,
        "gitolite@git.immae.eu:perso/florian_telles/stabilo"))
    withbranch.append(configure_symfony_isabelle_aten(c, worker_name))
    withbranch.append(_configure_symfony("Ludivine", c, worker_name,
        "gitolite@git.immae.eu:perso/Immae/Sites/Ludivine"))
    withbranch.append(_configure_symfony("Connexionswing", c, worker_name,
        "gitolite@git.immae.eu:perso/Immae/Projets/Connexionswing"))
    withbranch.append(_configure_symfony("Piedsjaloux", c, worker_name,
        "gitolite@git.immae.eu:Pieds_jaloux/NewSite"))

    c['schedulers'].append(force_scheduler("force_immae_eu", withbranch))
    c['schedulers'].append(force_scheduler("force_no_branch_immae_eu", withoutbranch, nobranch=True))

    configure_apprise_push(c, E.SECRETS_FILE, all_builder_names(c))
    configure_xmpp_push(c, E.SECRETS_FILE, all_builder_names(c), E.XMPP_RECIPIENTS)

BRANCH_TO_SYMFONY_ENV = {
        "Ludivine": {
            "master": "prod",
            "test": "dev",
            },
        "Connexionswing": {
            "master": "prod",
            "test": "dev",
            },
        "Piedsjaloux": {
            "master": "prod",
            "test": "dev",
            }
        }

BRANCH_TO_SERVICE_NAME = {
        "Chloe": {
            "test": "phpfpm-chloe_new_integration",
            #"master": "phpfpm-chloe_production",
            },
        "Florian": {
            "stabilo_dev": "phpfpm-florian_app",
            },
        "IsabelleAten": {
            "test": "phpfpm-isabelle_aten_integration",
            "master": "phpfpm-isabelle_aten_production",
            },
        "Ludivine": {
            "test": "phpfpm-ludivine_integration",
            "master": "phpfpm-ludivine_production",
            },
        "Connexionswing": {
            "test": "phpfpm-connexionswing_integration",
            "master": "phpfpm-connexionswing_production",
            },
        "Piedsjaloux": {
            "test": "phpfpm-piedsjaloux_integration",
            "master": "phpfpm-piedsjaloux_production",
            },
        }

BRANCH_TO_POST_STEP = {
        "Connexionswing": {
            "master": "SYMFONY_ENV=prod php ./bin/console assetic:dump --env=prod --no-debug"
            },
        }

def need_follow_systemd(name, step):
    return step.getProperty("branch") in BRANCH_TO_SERVICE_NAME.get(name, {})

def need_post_step(name, step):
    return step.getProperty("branch") in BRANCH_TO_POST_STEP.get(name, {})

@util.renderer
def get_post_step_command(props, name):
    if props.hasProperty("branch") and len(props["branch"]) > 0:
        post_step = BRANCH_TO_POST_STEP.get(name, {}).get(props["branch"])
        if post_step is not None:
            return post_step

@util.renderer
def get_parameters_file(props, name):
    if props.hasProperty("branch") and len(props["branch"]) > 0:
        env = BRANCH_TO_SYMFONY_ENV.get(name, {}).get(props["branch"], "dev")
    else:
        env = "dev"
    return E.SECRETS_FILE + "/symfony_{}_{}_parameters.yml".format(name, env)

@util.renderer
def get_composer_install_command(props, name):
    if props.hasProperty("branch") and len(props["branch"]) > 0:
        env = BRANCH_TO_SYMFONY_ENV.get(name, {}).get(props["branch"], "dev")
    else:
        env = "dev"

    return "SYMFONY_ENV={} composer install".format(env) + (" --no-dev" if env == "prod" else "")

@util.renderer
def get_systemd_service_invocation_command(props, name):
    if props.hasProperty("branch") and len(props["branch"]) > 0:
        service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"])
        if service is not None:
            return "ssh -o UserKnownHostsFile={0} -i {1} buildbot@eldiron systemctl show -p InvocationID --value {2}.service".format(E.SSH_HOST_PATH, E.SSH_KEY_PATH, service)

@util.renderer
def follow_systemd_command(props, name, invocation_id):
    if props.hasProperty("branch") and len(props["branch"]) > 0:
        service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"])
        if service is not None:
            return ["follow-systemd-unit", service, util.Property("service_invocation_id")]

def configure_gsm_cells(c, worker_name):
    builder_name = "GSMCells_build"
    c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly",
        builderNames=[builder_name], dayOfWeek=6, hour=3))
    c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=gsm_cells_build_factory()))

    return builder_name

def _configure(name, factory, c, worker_name):
    if name == "bip39":
        capitalized = "BIP39"
    else:
        capitalized = "".join([n.capitalize() for n in name.split('_')])
    builder_name = "{}_build".format(capitalized)

    c['schedulers'].append(git_hook_scheduler(capitalized, [builder_name]))
    c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=factory()))

    return builder_name

def configure_symfony_isabelle_aten(c, worker_name):
    return _configure_symfony("IsabelleAten", c, worker_name,
        "gitolite@git.immae.eu:perso/Immae/Sites/Aten", parameters_path=None,
        other_steps=lambda path_env : [
            NixShellCommand(name="Install yarn",
                logEnviron=False, haltOnFailure=True,
                env=path_env, command="yarn install"),
            NixShellCommand(name="Build frontend",
                logEnviron=False, haltOnFailure=True,
                env=path_env, command="yarn run encore production")
            ]
        )

def _configure_symfony(name, c, worker_name, *args, **kwargs):
    builder_name = "Symfony_{}_build".format(name)

    c['schedulers'].append(git_hook_scheduler("Symfony_{}".format(name), [builder_name]))
    c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=symfony_project_factory(name, *args, **kwargs)))

    return builder_name

def history_build_factory():
    path_env = {
            "PATH": "/run/current-system/sw/bin"
            }
    factory = util.BuildFactory()
    factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL,
        submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
        sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh"))
    factory.addStep(NixShellCommand(name="build website",
        logEnviron=False, haltOnFailure=True,
        env=path_env, command="jekyll build"))
    factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH)))
    factory.addStep(steps.DirectoryUpload(workersrc="_site",
        masterdest=E.HISTORY_RELEASE_PATH,
        url="https://www.immae.eu/history"))
    factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.HISTORY_RELEASE_PATH)))

    return factory

def docs_build_factory():
    path_env = {
            "PATH": "/run/current-system/sw/bin"
            }
    factory = util.BuildFactory()
    factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL,
        submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
        sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh"))
    factory.addStep(NixShellCommand(name="build website",
        logEnviron=False, haltOnFailure=True,
        env=path_env, command="make html"))
    factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH)))
    factory.addStep(steps.DirectoryUpload(workersrc="_build/html",
        masterdest=E.DOCS_RELEASE_PATH,
        url="https://www.immae.eu/docs"))
    factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.DOCS_RELEASE_PATH)))

    return factory

def recettes_build_factory():
    path_env = {
            "PATH": "/run/current-system/sw/bin"
            }
    factory = util.BuildFactory()
    factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL,
        submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
        sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh"))
    factory.addStep(NixShellCommand(name="build website",
        logEnviron=False, haltOnFailure=True,
        env=path_env, command="jekyll build --trace --baseurl /recettes"))
    factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RECETTES_RELEASE_PATH)))
    factory.addStep(steps.DirectoryUpload(workersrc="_site",
        masterdest=E.RECETTES_RELEASE_PATH,
        url="https://www.immae.eu/recettes"))
    factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RECETTES_RELEASE_PATH)))

    return factory

def bip39_build_factory():
    path_env = {
            "PATH": "/run/current-system/sw/bin",
            "NIX_PATH": "nixpkgs=channel:nixos-unstable",
            }
    factory = util.BuildFactory()
    factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL,
        submodules=True, mode="full", method="fresh"))
    factory.addStep(NixShellCommand(name="build file", nixPackages=["python3"],
        logEnviron=False, haltOnFailure=True, env=path_env, command="python compile.py"))
    factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html",
        masterdest=E.BIP39_RELEASE_PATH + "/index.html",
        url="https://tools.immae.eu/BIP39", mode=0o644))
    factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.BIP39_RELEASE_PATH)))

    return factory

def immae_eu_build_factory():
    path_env = {
            "PATH": "/run/current-system/sw/bin",
            }
    factory = util.BuildFactory()
    factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL,
        submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
        sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh"))
    factory.addStep(NixShellCommand(name="build website",
        logEnviron=False, haltOnFailure=True, pure=False,
        env=path_env, command="make html"))
    factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH)))
    factory.addStep(steps.DirectoryUpload(workersrc="output",
        masterdest=E.IMMAE_EU_RELEASE_PATH,
        url="https://www.immae.eu"))
    factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH)))

    return factory

def cours_build_factory():
    path_env = {
            "PATH": "/run/current-system/sw/bin",
            "CI": "yes"
            }
    factory = util.BuildFactory()
    factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL,
        submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
        sshHostKey=E.SSH_HOST_KEY, mode="incremental"))
    factory.addStep(NixShellCommand(name="build website",
        logEnviron=False, haltOnFailure=True, pure=True,
        command="make build", env=path_env))
    factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH)))
    factory.addStep(steps.DirectoryUpload(workersrc="build",
        masterdest=E.COURS_RELEASE_PATH,
        url="https://www.immae.eu/cours"))
    factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH)))

    factory.addStep(NixShellCommand(name="build pdfs",
        logEnviron=False, haltOnFailure=True, pure=True,
        command="make pdfs", env=path_env))

    package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch)
    release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz"
    package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch)
    package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch)
    factory.addStep(steps.ShellCommand(name="build pdf tarball",
        logEnviron=False, haltOnFailure=True,
        command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env))
    factory.addStep(steps.FileUpload(name="upload package", workersrc=package,
        masterdest=package_dest, url=package_url, mode=0o644))

    return factory

def normalesup_build_factory():
    path_env = {
            "PATH": "/run/current-system/sw/bin",
            "GIT_SSH_COMMAND": "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH),
            }
    factory = util.BuildFactory()
    factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL,
        submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
        sshHostKey=E.SSH_HOST_KEY, mode="incremental"))
    factory.addStep(NixShellCommand(name="build website",
        logEnviron=False, haltOnFailure=True, pure=False,
        command="make build", env=path_env))
    factory.addStep(steps.ShellCommand(name="give read access to all files",
        logEnviron=False, haltOnFailure=True,
        command="chmod -R a+rX build", env=path_env))
    factory.addStep(steps.ShellCommand(name="synchronize with phare",
        logEnviron=False, haltOnFailure=True,
        env=path_env, command=[
            "rsync", "-av", "--delete",
            "-e", "ssh -i {0} -o UserKnownHostsFile={1}".format(E.SSH_KEY_PATH, E.SSH_HOST_PATH),
            "build/",
            os.environ["BUILDBOT_NORMALESUP_HOST"]
            ]))
    factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH)))
    factory.addStep(steps.DirectoryUpload(workersrc="build", masterdest=E.NORMALESUP_RELEASE_PATH,
        url="https://www.immae.eu/recherche"))
    factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH)))

    return factory

def gsm_cells_build_factory():
    path_env = {
            "PATH": "/run/current-system/sw/bin",
            "NIX_PATH": "nixpkgs=channel:nixos-unstable",
            "IN_BUILDBOT": "yes",
            }
    master_env = {
            "HTACCESS": '''
            Options +FollowSymLinks
            IndexIgnore *
            '''
            }
    for k, v in os.environ.items():
        if k.startswith("BUILDBOT_GSM_CELLS_"):
            path_env[k[len("BUILDBOT_GSM_CELLS_"):]] = v

    script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download"
    factory = util.BuildFactory()
    factory.addStep(steps.FileDownload(mastersrc=script, workerdest="lacells_download", mode=0o755))
    factory.addStep(NixShellCommand(name="download files",
        logEnviron=False, haltOnFailure=True, command="./lacells_download",
        nixPackages=["sqlite", "wget", "gzip"], pure=False, env=path_env))
    factory.addStep(steps.ShellCommand(name="give read access to all files",
        logEnviron=False, haltOnFailure=True,
        command="chmod a+r lacells.db", env=path_env))
    factory.addStep(steps.FileUpload(workersrc="lacells.db",
        masterdest=(E.GSMCELLS_RELEASE_PATH+"/lacells.db"), url=(E.GSMCELLS_RELEASE_URL+"/lacells.db")))
    factory.addStep(steps.MasterShellCommand(command="touch {}/.duplicity-ignore".format(E.GSMCELLS_RELEASE_PATH)))
    factory.addStep(steps.MasterShellCommand(command='echo "$HTACCESS" > {}/.htaccess'.format(E.GSMCELLS_RELEASE_PATH),
        env=master_env))
    factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH)))
    factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH)))

    return factory

def symfony_project_factory(name, repourl, parameters_path="app/config/parameters.yml", other_steps=lambda a : []):
    master_path_env = {
            "PATH": os.environ["BUILDBOT_SYMFONY_MASTER_PATH"] + ":${PATH}"
            }
    path_env = {
            "PATH": "/run/current-system/sw/bin"
            }

    for k, v in os.environ.items():
        if k.startswith("BUILDBOT_SYMFONY_{}_".format(name)):
            path_env[k[len("BUILDBOT_SYMFONY_{}_".format(name)):]] = v

    factory = util.BuildFactory()
    factory.addStep(steps.Git(logEnviron=False, repourl=repourl,
        submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
        sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh"))
    if parameters_path is not None:
        factory.addStep(steps.FileDownload(mastersrc=get_parameters_file.withArgs(name),
                                           workerdest=parameters_path))
    factory.addStep(NixShellCommand(name="build website",
        logEnviron=False, haltOnFailure=True,
        env=path_env, command=get_composer_install_command.withArgs(name)))
    if name in BRANCH_TO_POST_STEP:
        factory.addStep(NixShellCommand(name="build website post",
            logEnviron=False, haltOnFailure=True, doStepIf=partial(need_post_step, name),
            env=path_env, command=get_post_step_command.withArgs(name)))
    if parameters_path is not None:
        factory.addStep(steps.ShellCommand(name="Remove parameters.yml",
            logEnviron=False, haltOnFailure=True,
            command="rm -f {}".format(parameters_path)))
    for step in other_steps(path_env):
        factory.addStep(step)
    package = util.Interpolate("{}_%(kw:clean_branch)s.tar.gz".format(name), clean_branch=clean_branch)
    release_file = "{0}/{1}/%(kw:clean_branch)s.tar.gz"
    package_dest = util.Interpolate(release_file.format("/var/lib/ftp/release.immae.eu/buildbot", name), clean_branch=clean_branch)
    # Tar doesn’t like creating the tarball in the same directory
    factory.addStep(steps.ShellCommand(name="Make a tarball 1/2",
        logEnviron=False, haltOnFailure=True, env=path_env,
        command=["touch", package]))
    factory.addStep(steps.ShellCommand(name="Make a tarball 2/2",
        logEnviron=False, haltOnFailure=True, env=path_env,
        command=["tar", "--exclude", package, "-czf", package, "."]))
    factory.addStep(steps.SetPropertyFromCommand(command=get_systemd_service_invocation_command.withArgs(name),
        property="service_invocation_id", doStepIf=partial(need_follow_systemd, name)))
    factory.addStep(steps.FileUpload(name="upload package",
        workersrc=package, masterdest=package_dest, mode=0o644))
    factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=master_path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name)))
    return factory