]>
git.immae.eu Git - perso/Immae/Config/Nix.git/blob - modules/private/buildbot/projects/immaeEu/__init__.py
1 from buildbot
.plugins
import *
2 from buildbot_common
.build_helpers
import *
4 from buildbot
.util
import bytes2unicode
7 __all__
= [ "configure", "E" ]
11 BUILDBOT_URL
= "https://git.immae.eu/buildbot/{}/".format(PROJECT
)
12 SOCKET
= "unix:/run/buildbot/{}.sock".format(PROJECT
)
13 PB_SOCKET
= "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT
)
14 SSH_KEY_PATH
= "/var/lib/buildbot/buildbot_key"
15 SSH_HOST_KEY
= "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF"
16 XMPP_RECIPIENTS
= os
.environ
["BUILDBOT_XMPP_RECIPIENTS"].split(" ")
18 BIP39_GIT_URL
= "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git"
19 IMMAE_EU_GIT_URL
= "gitolite@git.immae.eu:perso/Immae/Sites/Blog"
20 HISTORY_GIT_URL
= "gitolite@git.immae.eu:perso/Immae/Sites/History"
21 RECETTES_GIT_URL
= "gitolite@git.immae.eu:perso/Immae/Sites/Recettes"
22 COURS_GIT_URL
= "gitolite@git.immae.eu:perso/Immae/Sites/Cours"
23 DOCS_GIT_URL
= "gitolite@git.immae.eu:perso/Immae/Sites/Docs"
24 NORMALESUP_GIT_URL
= "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup"
26 COURS_RELEASE_PATH
= "/var/lib/buildbot/outputs/immae/cours"
27 COURS_TARBALL_PATH
= "/var/lib/ftp/release.immae.eu/cours"
28 COURS_TARBALL_URL
= "https://release.immae.eu/cours"
29 BIP39_RELEASE_PATH
= "/var/lib/buildbot/outputs/immae/bip39"
30 HISTORY_RELEASE_PATH
= "/var/lib/buildbot/outputs/immae/history"
31 IMMAE_EU_RELEASE_PATH
= "/var/lib/buildbot/outputs/immae/blog"
32 DOCS_RELEASE_PATH
= "/var/lib/buildbot/outputs/immae/docs"
33 RECETTES_RELEASE_PATH
= "/var/lib/buildbot/outputs/immae/recettes"
34 NORMALESUP_RELEASE_PATH
= "/var/lib/buildbot/outputs/immae/recherche"
35 GSMCELLS_RELEASE_PATH
= "/var/lib/ftp/release.immae.eu/gsm_cells"
36 GSMCELLS_RELEASE_URL
= "https://release.immae.eu/gsm_cells"
39 SECRETS_FILE
= os
.getcwd() + "/secrets"
40 LDAP_URL
= "ldaps://ldap.immae.eu:636"
41 LDAP_ADMIN_USER
= "cn=buildbot,ou=services,dc=immae,dc=eu"
42 LDAP_BASE
= "dc=immae,dc=eu"
43 LDAP_PATTERN
= "(uid=%(username)s)"
44 LDAP_GROUP_PATTERN
= "(&(memberOf=cn=groups,ou=immaeEu,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))"
45 TITLE_URL
= "https://www.immae.eu"
46 TITLE
= "Immae website"
48 class CustomBase(webhooks
.base
):
49 def getChanges(self
, request
):
51 content
= request
.content
.read()
52 args
= json
.loads(bytes2unicode(content
))
53 except Exception as e
:
54 raise ValueError("Error loading JSON: " + str(e
))
56 args
.setdefault("comments", "")
57 args
.setdefault("repository", "")
58 args
.setdefault("author", args
.get("who", "unknown"))
63 c
["buildbotURL"] = E
.BUILDBOT_URL
64 c
["www"]["port"] = E
.SOCKET
66 c
["www"]["change_hook_dialects"]["base"] = {
67 "custom_class": CustomBase
69 c
['workers'].append(worker
.LocalWorker("generic-worker-immae-eu"))
71 c
['schedulers'].append(hook_scheduler("ImmaeEu", timer
=1))
72 c
['schedulers'].append(hook_scheduler("Normalesup", timer
=1))
73 c
['schedulers'].append(hook_scheduler("Cours", timer
=1))
74 c
['schedulers'].append(hook_scheduler("Recettes", timer
=1))
75 c
['schedulers'].append(hook_scheduler("Docs", timer
=1))
76 c
['schedulers'].append(hook_scheduler("History", timer
=1))
77 c
['schedulers'].append(hook_scheduler("BIP39", timer
=1))
78 c
['schedulers'].append(schedulers
.Nightly(name
="GSMCells-weekly",
79 builderNames
=["GSMCells_build"], dayOfWeek
=6, hour
=3))
80 c
['schedulers'].append(force_scheduler("force_immae_eu", [
81 "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build",
82 "Recettes_build", "History_build", "BIP39_build"
84 c
['schedulers'].append(schedulers
.ForceScheduler(
85 name
="GSMCells-force", label
="Force build",
86 buttonName
="Force build",
87 reason
=util
.StringParameter(name
="reason", label
="Reason", default
="Force build"),
89 util
.CodebaseParameter("",
90 branch
=util
.FixedParameter(name
="branch", default
=""),
91 revision
=util
.FixedParameter(name
="revision", default
=""),
92 repository
=util
.FixedParameter(name
="repository", default
=""),
93 project
=util
.FixedParameter(name
="project", default
=""),
96 username
=util
.FixedParameter(name
="username", default
="Web button"),
97 builderNames
=["GSMCells_build"]
100 c
['builders'].append(immae_eu_factory())
101 c
['builders'].append(normalesup_factory())
102 c
['builders'].append(cours_factory())
103 c
['builders'].append(gsm_cells_factory())
104 c
['builders'].append(recettes_factory())
105 c
['builders'].append(docs_factory())
106 c
['builders'].append(history_factory())
107 c
['builders'].append(bip39_factory())
109 c
['services'].append(SlackStatusPush(
110 name
="slack_status_immae_eu_project",
112 "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build",
113 "GSMCells_build", "Recettes_build", "History_build",
116 serverUrl
=open(E
.SECRETS_FILE
+ "/slack_webhook", "r").read().rstrip()))
117 c
['services'].append(XMPPStatusPush(
118 name
="xmpp_status_immae_eu_project",
120 "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build",
121 "GSMCells_build", "Recettes_build", "History_build",
124 recipients
=E
.XMPP_RECIPIENTS
,
125 password
=open(E
.SECRETS_FILE
+ "/notify_xmpp_password", "r").read().rstrip()))
127 def history_factory():
129 "PATH": os
.environ
["BUILDBOT_PATH_History"] + ":${PATH}"
131 factory
= util
.BuildFactory()
132 factory
.addStep(steps
.Git(logEnviron
=False, repourl
=E
.HISTORY_GIT_URL
,
133 submodules
=True, sshPrivateKey
=open(E
.SSH_KEY_PATH
).read().rstrip(),
134 sshHostKey
=E
.SSH_HOST_KEY
, mode
="full", method
="copy"))
135 factory
.addStep(steps
.ShellCommand(name
="build website",
136 logEnviron
=False, haltOnFailure
=True, workdir
="source",
137 env
=path_env
, command
=["jekyll", "build"]))
138 factory
.addStep(steps
.MasterShellCommand(command
="rm -rf {}".format(E
.HISTORY_RELEASE_PATH
)))
139 factory
.addStep(steps
.DirectoryUpload(workersrc
="../source/_site",
140 masterdest
=E
.HISTORY_RELEASE_PATH
,
141 url
="https://www.immae.eu/history"))
142 factory
.addStep(steps
.MasterShellCommand(command
="chmod -R a+rX {}".format(E
.HISTORY_RELEASE_PATH
)))
144 return util
.BuilderConfig(name
="History_build", workernames
=["generic-worker-immae-eu"], factory
=factory
)
148 "PATH": os
.environ
["BUILDBOT_PATH_Docs"] + ":${PATH}"
150 factory
= util
.BuildFactory()
151 factory
.addStep(steps
.Git(logEnviron
=False, repourl
=E
.DOCS_GIT_URL
,
152 submodules
=True, sshPrivateKey
=open(E
.SSH_KEY_PATH
).read().rstrip(),
153 sshHostKey
=E
.SSH_HOST_KEY
, mode
="full", method
="copy"))
154 factory
.addStep(steps
.ShellCommand(name
="build website",
155 logEnviron
=False, haltOnFailure
=True, workdir
="source",
156 env
=path_env
, command
=["make", "clean", "html"]))
157 factory
.addStep(steps
.MasterShellCommand(command
="rm -rf {}".format(E
.DOCS_RELEASE_PATH
)))
158 factory
.addStep(steps
.DirectoryUpload(workersrc
="../source/_build/html",
159 masterdest
=E
.DOCS_RELEASE_PATH
,
160 url
="https://www.immae.eu/docs"))
161 factory
.addStep(steps
.MasterShellCommand(command
="chmod -R a+rX {}".format(E
.DOCS_RELEASE_PATH
)))
163 return util
.BuilderConfig(name
="Docs_build", workernames
=["generic-worker-immae-eu"], factory
=factory
)
165 def recettes_factory():
167 "PATH": os
.environ
["BUILDBOT_PATH_Recettes"] + ":${PATH}"
169 factory
= util
.BuildFactory()
170 factory
.addStep(steps
.Git(logEnviron
=False, repourl
=E
.RECETTES_GIT_URL
,
171 submodules
=True, sshPrivateKey
=open(E
.SSH_KEY_PATH
).read().rstrip(),
172 sshHostKey
=E
.SSH_HOST_KEY
, mode
="full", method
="copy"))
173 factory
.addStep(NixShellCommand(name
="build website",
174 logEnviron
=False, haltOnFailure
=True, workdir
="source",
175 env
=path_env
, command
="jekyll build --trace --baseurl /recettes"))
176 factory
.addStep(steps
.MasterShellCommand(command
="rm -rf {}".format(E
.RECETTES_RELEASE_PATH
)))
177 factory
.addStep(steps
.DirectoryUpload(workersrc
="../source/_site",
178 masterdest
=E
.RECETTES_RELEASE_PATH
,
179 url
="https://www.immae.eu/recettes"))
180 factory
.addStep(steps
.MasterShellCommand(command
="chmod -R a+rX {}".format(E
.RECETTES_RELEASE_PATH
)))
182 return util
.BuilderConfig(name
="Recettes_build", workernames
=["generic-worker-immae-eu"], factory
=factory
)
186 "PATH": os
.environ
["BUILDBOT_PATH_BIP39"] + ":${PATH}"
188 factory
= util
.BuildFactory()
189 factory
.addStep(steps
.Git(logEnviron
=False, repourl
=E
.BIP39_GIT_URL
,
190 submodules
=True, mode
="full", method
="copy"))
191 factory
.addStep(steps
.ShellCommand(name
="build file",
192 logEnviron
=False, haltOnFailure
=True, workdir
="source",
193 env
=path_env
, command
=["python", "compile.py"]))
194 factory
.addStep(steps
.FileUpload(name
="upload file", workersrc
="bip39-standalone.html",
195 workdir
="source", masterdest
=E
.BIP39_RELEASE_PATH
+ "/index.html",
196 url
="https://tools.immae.eu/BIP39", mode
=0o644))
197 factory
.addStep(steps
.MasterShellCommand(command
="chmod -R a+rX {}".format(E
.BIP39_RELEASE_PATH
)))
199 return util
.BuilderConfig(name
="BIP39_build", workernames
=["generic-worker-immae-eu"], factory
=factory
)
201 def immae_eu_factory():
203 "PATH": os
.environ
["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}"
205 factory
= util
.BuildFactory()
206 factory
.addStep(steps
.Git(logEnviron
=False, repourl
=E
.IMMAE_EU_GIT_URL
,
207 submodules
=True, sshPrivateKey
=open(E
.SSH_KEY_PATH
).read().rstrip(),
208 sshHostKey
=E
.SSH_HOST_KEY
, mode
="full", method
="copy"))
209 factory
.addStep(steps
.ShellCommand(name
="build website",
210 logEnviron
=False, haltOnFailure
=True, workdir
="source",
211 env
=path_env
, command
=["make", "html"]))
212 factory
.addStep(steps
.MasterShellCommand(command
="rm -rf {}".format(E
.IMMAE_EU_RELEASE_PATH
)))
213 factory
.addStep(steps
.DirectoryUpload(workersrc
="../source/output",
214 masterdest
=E
.IMMAE_EU_RELEASE_PATH
,
215 url
="https://www.immae.eu"))
216 factory
.addStep(steps
.MasterShellCommand(command
="chmod -R a+rX {}".format(E
.IMMAE_EU_RELEASE_PATH
)))
218 return util
.BuilderConfig(name
="ImmaeEu_build", workernames
=["generic-worker-immae-eu"], factory
=factory
)
222 "PATH": os
.environ
["BUILDBOT_PATH_Cours"] + ":${PATH}",
225 factory
= util
.BuildFactory()
226 factory
.addStep(steps
.Git(logEnviron
=False, repourl
=E
.COURS_GIT_URL
,
227 submodules
=True, sshPrivateKey
=open(E
.SSH_KEY_PATH
).read().rstrip(),
228 sshHostKey
=E
.SSH_HOST_KEY
, mode
="full", method
="copy"))
229 factory
.addStep(steps
.ShellCommand(name
="build website",
230 logEnviron
=False, haltOnFailure
=True, workdir
="source",
231 command
=["make", "build"], env
=path_env
))
232 factory
.addStep(steps
.MasterShellCommand(command
="rm -rf {}".format(E
.COURS_RELEASE_PATH
)))
233 factory
.addStep(steps
.DirectoryUpload(workersrc
="../source/build",
234 masterdest
=E
.COURS_RELEASE_PATH
,
235 url
="https://www.immae.eu/cours"))
236 factory
.addStep(steps
.MasterShellCommand(command
="chmod -R a+rX {}".format(E
.COURS_RELEASE_PATH
)))
238 factory
.addStep(steps
.ShellCommand(name
="build pdfs",
239 logEnviron
=False, haltOnFailure
=True, workdir
="source",
240 command
=["make", "pdfs"], env
=path_env
))
242 package
= util
.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch
=clean_branch
)
243 release_file
= "{0}/cours_%(kw:clean_branch)s.tar.gz"
244 package_dest
= util
.Interpolate(release_file
.format(E
.COURS_TARBALL_PATH
), clean_branch
=clean_branch
)
245 package_url
= util
.Interpolate(release_file
.format(E
.COURS_TARBALL_URL
), clean_branch
=clean_branch
)
246 factory
.addStep(steps
.ShellCommand(name
="build pdf tarball",
247 logEnviron
=False, haltOnFailure
=True, workdir
="source",
248 command
=["tar", "-cvf", package
, "-C", "pdfs", "mp", "mpsi"], env
=path_env
))
249 factory
.addStep(steps
.FileUpload(name
="upload package", workersrc
=package
,
250 workdir
="source", masterdest
=package_dest
,
251 url
=package_url
, mode
=0o644))
253 return util
.BuilderConfig(name
="Cours_build", workernames
=["generic-worker-immae-eu"], factory
=factory
)
255 def normalesup_factory():
257 "PATH": os
.environ
["BUILDBOT_PATH_Normalesup"] + ":${PATH}"
259 factory
= util
.BuildFactory()
260 factory
.addStep(steps
.Git(logEnviron
=False, repourl
=E
.NORMALESUP_GIT_URL
,
261 submodules
=True, sshPrivateKey
=open(E
.SSH_KEY_PATH
).read().rstrip(),
262 sshHostKey
=E
.SSH_HOST_KEY
, mode
="full", method
="copy"))
263 factory
.addStep(steps
.ShellCommand(name
="build website",
264 logEnviron
=False, haltOnFailure
=True, workdir
="source",
265 command
=["make", "build"], env
=path_env
))
266 factory
.addStep(steps
.ShellCommand(name
="give read access to all files",
267 logEnviron
=False, haltOnFailure
=True, workdir
="source",
268 command
="chmod -R a+rX build", env
=path_env
))
269 factory
.addStep(steps
.ShellCommand(name
="synchronize with phare",
270 logEnviron
=False, haltOnFailure
=True, workdir
="source",
271 env
=path_env
, command
=[
272 "rsync", "-av", "--delete",
273 "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E
.SSH_KEY_PATH
),
275 os
.environ
["BUILDBOT_NORMALESUP_HOST"]
277 factory
.addStep(steps
.MasterShellCommand(command
="rm -rf {}".format(E
.NORMALESUP_RELEASE_PATH
)))
278 factory
.addStep(steps
.DirectoryUpload(workersrc
="../source/build", masterdest
=E
.NORMALESUP_RELEASE_PATH
,
279 url
="https://www.immae.eu/recherche"))
280 factory
.addStep(steps
.MasterShellCommand(command
="chmod -R a+rX {}".format(E
.NORMALESUP_RELEASE_PATH
)))
282 return util
.BuilderConfig(name
="Normalesup_build", workernames
=["generic-worker-immae-eu"], factory
=factory
)
284 def gsm_cells_factory():
286 "PATH": os
.environ
["BUILDBOT_PATH_GSMCells"] + ":${PATH}",
287 "IN_BUILDBOT": "yes",
291 Options +FollowSymLinks
295 for k
, v
in os
.environ
.items():
296 if k
.startswith("BUILDBOT_GSM_CELLS_"):
297 path_env
[k
[len("BUILDBOT_GSM_CELLS_"):]] = v
299 script
= os
.environ
["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download"
300 factory
= util
.BuildFactory()
301 factory
.addStep(steps
.ShellCommand(name
="download files",
302 logEnviron
=False, haltOnFailure
=True, command
=[script
], env
=path_env
))
303 factory
.addStep(steps
.ShellCommand(name
="give read access to all files",
304 logEnviron
=False, haltOnFailure
=True,
305 command
="chmod a+r lacells.db", env
=path_env
))
306 factory
.addStep(steps
.FileUpload(workersrc
="lacells.db",
307 masterdest
=(E
.GSMCELLS_RELEASE_PATH
+"/lacells.db"), url
=(E
.GSMCELLS_RELEASE_URL
+"/lacells.db")))
308 factory
.addStep(steps
.MasterShellCommand(command
="touch {}/.duplicity-ignore".format(E
.GSMCELLS_RELEASE_PATH
)))
309 factory
.addStep(steps
.MasterShellCommand(command
='echo "$HTACCESS" > {}/.htaccess'.format(E
.GSMCELLS_RELEASE_PATH
),
311 factory
.addStep(steps
.MasterShellCommand(command
="ln -sf lacells.db {}/lacells.db.new".format(E
.GSMCELLS_RELEASE_PATH
)))
312 factory
.addStep(steps
.MasterShellCommand(command
="chmod -R a+rX {}".format(E
.GSMCELLS_RELEASE_PATH
)))
314 return util
.BuilderConfig(name
="GSMCells_build", workernames
=["generic-worker-immae-eu"], factory
=factory
)