]>
Commit | Line | Data |
---|---|---|
8fa7ff2c IB |
1 | from buildbot.plugins import * |
2 | from buildbot_common.build_helpers import * | |
3 | import os | |
4 | from buildbot.util import bytes2unicode | |
5 | import json | |
f6e48e3a | 6 | from functools import partial |
8fa7ff2c IB |
7 | |
8 | __all__ = [ "configure", "E" ] | |
9 | ||
10 | class E(): | |
11 | PROJECT = "immaeEu" | |
12 | BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) | |
13 | SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) | |
14 | PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) | |
8fa7ff2c IB |
15 | SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" |
16 | SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" | |
17 | XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") | |
18 | ||
251c0a13 | 19 | BIP39_GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git" |
0012da0f | 20 | IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" |
577d00ee | 21 | HISTORY_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/History" |
f2a18fd6 | 22 | RECETTES_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Recettes" |
0012da0f | 23 | COURS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Cours" |
97f5a24b | 24 | DOCS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Docs" |
573e1710 IB |
25 | NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup" |
26 | ||
cb589b2e | 27 | COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/cours" |
0012da0f IB |
28 | COURS_TARBALL_PATH = "/var/lib/ftp/release.immae.eu/cours" |
29 | COURS_TARBALL_URL = "https://release.immae.eu/cours" | |
cb589b2e | 30 | BIP39_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/bip39" |
577d00ee | 31 | HISTORY_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/history" |
cb589b2e | 32 | IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/blog" |
97f5a24b | 33 | DOCS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/docs" |
cb589b2e IB |
34 | RECETTES_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recettes" |
35 | NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recherche" | |
60d328e2 IB |
36 | GSMCELLS_RELEASE_PATH = "/var/lib/ftp/release.immae.eu/gsm_cells" |
37 | GSMCELLS_RELEASE_URL = "https://release.immae.eu/gsm_cells" | |
0012da0f | 38 | |
8fa7ff2c IB |
39 | # master.cfg |
40 | SECRETS_FILE = os.getcwd() + "/secrets" | |
41 | LDAP_URL = "ldaps://ldap.immae.eu:636" | |
42 | LDAP_ADMIN_USER = "cn=buildbot,ou=services,dc=immae,dc=eu" | |
43 | LDAP_BASE = "dc=immae,dc=eu" | |
44 | LDAP_PATTERN = "(uid=%(username)s)" | |
45 | LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=immaeEu,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))" | |
46 | TITLE_URL = "https://www.immae.eu" | |
47 | TITLE = "Immae website" | |
48 | ||
8fa7ff2c IB |
49 | def configure(c): |
50 | c["buildbotURL"] = E.BUILDBOT_URL | |
51 | c["www"]["port"] = E.SOCKET | |
52 | ||
f6e48e3a IB |
53 | worker_name = "generic-worker-immae-eu" |
54 | c['workers'].append(worker.LocalWorker(worker_name)) | |
55 | ||
56 | withbranch = [] | |
57 | withoutbranch = [] | |
58 | ||
59 | withoutbranch.append(configure_gsm_cells(c, worker_name)) | |
60 | ||
61 | withbranch.append(_configure("immae_eu", immae_eu_build_factory, c, worker_name)) | |
62 | withbranch.append(_configure("normalesup", normalesup_build_factory, c, worker_name)) | |
63 | withbranch.append(_configure("cours", cours_build_factory, c, worker_name)) | |
64 | withbranch.append(_configure("recettes", recettes_build_factory, c, worker_name)) | |
65 | withbranch.append(_configure("docs", docs_build_factory, c, worker_name)) | |
66 | withbranch.append(_configure("history", history_build_factory, c, worker_name)) | |
67 | withbranch.append(_configure("bip39", bip39_build_factory, c, worker_name)) | |
68 | ||
69 | withbranch.append(_configure_symfony("Chloe", c, worker_name, | |
70 | "gitolite@git.immae.eu:perso/Immae/Sites/Chloe/New")) | |
71 | withbranch.append(_configure_symfony("Florian", c, worker_name, | |
72 | "gitolite@git.immae.eu:perso/florian_telles/stabilo")) | |
73 | withbranch.append(configure_symfony_isabelle_aten(c, worker_name)) | |
74 | withbranch.append(_configure_symfony("Ludivine", c, worker_name, | |
75 | "gitolite@git.immae.eu:perso/Immae/Sites/Ludivine")) | |
76 | withbranch.append(_configure_symfony("Connexionswing", c, worker_name, | |
77 | "gitolite@git.immae.eu:perso/Immae/Projets/Connexionswing")) | |
78 | withbranch.append(_configure_symfony("Piedsjaloux", c, worker_name, | |
79 | "gitolite@git.immae.eu:Pieds_jaloux/NewSite")) | |
80 | ||
81 | c['schedulers'].append(force_scheduler("force_immae_eu", withbranch)) | |
82 | c['schedulers'].append(force_scheduler("force_no_branch_immae_eu", withoutbranch, nobranch=True)) | |
83 | ||
84 | configure_slack_push(c, E.SECRETS_FILE, all_builder_names(c)) | |
85 | configure_xmpp_push(c, E.SECRETS_FILE, all_builder_names(c), E.XMPP_RECIPIENTS) | |
86 | ||
87 | BRANCH_TO_SYMFONY_ENV = { | |
88 | "Ludivine": { | |
89 | "master": "prod", | |
90 | "test": "dev", | |
91 | }, | |
92 | "Connexionswing": { | |
93 | "master": "prod", | |
94 | "test": "dev", | |
95 | }, | |
96 | "Piedsjaloux": { | |
97 | "master": "prod", | |
98 | "test": "dev", | |
8fa7ff2c | 99 | } |
f6e48e3a IB |
100 | } |
101 | ||
102 | BRANCH_TO_SERVICE_NAME = { | |
103 | "Chloe": { | |
104 | "test": "phpfpm-chloe_new_integration", | |
105 | #"master": "phpfpm-chloe_production", | |
106 | }, | |
107 | "Florian": { | |
108 | "stabilo_dev": "phpfpm-florian_app", | |
109 | }, | |
110 | "IsabelleAten": { | |
111 | "test": "phpfpm-isabelle_aten_integration", | |
112 | "master": "phpfpm-isabelle_aten_production", | |
113 | }, | |
114 | "Ludivine": { | |
115 | "test": "phpfpm-ludivine_integration", | |
116 | "master": "phpfpm-ludivine_production", | |
117 | }, | |
118 | "Connexionswing": { | |
119 | "test": "phpfpm-connexionswing_integration", | |
120 | "master": "phpfpm-connexionswing_production", | |
121 | }, | |
122 | "Piedsjaloux": { | |
123 | "test": "phpfpm-piedsjaloux_integration", | |
124 | "master": "phpfpm-piedsjaloux_production", | |
125 | }, | |
126 | } | |
127 | ||
128 | BRANCH_TO_POST_STEP = { | |
129 | "Connexionswing": { | |
130 | "master": "SYMFONY_ENV=prod php ./bin/console assetic:dump --env=prod --no-debug" | |
131 | }, | |
132 | } | |
133 | ||
134 | def need_follow_systemd(name, step): | |
135 | return step.getProperty("branch") in BRANCH_TO_SERVICE_NAME.get(name, {}) | |
136 | ||
137 | def need_post_step(name, step): | |
138 | return step.getProperty("branch") in BRANCH_TO_POST_STEP.get(name, {}) | |
139 | ||
140 | @util.renderer | |
141 | def get_post_step_command(props, name): | |
142 | if props.hasProperty("branch") and len(props["branch"]) > 0: | |
143 | post_step = BRANCH_TO_POST_STEP.get(name, {}).get(props["branch"]) | |
144 | if post_step is not None: | |
145 | return post_step | |
146 | ||
147 | @util.renderer | |
148 | def get_parameters_file(props, name): | |
149 | if props.hasProperty("branch") and len(props["branch"]) > 0: | |
150 | env = BRANCH_TO_SYMFONY_ENV.get(name, {}).get(props["branch"], "dev") | |
151 | else: | |
152 | env = "dev" | |
153 | return E.SECRETS_FILE + "/symfony_{}_{}_parameters.yml".format(name, env) | |
154 | ||
155 | @util.renderer | |
156 | def get_composer_install_command(props, name): | |
157 | if props.hasProperty("branch") and len(props["branch"]) > 0: | |
158 | env = BRANCH_TO_SYMFONY_ENV.get(name, {}).get(props["branch"], "dev") | |
159 | else: | |
160 | env = "dev" | |
161 | ||
162 | return "SYMFONY_ENV={} composer install".format(env) + (" --no-dev" if env == "prod" else "") | |
163 | ||
164 | @util.renderer | |
165 | def get_systemd_service_invocation_command(props, name): | |
166 | if props.hasProperty("branch") and len(props["branch"]) > 0: | |
167 | service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"]) | |
168 | if service is not None: | |
169 | return "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no -i {} buildbot@eldiron systemctl show -p InvocationID --value {}.service".format(E.SSH_KEY_PATH, service) | |
170 | ||
171 | @util.renderer | |
172 | def follow_systemd_command(props, name, invocation_id): | |
173 | if props.hasProperty("branch") and len(props["branch"]) > 0: | |
174 | service = BRANCH_TO_SERVICE_NAME.get(name, {}).get(props["branch"]) | |
175 | if service is not None: | |
176 | return ["follow-systemd-unit", service, util.Property("service_invocation_id")] | |
177 | ||
178 | def configure_gsm_cells(c, worker_name): | |
179 | builder_name = "GSMCells_build" | |
60d328e2 | 180 | c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly", |
f6e48e3a IB |
181 | builderNames=[builder_name], dayOfWeek=6, hour=3)) |
182 | c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=gsm_cells_build_factory())) | |
183 | ||
184 | return builder_name | |
185 | ||
186 | def _configure(name, factory, c, worker_name): | |
187 | if name == "bip39": | |
188 | capitalized = "BIP39" | |
189 | else: | |
190 | capitalized = "".join([n.capitalize() for n in name.split('_')]) | |
191 | builder_name = "{}_build".format(capitalized) | |
192 | ||
193 | c['schedulers'].append(git_hook_scheduler(capitalized, [builder_name])) | |
194 | c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=factory())) | |
195 | ||
196 | return builder_name | |
197 | ||
198 | def configure_symfony_isabelle_aten(c, worker_name): | |
199 | return _configure_symfony("IsabelleAten", c, worker_name, | |
200 | "gitolite@git.immae.eu:perso/Immae/Sites/Aten", parameters_path=None, | |
2ff9258e IB |
201 | other_steps=lambda path_env : [ |
202 | NixShellCommand(name="Install yarn", | |
f6e48e3a | 203 | logEnviron=False, haltOnFailure=True, |
2ff9258e IB |
204 | env=path_env, command="yarn install"), |
205 | NixShellCommand(name="Build frontend", | |
f6e48e3a | 206 | logEnviron=False, haltOnFailure=True, |
2ff9258e IB |
207 | env=path_env, command="yarn run encore production") |
208 | ] | |
f6e48e3a IB |
209 | ) |
210 | ||
211 | def _configure_symfony(name, c, worker_name, *args, **kwargs): | |
212 | builder_name = "Symfony_{}_build".format(name) | |
213 | ||
214 | c['schedulers'].append(git_hook_scheduler("Symfony_{}".format(name), [builder_name])) | |
215 | c['builders'].append(util.BuilderConfig(name=builder_name, workernames=[worker_name], factory=symfony_project_factory(name, *args, **kwargs))) | |
216 | ||
217 | return builder_name | |
218 | ||
219 | def history_build_factory(): | |
577d00ee IB |
220 | path_env = { |
221 | "PATH": os.environ["BUILDBOT_PATH_History"] + ":${PATH}" | |
222 | } | |
223 | factory = util.BuildFactory() | |
224 | factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL, | |
225 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
f6e48e3a | 226 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) |
577d00ee | 227 | factory.addStep(steps.ShellCommand(name="build website", |
f6e48e3a | 228 | logEnviron=False, haltOnFailure=True, |
577d00ee IB |
229 | env=path_env, command=["jekyll", "build"])) |
230 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH))) | |
f6e48e3a | 231 | factory.addStep(steps.DirectoryUpload(workersrc="_site", |
577d00ee IB |
232 | masterdest=E.HISTORY_RELEASE_PATH, |
233 | url="https://www.immae.eu/history")) | |
234 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.HISTORY_RELEASE_PATH))) | |
235 | ||
f6e48e3a | 236 | return factory |
577d00ee | 237 | |
f6e48e3a | 238 | def docs_build_factory(): |
97f5a24b IB |
239 | path_env = { |
240 | "PATH": os.environ["BUILDBOT_PATH_Docs"] + ":${PATH}" | |
241 | } | |
242 | factory = util.BuildFactory() | |
243 | factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL, | |
244 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
f6e48e3a | 245 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) |
97f5a24b | 246 | factory.addStep(steps.ShellCommand(name="build website", |
f6e48e3a IB |
247 | logEnviron=False, haltOnFailure=True, |
248 | env=path_env, command=["make", "html"])) | |
97f5a24b | 249 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH))) |
f6e48e3a | 250 | factory.addStep(steps.DirectoryUpload(workersrc="_build/html", |
97f5a24b IB |
251 | masterdest=E.DOCS_RELEASE_PATH, |
252 | url="https://www.immae.eu/docs")) | |
253 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.DOCS_RELEASE_PATH))) | |
254 | ||
f6e48e3a | 255 | return factory |
97f5a24b | 256 | |
f6e48e3a | 257 | def recettes_build_factory(): |
f2a18fd6 IB |
258 | path_env = { |
259 | "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}" | |
260 | } | |
261 | factory = util.BuildFactory() | |
262 | factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL, | |
263 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
f6e48e3a | 264 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) |
6b017278 | 265 | factory.addStep(NixShellCommand(name="build website", |
f6e48e3a | 266 | logEnviron=False, haltOnFailure=True, |
6b017278 | 267 | env=path_env, command="jekyll build --trace --baseurl /recettes")) |
f2a18fd6 | 268 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RECETTES_RELEASE_PATH))) |
f6e48e3a | 269 | factory.addStep(steps.DirectoryUpload(workersrc="_site", |
f2a18fd6 IB |
270 | masterdest=E.RECETTES_RELEASE_PATH, |
271 | url="https://www.immae.eu/recettes")) | |
272 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RECETTES_RELEASE_PATH))) | |
273 | ||
f6e48e3a | 274 | return factory |
f2a18fd6 | 275 | |
f6e48e3a | 276 | def bip39_build_factory(): |
251c0a13 IB |
277 | path_env = { |
278 | "PATH": os.environ["BUILDBOT_PATH_BIP39"] + ":${PATH}" | |
279 | } | |
280 | factory = util.BuildFactory() | |
281 | factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL, | |
f6e48e3a | 282 | submodules=True, mode="full", method="fresh")) |
251c0a13 | 283 | factory.addStep(steps.ShellCommand(name="build file", |
f6e48e3a | 284 | logEnviron=False, haltOnFailure=True, |
251c0a13 IB |
285 | env=path_env, command=["python", "compile.py"])) |
286 | factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html", | |
f6e48e3a | 287 | masterdest=E.BIP39_RELEASE_PATH + "/index.html", |
251c0a13 IB |
288 | url="https://tools.immae.eu/BIP39", mode=0o644)) |
289 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.BIP39_RELEASE_PATH))) | |
290 | ||
f6e48e3a | 291 | return factory |
251c0a13 | 292 | |
f6e48e3a | 293 | def immae_eu_build_factory(): |
0012da0f IB |
294 | path_env = { |
295 | "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}" | |
296 | } | |
8fa7ff2c | 297 | factory = util.BuildFactory() |
573e1710 | 298 | factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, |
f78fbfbf | 299 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), |
f6e48e3a | 300 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) |
8fa7ff2c | 301 | factory.addStep(steps.ShellCommand(name="build website", |
f6e48e3a | 302 | logEnviron=False, haltOnFailure=True, |
0012da0f IB |
303 | env=path_env, command=["make", "html"])) |
304 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) | |
f6e48e3a | 305 | factory.addStep(steps.DirectoryUpload(workersrc="output", |
0012da0f | 306 | masterdest=E.IMMAE_EU_RELEASE_PATH, |
8fa7ff2c | 307 | url="https://www.immae.eu")) |
0012da0f | 308 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH))) |
8fa7ff2c | 309 | |
f6e48e3a | 310 | return factory |
573e1710 | 311 | |
f6e48e3a | 312 | def cours_build_factory(): |
0012da0f | 313 | path_env = { |
830b619f IB |
314 | "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}", |
315 | "CI": "yes" | |
0012da0f IB |
316 | } |
317 | factory = util.BuildFactory() | |
318 | factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, | |
319 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
f6e48e3a | 320 | sshHostKey=E.SSH_HOST_KEY, mode="incremental")) |
0012da0f | 321 | factory.addStep(steps.ShellCommand(name="build website", |
f6e48e3a | 322 | logEnviron=False, haltOnFailure=True, |
0012da0f IB |
323 | command=["make", "build"], env=path_env)) |
324 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) | |
f6e48e3a | 325 | factory.addStep(steps.DirectoryUpload(workersrc="build", |
0012da0f IB |
326 | masterdest=E.COURS_RELEASE_PATH, |
327 | url="https://www.immae.eu/cours")) | |
328 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) | |
329 | ||
330 | factory.addStep(steps.ShellCommand(name="build pdfs", | |
f6e48e3a | 331 | logEnviron=False, haltOnFailure=True, |
0012da0f IB |
332 | command=["make", "pdfs"], env=path_env)) |
333 | ||
334 | package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) | |
335 | release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" | |
336 | package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch) | |
337 | package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch) | |
338 | factory.addStep(steps.ShellCommand(name="build pdf tarball", | |
f6e48e3a | 339 | logEnviron=False, haltOnFailure=True, |
0012da0f IB |
340 | command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env)) |
341 | factory.addStep(steps.FileUpload(name="upload package", workersrc=package, | |
f6e48e3a | 342 | masterdest=package_dest, url=package_url, mode=0o644)) |
0012da0f | 343 | |
f6e48e3a | 344 | return factory |
0012da0f | 345 | |
f6e48e3a | 346 | def normalesup_build_factory(): |
0012da0f IB |
347 | path_env = { |
348 | "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}" | |
349 | } | |
573e1710 IB |
350 | factory = util.BuildFactory() |
351 | factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, | |
352 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
f6e48e3a | 353 | sshHostKey=E.SSH_HOST_KEY, mode="incremental")) |
573e1710 | 354 | factory.addStep(steps.ShellCommand(name="build website", |
f6e48e3a | 355 | logEnviron=False, haltOnFailure=True, |
0012da0f | 356 | command=["make", "build"], env=path_env)) |
573e1710 | 357 | factory.addStep(steps.ShellCommand(name="give read access to all files", |
f6e48e3a | 358 | logEnviron=False, haltOnFailure=True, |
0012da0f | 359 | command="chmod -R a+rX build", env=path_env)) |
573e1710 | 360 | factory.addStep(steps.ShellCommand(name="synchronize with phare", |
f6e48e3a | 361 | logEnviron=False, haltOnFailure=True, |
0012da0f | 362 | env=path_env, command=[ |
573e1710 IB |
363 | "rsync", "-av", "--delete", |
364 | "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH), | |
365 | "build/", | |
0012da0f | 366 | os.environ["BUILDBOT_NORMALESUP_HOST"] |
573e1710 | 367 | ])) |
0012da0f | 368 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH))) |
f6e48e3a | 369 | factory.addStep(steps.DirectoryUpload(workersrc="build", masterdest=E.NORMALESUP_RELEASE_PATH, |
0012da0f IB |
370 | url="https://www.immae.eu/recherche")) |
371 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH))) | |
573e1710 | 372 | |
f6e48e3a | 373 | return factory |
60d328e2 | 374 | |
f6e48e3a | 375 | def gsm_cells_build_factory(): |
60d328e2 IB |
376 | path_env = { |
377 | "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}", | |
378 | "IN_BUILDBOT": "yes", | |
379 | } | |
380 | master_env = { | |
381 | "HTACCESS": ''' | |
382 | Options +FollowSymLinks | |
383 | IndexIgnore * | |
384 | ''' | |
385 | } | |
386 | for k, v in os.environ.items(): | |
387 | if k.startswith("BUILDBOT_GSM_CELLS_"): | |
388 | path_env[k[len("BUILDBOT_GSM_CELLS_"):]] = v | |
389 | ||
390 | script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" | |
391 | factory = util.BuildFactory() | |
392 | factory.addStep(steps.ShellCommand(name="download files", | |
393 | logEnviron=False, haltOnFailure=True, command=[script], env=path_env)) | |
394 | factory.addStep(steps.ShellCommand(name="give read access to all files", | |
395 | logEnviron=False, haltOnFailure=True, | |
396 | command="chmod a+r lacells.db", env=path_env)) | |
397 | factory.addStep(steps.FileUpload(workersrc="lacells.db", | |
398 | masterdest=(E.GSMCELLS_RELEASE_PATH+"/lacells.db"), url=(E.GSMCELLS_RELEASE_URL+"/lacells.db"))) | |
399 | factory.addStep(steps.MasterShellCommand(command="touch {}/.duplicity-ignore".format(E.GSMCELLS_RELEASE_PATH))) | |
400 | factory.addStep(steps.MasterShellCommand(command='echo "$HTACCESS" > {}/.htaccess'.format(E.GSMCELLS_RELEASE_PATH), | |
401 | env=master_env)) | |
402 | factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH))) | |
403 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH))) | |
404 | ||
f6e48e3a | 405 | return factory |
2ff9258e | 406 | |
f6e48e3a | 407 | def symfony_project_factory(name, repourl, parameters_path="app/config/parameters.yml", other_steps=lambda a : []): |
2ff9258e IB |
408 | if "BUILDBOT_PATH_SYMFONY_{}".format(name) in os.environ: |
409 | path_env = { | |
410 | "PATH": os.environ["BUILDBOT_PATH_SYMFONY_{}".format(name)] + ":${PATH}" | |
411 | } | |
412 | else: | |
413 | path_env = { | |
414 | "PATH": "${PATH}" | |
415 | } | |
416 | ||
417 | for k, v in os.environ.items(): | |
418 | if k.startswith("BUILDBOT_SYMFONY_{}_".format(name)): | |
419 | path_env[k[len("BUILDBOT_SYMFONY_{}_".format(name)):]] = v | |
420 | ||
421 | factory = util.BuildFactory() | |
422 | factory.addStep(steps.Git(logEnviron=False, repourl=repourl, | |
423 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
f6e48e3a | 424 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="fresh")) |
2ff9258e | 425 | if parameters_path is not None: |
f6e48e3a | 426 | factory.addStep(steps.FileDownload(mastersrc=get_parameters_file.withArgs(name), |
2ff9258e IB |
427 | workerdest=parameters_path)) |
428 | factory.addStep(NixShellCommand(name="build website", | |
f6e48e3a IB |
429 | logEnviron=False, haltOnFailure=True, |
430 | env=path_env, command=get_composer_install_command.withArgs(name))) | |
431 | if name in BRANCH_TO_POST_STEP: | |
432 | factory.addStep(NixShellCommand(name="build website post", | |
433 | logEnviron=False, haltOnFailure=True, doStepIf=partial(need_post_step, name), | |
434 | env=path_env, command=get_post_step_command.withArgs(name))) | |
2ff9258e IB |
435 | if parameters_path is not None: |
436 | factory.addStep(steps.ShellCommand(name="Remove parameters.yml", | |
437 | logEnviron=False, haltOnFailure=True, | |
438 | command="rm -f {}".format(parameters_path))) | |
439 | for step in other_steps(path_env): | |
440 | factory.addStep(step) | |
441 | package = util.Interpolate("{}_%(kw:clean_branch)s.tar.gz".format(name), clean_branch=clean_branch) | |
442 | release_file = "{0}/{1}/%(kw:clean_branch)s.tar.gz" | |
443 | package_dest = util.Interpolate(release_file.format("/var/lib/ftp/release.immae.eu/buildbot", name), clean_branch=clean_branch) | |
444 | # Tar doesn’t like creating the tarball in the same directory | |
445 | factory.addStep(steps.ShellCommand(name="Make a tarball 1/2", | |
f6e48e3a | 446 | logEnviron=False, haltOnFailure=True, env=path_env, |
2ff9258e IB |
447 | command=["touch", package])) |
448 | factory.addStep(steps.ShellCommand(name="Make a tarball 2/2", | |
f6e48e3a | 449 | logEnviron=False, haltOnFailure=True, env=path_env, |
2ff9258e | 450 | command=["tar", "--exclude", package, "-czf", package, "."])) |
f6e48e3a IB |
451 | factory.addStep(steps.SetPropertyFromCommand(command=get_systemd_service_invocation_command.withArgs(name), |
452 | property="service_invocation_id", doStepIf=partial(need_follow_systemd, name))) | |
2ff9258e | 453 | factory.addStep(steps.FileUpload(name="upload package", |
f6e48e3a IB |
454 | workersrc=package, masterdest=package_dest, mode=0o644)) |
455 | factory.addStep(steps.MasterShellCommand(command=follow_systemd_command.withArgs(name, util.Property("service_invocation_id")), env=path_env, logEnviron=False, doStepIf=partial(need_follow_systemd, name))) | |
456 | return factory |