]>
Commit | Line | Data |
---|---|---|
1 | from buildbot.plugins import * | |
2 | from buildbot_common.build_helpers import * | |
3 | import os | |
4 | from buildbot.util import bytes2unicode | |
5 | import json | |
6 | ||
7 | __all__ = [ "configure", "E" ] | |
8 | ||
9 | class E(): | |
10 | PROJECT = "immaeEu" | |
11 | BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT) | |
12 | SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT) | |
13 | PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT) | |
14 | SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key" | |
15 | SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF" | |
16 | XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ") | |
17 | ||
18 | BIP39_GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git" | |
19 | IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog" | |
20 | HISTORY_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/History" | |
21 | RECETTES_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Recettes" | |
22 | COURS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Cours" | |
23 | NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup" | |
24 | ||
25 | COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/cours" | |
26 | COURS_TARBALL_PATH = "/var/lib/ftp/release.immae.eu/cours" | |
27 | COURS_TARBALL_URL = "https://release.immae.eu/cours" | |
28 | BIP39_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/bip39" | |
29 | HISTORY_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/history" | |
30 | IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/blog" | |
31 | RECETTES_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recettes" | |
32 | NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recherche" | |
33 | GSMCELLS_RELEASE_PATH = "/var/lib/ftp/release.immae.eu/gsm_cells" | |
34 | GSMCELLS_RELEASE_URL = "https://release.immae.eu/gsm_cells" | |
35 | ||
36 | # master.cfg | |
37 | SECRETS_FILE = os.getcwd() + "/secrets" | |
38 | LDAP_URL = "ldaps://ldap.immae.eu:636" | |
39 | LDAP_ADMIN_USER = "cn=buildbot,ou=services,dc=immae,dc=eu" | |
40 | LDAP_BASE = "dc=immae,dc=eu" | |
41 | LDAP_PATTERN = "(uid=%(username)s)" | |
42 | LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=immaeEu,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))" | |
43 | TITLE_URL = "https://www.immae.eu" | |
44 | TITLE = "Immae website" | |
45 | ||
46 | class CustomBase(webhooks.base): | |
47 | def getChanges(self, request): | |
48 | try: | |
49 | content = request.content.read() | |
50 | args = json.loads(bytes2unicode(content)) | |
51 | except Exception as e: | |
52 | raise ValueError("Error loading JSON: " + str(e)) | |
53 | ||
54 | args.setdefault("comments", "") | |
55 | args.setdefault("repository", "") | |
56 | args.setdefault("author", args.get("who", "unknown")) | |
57 | ||
58 | return ([args], None) | |
59 | ||
60 | def configure(c): | |
61 | c["buildbotURL"] = E.BUILDBOT_URL | |
62 | c["www"]["port"] = E.SOCKET | |
63 | ||
64 | c["www"]["change_hook_dialects"]["base"] = { | |
65 | "custom_class": CustomBase | |
66 | } | |
67 | c['workers'].append(worker.LocalWorker("generic-worker-immae-eu")) | |
68 | ||
69 | c['schedulers'].append(hook_scheduler("ImmaeEu", timer=1)) | |
70 | c['schedulers'].append(hook_scheduler("Normalesup", timer=1)) | |
71 | c['schedulers'].append(hook_scheduler("Cours", timer=1)) | |
72 | c['schedulers'].append(hook_scheduler("Recettes", timer=1)) | |
73 | c['schedulers'].append(hook_scheduler("History", timer=1)) | |
74 | c['schedulers'].append(hook_scheduler("BIP39", timer=1)) | |
75 | c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly", | |
76 | builderNames=["GSMCells_build"], dayOfWeek=6, hour=3)) | |
77 | c['schedulers'].append(force_scheduler("force_immae_eu", [ | |
78 | "ImmaeEu_build", "Normalesup_build", "Cours_build", | |
79 | "Recettes_build", "History_build", "BIP39_build" | |
80 | ])) | |
81 | c['schedulers'].append(schedulers.ForceScheduler( | |
82 | name="GSMCells-force", label="Force build", | |
83 | buttonName="Force build", | |
84 | reason=util.StringParameter(name="reason", label="Reason", default="Force build"), | |
85 | codebases=[ | |
86 | util.CodebaseParameter("", | |
87 | branch=util.FixedParameter(name="branch", default=""), | |
88 | revision=util.FixedParameter(name="revision", default=""), | |
89 | repository=util.FixedParameter(name="repository", default=""), | |
90 | project=util.FixedParameter(name="project", default=""), | |
91 | ), | |
92 | ], | |
93 | username=util.FixedParameter(name="username", default="Web button"), | |
94 | builderNames=["GSMCells_build"] | |
95 | )) | |
96 | ||
97 | c['builders'].append(immae_eu_factory()) | |
98 | c['builders'].append(normalesup_factory()) | |
99 | c['builders'].append(cours_factory()) | |
100 | c['builders'].append(gsm_cells_factory()) | |
101 | c['builders'].append(recettes_factory()) | |
102 | c['builders'].append(history_factory()) | |
103 | c['builders'].append(bip39_factory()) | |
104 | ||
105 | c['services'].append(SlackStatusPush( | |
106 | name="slack_status_immae_eu_project", | |
107 | builders=[ | |
108 | "ImmaeEu_build", "Normalesup_build", "Cours_build", | |
109 | "GSMCells_build", "Recettes_build", "History_build", | |
110 | "BIP39_build" | |
111 | ], | |
112 | serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) | |
113 | c['services'].append(XMPPStatusPush( | |
114 | name="xmpp_status_immae_eu_project", | |
115 | builders=[ | |
116 | "ImmaeEu_build", "Normalesup_build", "Cours_build", | |
117 | "GSMCells_build", "Recettes_build", "History_build", | |
118 | "BIP39_build" | |
119 | ], | |
120 | recipients=E.XMPP_RECIPIENTS, | |
121 | password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip())) | |
122 | ||
123 | def history_factory(): | |
124 | path_env = { | |
125 | "PATH": os.environ["BUILDBOT_PATH_History"] + ":${PATH}" | |
126 | } | |
127 | factory = util.BuildFactory() | |
128 | factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL, | |
129 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
130 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) | |
131 | factory.addStep(steps.ShellCommand(name="build website", | |
132 | logEnviron=False, haltOnFailure=True, workdir="source", | |
133 | env=path_env, command=["jekyll", "build"])) | |
134 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH))) | |
135 | factory.addStep(steps.DirectoryUpload(workersrc="../source/_site", | |
136 | masterdest=E.HISTORY_RELEASE_PATH, | |
137 | url="https://www.immae.eu/history")) | |
138 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.HISTORY_RELEASE_PATH))) | |
139 | ||
140 | return util.BuilderConfig(name="History_build", workernames=["generic-worker-immae-eu"], factory=factory) | |
141 | ||
142 | def recettes_factory(): | |
143 | path_env = { | |
144 | "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}" | |
145 | } | |
146 | factory = util.BuildFactory() | |
147 | factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL, | |
148 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
149 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) | |
150 | factory.addStep(steps.ShellCommand(name="build website", | |
151 | logEnviron=False, haltOnFailure=True, workdir="source", | |
152 | env=path_env, command=["jekyll", "build", "--baseurl", "/recettes"])) | |
153 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RECETTES_RELEASE_PATH))) | |
154 | factory.addStep(steps.DirectoryUpload(workersrc="../source/_site", | |
155 | masterdest=E.RECETTES_RELEASE_PATH, | |
156 | url="https://www.immae.eu/recettes")) | |
157 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RECETTES_RELEASE_PATH))) | |
158 | ||
159 | return util.BuilderConfig(name="Recettes_build", workernames=["generic-worker-immae-eu"], factory=factory) | |
160 | ||
161 | def bip39_factory(): | |
162 | path_env = { | |
163 | "PATH": os.environ["BUILDBOT_PATH_BIP39"] + ":${PATH}" | |
164 | } | |
165 | factory = util.BuildFactory() | |
166 | factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL, | |
167 | submodules=True, mode="full", method="copy")) | |
168 | factory.addStep(steps.ShellCommand(name="build file", | |
169 | logEnviron=False, haltOnFailure=True, workdir="source", | |
170 | env=path_env, command=["python", "compile.py"])) | |
171 | factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html", | |
172 | workdir="source", masterdest=E.BIP39_RELEASE_PATH + "/index.html", | |
173 | url="https://tools.immae.eu/BIP39", mode=0o644)) | |
174 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.BIP39_RELEASE_PATH))) | |
175 | ||
176 | return util.BuilderConfig(name="BIP39_build", workernames=["generic-worker-immae-eu"], factory=factory) | |
177 | ||
178 | def immae_eu_factory(): | |
179 | path_env = { | |
180 | "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}" | |
181 | } | |
182 | factory = util.BuildFactory() | |
183 | factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL, | |
184 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
185 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) | |
186 | factory.addStep(steps.ShellCommand(name="build website", | |
187 | logEnviron=False, haltOnFailure=True, workdir="source", | |
188 | env=path_env, command=["make", "html"])) | |
189 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH))) | |
190 | factory.addStep(steps.DirectoryUpload(workersrc="../source/output", | |
191 | masterdest=E.IMMAE_EU_RELEASE_PATH, | |
192 | url="https://www.immae.eu")) | |
193 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH))) | |
194 | ||
195 | return util.BuilderConfig(name="ImmaeEu_build", workernames=["generic-worker-immae-eu"], factory=factory) | |
196 | ||
197 | def cours_factory(): | |
198 | path_env = { | |
199 | "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}", | |
200 | "CI": "yes" | |
201 | } | |
202 | factory = util.BuildFactory() | |
203 | factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL, | |
204 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
205 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) | |
206 | factory.addStep(steps.ShellCommand(name="build website", | |
207 | logEnviron=False, haltOnFailure=True, workdir="source", | |
208 | command=["make", "build"], env=path_env)) | |
209 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH))) | |
210 | factory.addStep(steps.DirectoryUpload(workersrc="../source/build", | |
211 | masterdest=E.COURS_RELEASE_PATH, | |
212 | url="https://www.immae.eu/cours")) | |
213 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH))) | |
214 | ||
215 | factory.addStep(steps.ShellCommand(name="build pdfs", | |
216 | logEnviron=False, haltOnFailure=True, workdir="source", | |
217 | command=["make", "pdfs"], env=path_env)) | |
218 | ||
219 | package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch) | |
220 | release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz" | |
221 | package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch) | |
222 | package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch) | |
223 | factory.addStep(steps.ShellCommand(name="build pdf tarball", | |
224 | logEnviron=False, haltOnFailure=True, workdir="source", | |
225 | command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env)) | |
226 | factory.addStep(steps.FileUpload(name="upload package", workersrc=package, | |
227 | workdir="source", masterdest=package_dest, | |
228 | url=package_url, mode=0o644)) | |
229 | ||
230 | return util.BuilderConfig(name="Cours_build", workernames=["generic-worker-immae-eu"], factory=factory) | |
231 | ||
232 | def normalesup_factory(): | |
233 | path_env = { | |
234 | "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}" | |
235 | } | |
236 | factory = util.BuildFactory() | |
237 | factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL, | |
238 | submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(), | |
239 | sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy")) | |
240 | factory.addStep(steps.ShellCommand(name="build website", | |
241 | logEnviron=False, haltOnFailure=True, workdir="source", | |
242 | command=["make", "build"], env=path_env)) | |
243 | factory.addStep(steps.ShellCommand(name="give read access to all files", | |
244 | logEnviron=False, haltOnFailure=True, workdir="source", | |
245 | command="chmod -R a+rX build", env=path_env)) | |
246 | factory.addStep(steps.ShellCommand(name="synchronize with phare", | |
247 | logEnviron=False, haltOnFailure=True, workdir="source", | |
248 | env=path_env, command=[ | |
249 | "rsync", "-av", "--delete", | |
250 | "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH), | |
251 | "build/", | |
252 | os.environ["BUILDBOT_NORMALESUP_HOST"] | |
253 | ])) | |
254 | factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH))) | |
255 | factory.addStep(steps.DirectoryUpload(workersrc="../source/build", masterdest=E.NORMALESUP_RELEASE_PATH, | |
256 | url="https://www.immae.eu/recherche")) | |
257 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH))) | |
258 | ||
259 | return util.BuilderConfig(name="Normalesup_build", workernames=["generic-worker-immae-eu"], factory=factory) | |
260 | ||
261 | def gsm_cells_factory(): | |
262 | path_env = { | |
263 | "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}", | |
264 | "IN_BUILDBOT": "yes", | |
265 | } | |
266 | master_env = { | |
267 | "HTACCESS": ''' | |
268 | Options +FollowSymLinks | |
269 | IndexIgnore * | |
270 | ''' | |
271 | } | |
272 | for k, v in os.environ.items(): | |
273 | if k.startswith("BUILDBOT_GSM_CELLS_"): | |
274 | path_env[k[len("BUILDBOT_GSM_CELLS_"):]] = v | |
275 | ||
276 | script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" | |
277 | factory = util.BuildFactory() | |
278 | factory.addStep(steps.ShellCommand(name="download files", | |
279 | logEnviron=False, haltOnFailure=True, command=[script], env=path_env)) | |
280 | factory.addStep(steps.ShellCommand(name="give read access to all files", | |
281 | logEnviron=False, haltOnFailure=True, | |
282 | command="chmod a+r lacells.db", env=path_env)) | |
283 | factory.addStep(steps.FileUpload(workersrc="lacells.db", | |
284 | masterdest=(E.GSMCELLS_RELEASE_PATH+"/lacells.db"), url=(E.GSMCELLS_RELEASE_URL+"/lacells.db"))) | |
285 | factory.addStep(steps.MasterShellCommand(command="touch {}/.duplicity-ignore".format(E.GSMCELLS_RELEASE_PATH))) | |
286 | factory.addStep(steps.MasterShellCommand(command='echo "$HTACCESS" > {}/.htaccess'.format(E.GSMCELLS_RELEASE_PATH), | |
287 | env=master_env)) | |
288 | factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH))) | |
289 | factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH))) | |
290 | ||
291 | return util.BuilderConfig(name="GSMCells_build", workernames=["generic-worker-immae-eu"], factory=factory) |