aboutsummaryrefslogtreecommitdiff
path: root/modules/private/buildbot/projects/immaeEu
diff options
context:
space:
mode:
Diffstat (limited to 'modules/private/buildbot/projects/immaeEu')
-rw-r--r--modules/private/buildbot/projects/immaeEu/__init__.py314
-rwxr-xr-xmodules/private/buildbot/projects/immaeEu/scripts/lacells_download163
2 files changed, 0 insertions, 477 deletions
diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py
deleted file mode 100644
index 83265cd..0000000
--- a/modules/private/buildbot/projects/immaeEu/__init__.py
+++ /dev/null
@@ -1,314 +0,0 @@
1from buildbot.plugins import *
2from buildbot_common.build_helpers import *
3import os
4from buildbot.util import bytes2unicode
5import json
6
7__all__ = [ "configure", "E" ]
8
9class E():
10 PROJECT = "immaeEu"
11 BUILDBOT_URL = "https://git.immae.eu/buildbot/{}/".format(PROJECT)
12 SOCKET = "unix:/run/buildbot/{}.sock".format(PROJECT)
13 PB_SOCKET = "unix:address=/run/buildbot/{}_pb.sock".format(PROJECT)
14 SSH_KEY_PATH = "/var/lib/buildbot/buildbot_key"
15 SSH_HOST_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFbhFTl2A2RJn5L51yxJM4XfCS2ZaiSX/jo9jFSdghF"
16 XMPP_RECIPIENTS = os.environ["BUILDBOT_XMPP_RECIPIENTS"].split(" ")
17
18 BIP39_GIT_URL = "https://git.immae.eu/perso/Immae/Projets/Cryptomonnaies/BIP39.git"
19 IMMAE_EU_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Blog"
20 HISTORY_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/History"
21 RECETTES_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Recettes"
22 COURS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Cours"
23 DOCS_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Sites/Docs"
24 NORMALESUP_GIT_URL = "gitolite@git.immae.eu:perso/Immae/Projets/Sites/Normalesup"
25
26 COURS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/cours"
27 COURS_TARBALL_PATH = "/var/lib/ftp/release.immae.eu/cours"
28 COURS_TARBALL_URL = "https://release.immae.eu/cours"
29 BIP39_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/bip39"
30 HISTORY_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/history"
31 IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/blog"
32 DOCS_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/docs"
33 RECETTES_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recettes"
34 NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/immae/recherche"
35 GSMCELLS_RELEASE_PATH = "/var/lib/ftp/release.immae.eu/gsm_cells"
36 GSMCELLS_RELEASE_URL = "https://release.immae.eu/gsm_cells"
37
38 # master.cfg
39 SECRETS_FILE = os.getcwd() + "/secrets"
40 LDAP_URL = "ldaps://ldap.immae.eu:636"
41 LDAP_ADMIN_USER = "cn=buildbot,ou=services,dc=immae,dc=eu"
42 LDAP_BASE = "dc=immae,dc=eu"
43 LDAP_PATTERN = "(uid=%(username)s)"
44 LDAP_GROUP_PATTERN = "(&(memberOf=cn=groups,ou=immaeEu,cn=buildbot,ou=services,dc=immae,dc=eu)(member=%(dn)s))"
45 TITLE_URL = "https://www.immae.eu"
46 TITLE = "Immae website"
47
48class CustomBase(webhooks.base):
49 def getChanges(self, request):
50 try:
51 content = request.content.read()
52 args = json.loads(bytes2unicode(content))
53 except Exception as e:
54 raise ValueError("Error loading JSON: " + str(e))
55
56 args.setdefault("comments", "")
57 args.setdefault("repository", "")
58 args.setdefault("author", args.get("who", "unknown"))
59
60 return ([args], None)
61
62def configure(c):
63 c["buildbotURL"] = E.BUILDBOT_URL
64 c["www"]["port"] = E.SOCKET
65
66 c["www"]["change_hook_dialects"]["base"] = {
67 "custom_class": CustomBase
68 }
69 c['workers'].append(worker.LocalWorker("generic-worker-immae-eu"))
70
71 c['schedulers'].append(hook_scheduler("ImmaeEu", timer=1))
72 c['schedulers'].append(hook_scheduler("Normalesup", timer=1))
73 c['schedulers'].append(hook_scheduler("Cours", timer=1))
74 c['schedulers'].append(hook_scheduler("Recettes", timer=1))
75 c['schedulers'].append(hook_scheduler("Docs", timer=1))
76 c['schedulers'].append(hook_scheduler("History", timer=1))
77 c['schedulers'].append(hook_scheduler("BIP39", timer=1))
78 c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly",
79 builderNames=["GSMCells_build"], dayOfWeek=6, hour=3))
80 c['schedulers'].append(force_scheduler("force_immae_eu", [
81 "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build",
82 "Recettes_build", "History_build", "BIP39_build"
83 ]))
84 c['schedulers'].append(schedulers.ForceScheduler(
85 name="GSMCells-force", label="Force build",
86 buttonName="Force build",
87 reason=util.StringParameter(name="reason", label="Reason", default="Force build"),
88 codebases=[
89 util.CodebaseParameter("",
90 branch=util.FixedParameter(name="branch", default=""),
91 revision=util.FixedParameter(name="revision", default=""),
92 repository=util.FixedParameter(name="repository", default=""),
93 project=util.FixedParameter(name="project", default=""),
94 ),
95 ],
96 username=util.FixedParameter(name="username", default="Web button"),
97 builderNames=["GSMCells_build"]
98 ))
99
100 c['builders'].append(immae_eu_factory())
101 c['builders'].append(normalesup_factory())
102 c['builders'].append(cours_factory())
103 c['builders'].append(gsm_cells_factory())
104 c['builders'].append(recettes_factory())
105 c['builders'].append(docs_factory())
106 c['builders'].append(history_factory())
107 c['builders'].append(bip39_factory())
108
109 c['services'].append(SlackStatusPush(
110 name="slack_status_immae_eu_project",
111 builders=[
112 "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build",
113 "GSMCells_build", "Recettes_build", "History_build",
114 "BIP39_build"
115 ],
116 serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip()))
117 c['services'].append(XMPPStatusPush(
118 name="xmpp_status_immae_eu_project",
119 builders=[
120 "ImmaeEu_build", "Normalesup_build", "Cours_build", "Docs_build",
121 "GSMCells_build", "Recettes_build", "History_build",
122 "BIP39_build"
123 ],
124 recipients=E.XMPP_RECIPIENTS,
125 password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip()))
126
127def history_factory():
128 path_env = {
129 "PATH": os.environ["BUILDBOT_PATH_History"] + ":${PATH}"
130 }
131 factory = util.BuildFactory()
132 factory.addStep(steps.Git(logEnviron=False, repourl=E.HISTORY_GIT_URL,
133 submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
134 sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy"))
135 factory.addStep(steps.ShellCommand(name="build website",
136 logEnviron=False, haltOnFailure=True, workdir="source",
137 env=path_env, command=["jekyll", "build"]))
138 factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.HISTORY_RELEASE_PATH)))
139 factory.addStep(steps.DirectoryUpload(workersrc="../source/_site",
140 masterdest=E.HISTORY_RELEASE_PATH,
141 url="https://www.immae.eu/history"))
142 factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.HISTORY_RELEASE_PATH)))
143
144 return util.BuilderConfig(name="History_build", workernames=["generic-worker-immae-eu"], factory=factory)
145
146def docs_factory():
147 path_env = {
148 "PATH": os.environ["BUILDBOT_PATH_Docs"] + ":${PATH}"
149 }
150 factory = util.BuildFactory()
151 factory.addStep(steps.Git(logEnviron=False, repourl=E.DOCS_GIT_URL,
152 submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
153 sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy"))
154 factory.addStep(steps.ShellCommand(name="build website",
155 logEnviron=False, haltOnFailure=True, workdir="source",
156 env=path_env, command=["make", "clean", "html"]))
157 factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.DOCS_RELEASE_PATH)))
158 factory.addStep(steps.DirectoryUpload(workersrc="../source/_build/html",
159 masterdest=E.DOCS_RELEASE_PATH,
160 url="https://www.immae.eu/docs"))
161 factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.DOCS_RELEASE_PATH)))
162
163 return util.BuilderConfig(name="Docs_build", workernames=["generic-worker-immae-eu"], factory=factory)
164
165def recettes_factory():
166 path_env = {
167 "PATH": os.environ["BUILDBOT_PATH_Recettes"] + ":${PATH}"
168 }
169 factory = util.BuildFactory()
170 factory.addStep(steps.Git(logEnviron=False, repourl=E.RECETTES_GIT_URL,
171 submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
172 sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy"))
173 factory.addStep(NixShellCommand(name="build website",
174 logEnviron=False, haltOnFailure=True, workdir="source",
175 env=path_env, command="jekyll build --trace --baseurl /recettes"))
176 factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.RECETTES_RELEASE_PATH)))
177 factory.addStep(steps.DirectoryUpload(workersrc="../source/_site",
178 masterdest=E.RECETTES_RELEASE_PATH,
179 url="https://www.immae.eu/recettes"))
180 factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.RECETTES_RELEASE_PATH)))
181
182 return util.BuilderConfig(name="Recettes_build", workernames=["generic-worker-immae-eu"], factory=factory)
183
184def bip39_factory():
185 path_env = {
186 "PATH": os.environ["BUILDBOT_PATH_BIP39"] + ":${PATH}"
187 }
188 factory = util.BuildFactory()
189 factory.addStep(steps.Git(logEnviron=False, repourl=E.BIP39_GIT_URL,
190 submodules=True, mode="full", method="copy"))
191 factory.addStep(steps.ShellCommand(name="build file",
192 logEnviron=False, haltOnFailure=True, workdir="source",
193 env=path_env, command=["python", "compile.py"]))
194 factory.addStep(steps.FileUpload(name="upload file", workersrc="bip39-standalone.html",
195 workdir="source", masterdest=E.BIP39_RELEASE_PATH + "/index.html",
196 url="https://tools.immae.eu/BIP39", mode=0o644))
197 factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.BIP39_RELEASE_PATH)))
198
199 return util.BuilderConfig(name="BIP39_build", workernames=["generic-worker-immae-eu"], factory=factory)
200
201def immae_eu_factory():
202 path_env = {
203 "PATH": os.environ["BUILDBOT_PATH_ImmaeEu"] + ":${PATH}"
204 }
205 factory = util.BuildFactory()
206 factory.addStep(steps.Git(logEnviron=False, repourl=E.IMMAE_EU_GIT_URL,
207 submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
208 sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy"))
209 factory.addStep(steps.ShellCommand(name="build website",
210 logEnviron=False, haltOnFailure=True, workdir="source",
211 env=path_env, command=["make", "html"]))
212 factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.IMMAE_EU_RELEASE_PATH)))
213 factory.addStep(steps.DirectoryUpload(workersrc="../source/output",
214 masterdest=E.IMMAE_EU_RELEASE_PATH,
215 url="https://www.immae.eu"))
216 factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.IMMAE_EU_RELEASE_PATH)))
217
218 return util.BuilderConfig(name="ImmaeEu_build", workernames=["generic-worker-immae-eu"], factory=factory)
219
220def cours_factory():
221 path_env = {
222 "PATH": os.environ["BUILDBOT_PATH_Cours"] + ":${PATH}",
223 "CI": "yes"
224 }
225 factory = util.BuildFactory()
226 factory.addStep(steps.Git(logEnviron=False, repourl=E.COURS_GIT_URL,
227 submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
228 sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy"))
229 factory.addStep(steps.ShellCommand(name="build website",
230 logEnviron=False, haltOnFailure=True, workdir="source",
231 command=["make", "build"], env=path_env))
232 factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.COURS_RELEASE_PATH)))
233 factory.addStep(steps.DirectoryUpload(workersrc="../source/build",
234 masterdest=E.COURS_RELEASE_PATH,
235 url="https://www.immae.eu/cours"))
236 factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.COURS_RELEASE_PATH)))
237
238 factory.addStep(steps.ShellCommand(name="build pdfs",
239 logEnviron=False, haltOnFailure=True, workdir="source",
240 command=["make", "pdfs"], env=path_env))
241
242 package = util.Interpolate("cours_%(kw:clean_branch)s.tar.gz", clean_branch=clean_branch)
243 release_file = "{0}/cours_%(kw:clean_branch)s.tar.gz"
244 package_dest = util.Interpolate(release_file.format(E.COURS_TARBALL_PATH), clean_branch=clean_branch)
245 package_url = util.Interpolate(release_file.format(E.COURS_TARBALL_URL), clean_branch=clean_branch)
246 factory.addStep(steps.ShellCommand(name="build pdf tarball",
247 logEnviron=False, haltOnFailure=True, workdir="source",
248 command=["tar", "-cvf", package, "-C", "pdfs", "mp", "mpsi"], env=path_env))
249 factory.addStep(steps.FileUpload(name="upload package", workersrc=package,
250 workdir="source", masterdest=package_dest,
251 url=package_url, mode=0o644))
252
253 return util.BuilderConfig(name="Cours_build", workernames=["generic-worker-immae-eu"], factory=factory)
254
255def normalesup_factory():
256 path_env = {
257 "PATH": os.environ["BUILDBOT_PATH_Normalesup"] + ":${PATH}"
258 }
259 factory = util.BuildFactory()
260 factory.addStep(steps.Git(logEnviron=False, repourl=E.NORMALESUP_GIT_URL,
261 submodules=True, sshPrivateKey=open(E.SSH_KEY_PATH).read().rstrip(),
262 sshHostKey=E.SSH_HOST_KEY, mode="full", method="copy"))
263 factory.addStep(steps.ShellCommand(name="build website",
264 logEnviron=False, haltOnFailure=True, workdir="source",
265 command=["make", "build"], env=path_env))
266 factory.addStep(steps.ShellCommand(name="give read access to all files",
267 logEnviron=False, haltOnFailure=True, workdir="source",
268 command="chmod -R a+rX build", env=path_env))
269 factory.addStep(steps.ShellCommand(name="synchronize with phare",
270 logEnviron=False, haltOnFailure=True, workdir="source",
271 env=path_env, command=[
272 "rsync", "-av", "--delete",
273 "-e", "ssh -i {} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no".format(E.SSH_KEY_PATH),
274 "build/",
275 os.environ["BUILDBOT_NORMALESUP_HOST"]
276 ]))
277 factory.addStep(steps.MasterShellCommand(command="rm -rf {}".format(E.NORMALESUP_RELEASE_PATH)))
278 factory.addStep(steps.DirectoryUpload(workersrc="../source/build", masterdest=E.NORMALESUP_RELEASE_PATH,
279 url="https://www.immae.eu/recherche"))
280 factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH)))
281
282 return util.BuilderConfig(name="Normalesup_build", workernames=["generic-worker-immae-eu"], factory=factory)
283
284def gsm_cells_factory():
285 path_env = {
286 "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}",
287 "IN_BUILDBOT": "yes",
288 }
289 master_env = {
290 "HTACCESS": '''
291 Options +FollowSymLinks
292 IndexIgnore *
293 '''
294 }
295 for k, v in os.environ.items():
296 if k.startswith("BUILDBOT_GSM_CELLS_"):
297 path_env[k[len("BUILDBOT_GSM_CELLS_"):]] = v
298
299 script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download"
300 factory = util.BuildFactory()
301 factory.addStep(steps.ShellCommand(name="download files",
302 logEnviron=False, haltOnFailure=True, command=[script], env=path_env))
303 factory.addStep(steps.ShellCommand(name="give read access to all files",
304 logEnviron=False, haltOnFailure=True,
305 command="chmod a+r lacells.db", env=path_env))
306 factory.addStep(steps.FileUpload(workersrc="lacells.db",
307 masterdest=(E.GSMCELLS_RELEASE_PATH+"/lacells.db"), url=(E.GSMCELLS_RELEASE_URL+"/lacells.db")))
308 factory.addStep(steps.MasterShellCommand(command="touch {}/.duplicity-ignore".format(E.GSMCELLS_RELEASE_PATH)))
309 factory.addStep(steps.MasterShellCommand(command='echo "$HTACCESS" > {}/.htaccess'.format(E.GSMCELLS_RELEASE_PATH),
310 env=master_env))
311 factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH)))
312 factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH)))
313
314 return util.BuilderConfig(name="GSMCells_build", workernames=["generic-worker-immae-eu"], factory=factory)
diff --git a/modules/private/buildbot/projects/immaeEu/scripts/lacells_download b/modules/private/buildbot/projects/immaeEu/scripts/lacells_download
deleted file mode 100755
index 1193cf3..0000000
--- a/modules/private/buildbot/projects/immaeEu/scripts/lacells_download
+++ /dev/null
@@ -1,163 +0,0 @@
1#!/usr/bin/env bash
2
3# FLG - Fast Lacells.db Generator
4#
5# Simple script to quickly download and generate lacells.db for LocalGSMBackend by n76
6# https://github.com/n76/Local-GSM-Backend
7# Uses Mozilla Location Service, OpenCellID and radiocells.org databases as source
8# Based on lacells-creator by wvengen and n76
9#
10# Licensed under GPLv3 or later
11# (C)2016 Sebastian Obrusiewicz
12# sobrus@o2.pl
13
14if [ -z "$IN_BUILDBOT" ]; then
15#DEFAULT_CONFIG_BEGIN
16ENABLE_OCI=1 #enable OpenCellID data source
17ENABLE_MOZ=1 #enable Mozilla Location Services (MLS) data source
18ENABLE_RCO=0 #enable radiocells.org data source (it can be quite slow)
19
20# See https://en.wikipedia.org/wiki/Mobile_country_code
21# 208 France
22MCC="" #contry codes separated with "|", for example "260|262". Leave dot+asterisk ".*" for all countries
23RCO_SRC_FILE="fr.sqlite" #radiocells.org source database file, set "openbmap.sqlite" for entire world database, see https://radiocells.org/downloads for smaller country specific files, for example "pl.sqlite" for Poland
24RADIO="" #you can remove LTE if your phone does not support it
25TOKEN="" #your OCID token, required to download from OpenCellID. Get your free token at https://opencellid.org/
26fi
27TMPDIR='.' #for temporary files only, use disk if you don't have enough RAM, AND remember to have enough disk space in /var/tmp for sqlite temporary files
28KEEP_FILES=1 #whether to keep (1) or delete (0) the CSV files after processing
29
30#do not edit following variables, unless you know what you're doing
31EMPTY=',,,,,,,,,,,,,' #dummy empty file for disabled sources
32OCI_FILE=$TMPDIR"/ocid.csv" #opencellid temporary file
33MOZ_FILE=$TMPDIR"/mozilla.csv" #mozilla temporary file
34RCO_FILE=$TMPDIR"/rco.csv" #radiocells.org temporary file
35#DEFAULT_CONFIG_END
36
37#USER_CONFIG_BEGIN
38BINDIR=$( dirname "$(readlink -f "$0")" ) #"
39if [[ -f "${BINDIR}/config" ]]; then
40 . "${BINDIR}/config"
41fi
42#USER_CONFIG_END
43
44function manage_backup
45{
46 file=$1
47 if [ -s $file ]
48 then
49 if [ $KEEP_FILES == "1" ]
50 then
51 gzip -kf $file
52 fi
53 elif [ -s $file".gz" ] && [ "${file##*.}" == "csv" ]
54 then
55 echo "Using" $file".gz backup file"
56 gzip -dkf $file".gz"
57 fi
58}
59
60
61function download_ocid
62{
63 URL="https://opencellid.org/ocid/downloads?token=${TOKEN}&type=full&file=cell_towers.csv.gz"
64 if [ $ENABLE_OCI == "1" ]
65 then
66 wget -qO- "$URL" | gunzip | egrep "^($RADIO),($MCC)," > $OCI_FILE
67 manage_backup $OCI_FILE
68 else
69 echo $EMPTY > $OCI_FILE
70 fi
71}
72
73function download_mozilla
74{
75 if [ $ENABLE_MOZ == "1" ]
76 then
77 NW=`date -u "+%Y-%m-%d"`
78 wget -qO- "https://d17pt8qph6ncyq.cloudfront.net/export/MLS-full-cell-export-${NW}T000000.csv.gz" | gunzip | egrep "^($RADIO),($MCC)," > $MOZ_FILE
79 manage_backup $MOZ_FILE
80 else
81 echo $EMPTY > $MOZ_FILE
82 fi
83}
84
85function download_radiocells
86{
87 if [ $ENABLE_RCO == "1" ]
88 then
89 RCO_SELECT="SELECT technology, mcc, mnc, area, cid, NULL, longitude, latitude, 1000 accuracy, measurements, NULL, NULL, NULL, NULL FROM cell_zone;"
90 wget -qO- "https://cdn.radiocells.org/"$RCO_SRC_FILE > $TMPDIR"/"$RCO_SRC_FILE
91 sqlite3 -header -csv $TMPDIR"/"$RCO_SRC_FILE "$RCO_SELECT" | egrep "^($RADIO),($MCC)," > $RCO_FILE
92 rm $TMPDIR"/"$RCO_SRC_FILE
93 manage_backup $RCO_FILE
94 else
95 echo $EMPTY > $RCO_FILE
96 fi
97}
98
99echo "Downloading data"
100
101download_ocid &
102OP=$!
103download_mozilla &
104MO=$!
105download_radiocells &
106RO=$!
107
108wait $OP
109wait $MO
110wait $RO
111
112if [ -s $MOZ_FILE ] && [ -s $OCI_FILE ] && [ -s $RCO_FILE ]
113then
114
115manage_backup lacells.db
116rm lacells.db
117
118echo "Generating database"
119
120sqlite3 lacells.db <<-SQL
121PRAGMA synchronous = OFF;
122PRAGMA journal_mode = OFF;
123
124CREATE TEMP TABLE cells_import (radio TEXT,mcc INTEGER,mnc INTEGER,lac INTEGER,cid INTEGER,unit STRING,longitude NUMERIC,latitude NUMERIC,accuracy INTEGER,samples INTEGER,changeable BOOLEAN,created INTEGER,updated INTEGER, avgSignal INTEGER);
125CREATE TABLE cells (mcc INTEGER,mnc INTEGER,lac INTEGER,cid INTEGER,longitude REAL,latitude REAL,altitude REAL,accuracy REAL,samples INTEGER);
126
127.header on
128.mode csv
129
130.import "$OCI_FILE" cells_import
131.import "$MOZ_FILE" cells_import
132.import "$RCO_FILE" cells_import
133
134UPDATE cells_import SET samples=1 WHERE samples IS NULL OR samples < 1;
135
136INSERT INTO cells
137SELECT mcc, mnc, lac, cid,
138 sum(longitude * samples) / sum(samples) as longitude,
139 sum(latitude * samples) / sum(samples) as latitude,
140 -1 as altitude,
141 sum(accuracy * samples) / sum(samples) as accuracy,
142 sum(samples) as samples
143FROM cells_import
144GROUP BY mcc, mnc, lac, cid;
145
146DROP TABLE cells_import;
147
148UPDATE cells SET accuracy=500 WHERE accuracy < 500;
149UPDATE cells SET accuracy=100000 WHERE accuracy > 100000;
150
151CREATE INDEX _idx1 ON cells (mcc, mnc, lac, cid);
152CREATE INDEX _idx2 ON cells (lac, cid);
153
154VACUUM;
155SQL
156
157else
158 echo "Download error"
159fi
160
161rm $OCI_FILE
162rm $MOZ_FILE
163rm $RCO_FILE
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-26 15:15:11 +0000