From 60d328e2558e982d441f64b54e089b0e1f6b70f9 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Sun, 23 Feb 2020 19:57:08 +0100 Subject: [PATCH] Add gsm cells download script --- modules/private/buildbot/default.nix | 3 +- .../buildbot/projects/immaeEu/__init__.py | 66 ++++++- .../projects/immaeEu/scripts/lacells_download | 163 ++++++++++++++++++ 3 files changed, 228 insertions(+), 4 deletions(-) create mode 100755 modules/private/buildbot/projects/immaeEu/scripts/lacells_download diff --git a/modules/private/buildbot/default.nix b/modules/private/buildbot/default.nix index d42f659..47e30fc 100644 --- a/modules/private/buildbot/default.nix +++ b/modules/private/buildbot/default.nix @@ -184,7 +184,8 @@ in environment = let project_env = with lib.attrsets; mapAttrs' (k: v: nameValuePair "BUILDBOT_${k}" v) project.environment // - mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project); + mapAttrs' (k: v: nameValuePair "BUILDBOT_PATH_${k}" (v pkgs)) (attrByPath ["builderPaths"] {} project) // + { BUILDBOT_PROJECT_DIR = ./projects + "/${project.name}"; }; buildbot_config = pkgs.python3Packages.buildPythonPackage (rec { name = "buildbot_config-${project.name}"; src = ./projects + "/${project.name}"; diff --git a/modules/private/buildbot/projects/immaeEu/__init__.py b/modules/private/buildbot/projects/immaeEu/__init__.py index f29eb14..ad29531 100644 --- a/modules/private/buildbot/projects/immaeEu/__init__.py +++ b/modules/private/buildbot/projects/immaeEu/__init__.py @@ -24,6 +24,8 @@ class E(): COURS_TARBALL_URL = "https://release.immae.eu/cours" IMMAE_EU_RELEASE_PATH = "/var/lib/buildbot/outputs/immaeEu" NORMALESUP_RELEASE_PATH = "/var/lib/buildbot/outputs/recherche" + GSMCELLS_RELEASE_PATH = "/var/lib/ftp/release.immae.eu/gsm_cells" + GSMCELLS_RELEASE_URL = "https://release.immae.eu/gsm_cells" # master.cfg SECRETS_FILE = os.getcwd() + "/secrets" @@ -61,19 +63,45 @@ def configure(c): c['schedulers'].append(hook_scheduler("ImmaeEu", timer=1)) c['schedulers'].append(hook_scheduler("Normalesup", timer=1)) c['schedulers'].append(hook_scheduler("Cours", timer=1)) - c['schedulers'].append(force_scheduler("force_immae_eu", ["ImmaeEu_build", "Normalesup_build", "Cours_build"])) + c['schedulers'].append(schedulers.Nightly(name="GSMCells-weekly", + builderNames=["GSMCells_build"], dayOfWeek=6, hour=3)) + c['schedulers'].append(force_scheduler("force_immae_eu", [ + "ImmaeEu_build", "Normalesup_build", "Cours_build", + ])) + c['schedulers'].append(schedulers.ForceScheduler( + name="GSMCells-force", label="Force build", + buttonName="Force build", + reason=util.StringParameter(name="reason", label="Reason", default="Force build"), + codebases=[ + util.CodebaseParameter("", + branch=util.FixedParameter(name="branch", default=""), + revision=util.FixedParameter(name="revision", default=""), + repository=util.FixedParameter(name="repository", default=""), + project=util.FixedParameter(name="project", default=""), + ), + ], + username=util.FixedParameter(name="username", default="Web button"), + builderNames=["GSMCells_build"] + )) c['builders'].append(immae_eu_factory()) c['builders'].append(normalesup_factory()) c['builders'].append(cours_factory()) + c['builders'].append(gsm_cells_factory()) c['services'].append(SlackStatusPush( name="slack_status_immae_eu_project", - builders=["ImmaeEu_build", "Normalesup_build", "Cours_build"], + builders=[ + "ImmaeEu_build", "Normalesup_build", "Cours_build", + "GSMCells_build" + ], serverUrl=open(E.SECRETS_FILE + "/slack_webhook", "r").read().rstrip())) c['services'].append(XMPPStatusPush( name="xmpp_status_immae_eu_project", - builders=["ImmaeEu_build", "Normalesup_build", "Cours_build"], + builders=[ + "ImmaeEu_build", "Normalesup_build", "Cours_build", + "GSMCells_build" + ], recipients=E.XMPP_RECIPIENTS, password=open(E.SECRETS_FILE + "/notify_xmpp_password", "r").read().rstrip())) @@ -158,3 +186,35 @@ def normalesup_factory(): factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.NORMALESUP_RELEASE_PATH))) return util.BuilderConfig(name="Normalesup_build", workernames=["generic-worker-immae-eu"], factory=factory) + +def gsm_cells_factory(): + path_env = { + "PATH": os.environ["BUILDBOT_PATH_GSMCells"] + ":${PATH}", + "IN_BUILDBOT": "yes", + } + master_env = { + "HTACCESS": ''' + Options +FollowSymLinks + IndexIgnore * + ''' + } + for k, v in os.environ.items(): + if k.startswith("BUILDBOT_GSM_CELLS_"): + path_env[k[len("BUILDBOT_GSM_CELLS_"):]] = v + + script = os.environ["BUILDBOT_PROJECT_DIR"] + "/scripts/lacells_download" + factory = util.BuildFactory() + factory.addStep(steps.ShellCommand(name="download files", + logEnviron=False, haltOnFailure=True, command=[script], env=path_env)) + factory.addStep(steps.ShellCommand(name="give read access to all files", + logEnviron=False, haltOnFailure=True, + command="chmod a+r lacells.db", env=path_env)) + factory.addStep(steps.FileUpload(workersrc="lacells.db", + masterdest=(E.GSMCELLS_RELEASE_PATH+"/lacells.db"), url=(E.GSMCELLS_RELEASE_URL+"/lacells.db"))) + factory.addStep(steps.MasterShellCommand(command="touch {}/.duplicity-ignore".format(E.GSMCELLS_RELEASE_PATH))) + factory.addStep(steps.MasterShellCommand(command='echo "$HTACCESS" > {}/.htaccess'.format(E.GSMCELLS_RELEASE_PATH), + env=master_env)) + factory.addStep(steps.MasterShellCommand(command="ln -sf lacells.db {}/lacells.db.new".format(E.GSMCELLS_RELEASE_PATH))) + factory.addStep(steps.MasterShellCommand(command="chmod -R a+rX {}".format(E.GSMCELLS_RELEASE_PATH))) + + return util.BuilderConfig(name="GSMCells_build", workernames=["generic-worker-immae-eu"], factory=factory) diff --git a/modules/private/buildbot/projects/immaeEu/scripts/lacells_download b/modules/private/buildbot/projects/immaeEu/scripts/lacells_download new file mode 100755 index 0000000..1193cf3 --- /dev/null +++ b/modules/private/buildbot/projects/immaeEu/scripts/lacells_download @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +# FLG - Fast Lacells.db Generator +# +# Simple script to quickly download and generate lacells.db for LocalGSMBackend by n76 +# https://github.com/n76/Local-GSM-Backend +# Uses Mozilla Location Service, OpenCellID and radiocells.org databases as source +# Based on lacells-creator by wvengen and n76 +# +# Licensed under GPLv3 or later +# (C)2016 Sebastian Obrusiewicz +# sobrus@o2.pl + +if [ -z "$IN_BUILDBOT" ]; then +#DEFAULT_CONFIG_BEGIN +ENABLE_OCI=1 #enable OpenCellID data source +ENABLE_MOZ=1 #enable Mozilla Location Services (MLS) data source +ENABLE_RCO=0 #enable radiocells.org data source (it can be quite slow) + +# See https://en.wikipedia.org/wiki/Mobile_country_code +# 208 France +MCC="" #contry codes separated with "|", for example "260|262". Leave dot+asterisk ".*" for all countries +RCO_SRC_FILE="fr.sqlite" #radiocells.org source database file, set "openbmap.sqlite" for entire world database, see https://radiocells.org/downloads for smaller country specific files, for example "pl.sqlite" for Poland +RADIO="" #you can remove LTE if your phone does not support it +TOKEN="" #your OCID token, required to download from OpenCellID. Get your free token at https://opencellid.org/ +fi +TMPDIR='.' #for temporary files only, use disk if you don't have enough RAM, AND remember to have enough disk space in /var/tmp for sqlite temporary files +KEEP_FILES=1 #whether to keep (1) or delete (0) the CSV files after processing + +#do not edit following variables, unless you know what you're doing +EMPTY=',,,,,,,,,,,,,' #dummy empty file for disabled sources +OCI_FILE=$TMPDIR"/ocid.csv" #opencellid temporary file +MOZ_FILE=$TMPDIR"/mozilla.csv" #mozilla temporary file +RCO_FILE=$TMPDIR"/rco.csv" #radiocells.org temporary file +#DEFAULT_CONFIG_END + +#USER_CONFIG_BEGIN +BINDIR=$( dirname "$(readlink -f "$0")" ) #" +if [[ -f "${BINDIR}/config" ]]; then + . "${BINDIR}/config" +fi +#USER_CONFIG_END + +function manage_backup +{ + file=$1 + if [ -s $file ] + then + if [ $KEEP_FILES == "1" ] + then + gzip -kf $file + fi + elif [ -s $file".gz" ] && [ "${file##*.}" == "csv" ] + then + echo "Using" $file".gz backup file" + gzip -dkf $file".gz" + fi +} + + +function download_ocid +{ + URL="https://opencellid.org/ocid/downloads?token=${TOKEN}&type=full&file=cell_towers.csv.gz" + if [ $ENABLE_OCI == "1" ] + then + wget -qO- "$URL" | gunzip | egrep "^($RADIO),($MCC)," > $OCI_FILE + manage_backup $OCI_FILE + else + echo $EMPTY > $OCI_FILE + fi +} + +function download_mozilla +{ + if [ $ENABLE_MOZ == "1" ] + then + NW=`date -u "+%Y-%m-%d"` + wget -qO- "https://d17pt8qph6ncyq.cloudfront.net/export/MLS-full-cell-export-${NW}T000000.csv.gz" | gunzip | egrep "^($RADIO),($MCC)," > $MOZ_FILE + manage_backup $MOZ_FILE + else + echo $EMPTY > $MOZ_FILE + fi +} + +function download_radiocells +{ + if [ $ENABLE_RCO == "1" ] + then + RCO_SELECT="SELECT technology, mcc, mnc, area, cid, NULL, longitude, latitude, 1000 accuracy, measurements, NULL, NULL, NULL, NULL FROM cell_zone;" + wget -qO- "https://cdn.radiocells.org/"$RCO_SRC_FILE > $TMPDIR"/"$RCO_SRC_FILE + sqlite3 -header -csv $TMPDIR"/"$RCO_SRC_FILE "$RCO_SELECT" | egrep "^($RADIO),($MCC)," > $RCO_FILE + rm $TMPDIR"/"$RCO_SRC_FILE + manage_backup $RCO_FILE + else + echo $EMPTY > $RCO_FILE + fi +} + +echo "Downloading data" + +download_ocid & +OP=$! +download_mozilla & +MO=$! +download_radiocells & +RO=$! + +wait $OP +wait $MO +wait $RO + +if [ -s $MOZ_FILE ] && [ -s $OCI_FILE ] && [ -s $RCO_FILE ] +then + +manage_backup lacells.db +rm lacells.db + +echo "Generating database" + +sqlite3 lacells.db <<-SQL +PRAGMA synchronous = OFF; +PRAGMA journal_mode = OFF; + +CREATE TEMP TABLE cells_import (radio TEXT,mcc INTEGER,mnc INTEGER,lac INTEGER,cid INTEGER,unit STRING,longitude NUMERIC,latitude NUMERIC,accuracy INTEGER,samples INTEGER,changeable BOOLEAN,created INTEGER,updated INTEGER, avgSignal INTEGER); +CREATE TABLE cells (mcc INTEGER,mnc INTEGER,lac INTEGER,cid INTEGER,longitude REAL,latitude REAL,altitude REAL,accuracy REAL,samples INTEGER); + +.header on +.mode csv + +.import "$OCI_FILE" cells_import +.import "$MOZ_FILE" cells_import +.import "$RCO_FILE" cells_import + +UPDATE cells_import SET samples=1 WHERE samples IS NULL OR samples < 1; + +INSERT INTO cells +SELECT mcc, mnc, lac, cid, + sum(longitude * samples) / sum(samples) as longitude, + sum(latitude * samples) / sum(samples) as latitude, + -1 as altitude, + sum(accuracy * samples) / sum(samples) as accuracy, + sum(samples) as samples +FROM cells_import +GROUP BY mcc, mnc, lac, cid; + +DROP TABLE cells_import; + +UPDATE cells SET accuracy=500 WHERE accuracy < 500; +UPDATE cells SET accuracy=100000 WHERE accuracy > 100000; + +CREATE INDEX _idx1 ON cells (mcc, mnc, lac, cid); +CREATE INDEX _idx2 ON cells (lac, cid); + +VACUUM; +SQL + +else + echo "Download error" +fi + +rm $OCI_FILE +rm $MOZ_FILE +rm $RCO_FILE -- 2.41.0