cerxes/openmaptiles
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use v4.1, move import-wikidata before import-sql (#805)

Browse Source 
import-wikidata should run before import-sql, and v4.1 allows for that.
Also it optimizes the wd_names table to just be a simple Wikidata ID -> labels lookup, with a proper index.

Minor other changes:
* `test-perf-null` target is now part of the Makefile
* `./data/osmstat.txt` is no longer created
* area download file now in this format:   `${osm_area}-latest.osm.pbf`
This commit is contained in:
Yuri Astrakhan
2020-04-09 10:49:35 -04:00
committed by GitHub
parent f55579cd16
commit 4f5ea5f26e
4 changed files with 43 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
quickstart.sh
View File

@@ -27,7 +27,7 @@ if [ $# -eq 0 ]; then
else
osm_area=$1
fi
testdata="${osm_area}.osm.pbf"
testdata="${osm_area}-latest.osm.pbf"
## Min versions ...
MIN_COMPOSE_VER=1.7.1
@@ -36,11 +36,6 @@ STARTTIME=$(date +%s)
STARTDATE=$(date +"%Y-%m-%dT%H:%M%z")
githash=$( git rev-parse HEAD )
# Options to run with docker and docker-compose - ensure the container is destroyed on exit,
# as well as pass any other common parameters.
# In the future this should use -u $(id -u "$USER"):$(id -g "$USER") instead of running docker as root.
DC_OPTS="--rm -u $(id -u "$USER"):$(id -g "$USER")"
log_file=./quickstart.log
rm -f $log_file
echo " "
@@ -138,21 +133,12 @@ echo "--------------------------------------------------------------------------
echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) "
rm -f ./data/*.mbtiles
if [ ! -f "./data/${testdata}" ]; then
if [[ ! -f "./data/${testdata}" || ! -f "./data/docker-compose-config.yml" ]]; then
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Downloading testdata $testdata"
rm -f ./data/*
#wget $testdataurl -P ./data
echo "====> : Downloading ${osm_area} from Geofabrik..."
rm -rf ./data/*
make download-geofabrik "area=${osm_area}"
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Osm metadata : $testdata"
cat ./data/osmstat.txt
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Generated docker-compose config"
cat ./data/docker-compose-config.yml
else
echo " "
echo "-------------------------------------------------------------------------------------"
@@ -236,7 +222,14 @@ make import-borders
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL "
echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL"
echo " : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page "
echo " : Thank you Wikidata Contributors ! "
make import-wikidata
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start SQL postprocessing: ./build/sql/* -> PostgreSQL "
echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/blob/master/bin/import-sql"
# If the output contains a WARNING, stop further processing
# Adapted from https://unix.stackexchange.com/questions/307562
@@ -248,17 +241,10 @@ echo "--------------------------------------------------------------------------
echo "====> : Analyze PostgreSQL tables"
make psql-analyze
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL"
echo " : The Wikidata license: CC0 - https://www.wikidata.org/wiki/Wikidata:Main_Page "
echo " : Thank you Wikidata Contributors ! "
make import-wikidata
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Testing PostgreSQL tables to match layer definitions metadata"
docker-compose run $DC_OPTS openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color
make test-perf-null
echo " "
echo "-------------------------------------------------------------------------------------"
@@ -289,7 +275,6 @@ md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk
md5sum "./data/${testdata}" >> ./data/quickstart_checklist.chk
md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk
md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk
md5sum ./data/osmstat.txt >> ./data/quickstart_checklist.chk
cat ./data/quickstart_checklist.chk
ENDTIME=$(date +%s)