cerxes/openmaptiles
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

more improvments

Browse Source
This commit is contained in:
ImreSamu 2016-11-28 23:15:47 +01:00
parent ceb66e638c
commit 355f4baf57
2 changed files with 118 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
Makefile
View File

@ -57,9 +57,55 @@ clean:
clean_build: clean_build:
rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql
clean-docker:
docker-compose down -v --remove-orphans
docker-compose rm -fv
docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true
list-docker-images:
docker images | grep openmaptiles
refresh-docker-images:
docker pull openmaptiles/generate-vectortiles
docker pull openmaptiles/import-lakelines
docker pull openmaptiles/import-natural-earth
docker pull openmaptiles/import-osm
docker pull openmaptiles/import-sql
docker pull openmaptiles/import-water
docker pull openmaptiles/openmaptiles-tools
docker pull openmaptiles/postgis
docker pull osm2vectortiles/mapbox-studio
remove-docker-images:
docker rmi openmaptiles/generate-vectortiles
docker rmi openmaptiles/import-lakelines
docker rmi openmaptiles/import-natural-earth
docker rmi openmaptiles/import-osm
docker rmi openmaptiles/import-sql
docker rmi openmaptiles/import-water
docker rmi openmaptiles/openmaptiles-tools
docker rmi openmaptiles/postgis
docker rmi osm2vectortiles/mapbox-studio
psql: psql:
docker-compose run --rm import-osm /usr/src/app/psql.sh docker-compose run --rm import-osm /usr/src/app/psql.sh
psql-list-tables:
docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c "\d+"
psql-pg-stat-reset:
docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c 'SELECT pg_stat_statements_reset();'
forced-clean-sql:
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE"
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE SCHEMA IF NOT EXISTS public"
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION hstore"
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION postgis"
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION pg_stat_statements"
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO postgres;"
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO public;"
docker-compose run --rm import-osm /usr/src/app/psql.sh -c "COMMENT ON SCHEMA public IS 'standard public schema';"
import-sql-dev: import-sql-dev:
docker-compose run --rm import-sql /bin/bash docker-compose run --rm import-sql /bin/bash
@ -91,5 +137,4 @@ etlgraph:
generate-etlgraph layers/place/place.yaml generate-etlgraph layers/place/place.yaml
generate-etlgraph layers/railway/railway.yaml generate-etlgraph layers/railway/railway.yaml
generate-etlgraph layers/water_name/water_name.yaml generate-etlgraph layers/water_name/water_name.yaml

123
quickstart.sh
View File

@ -3,42 +3,41 @@ set -o errexit
set -o pipefail set -o pipefail
set -o nounset set -o nounset
###########################################
# OpenMapTiles quickstart.sh for x86_64 linux
#
# Example calls ... # Example calls ...
# ./quickstart.sh # ./quickstart.sh
# ./quickstart.sh africa # ./quickstart.sh africa
# ./quickstart.sh alabama # ./quickstart.sh alabama
# ./quickstart.sh alaska # ./quickstart.sh alaska
# ./quickstart.sh albania # ./quickstart.sh albania
# ./quickstart.sh alberta # ./quickstart.sh alberta
# ./quickstart.sh alps # ./quickstart.sh alps
# .... # ....
# #
# to list areas : make download-geofabrik-list # to list areas : make download-geofabrik-list
# see more QUICKSTART.md # see more QUICKSTART.md
# #
if [ $# -eq 0 ]; then if [ $# -eq 0 ]; then
echo "No parameter - set area=albania " osm_area=albania # default test country
osm_area=albania echo "No parameter - set area=$osm_area "
else else
osm_area=$1 osm_area=$1
fi fi
testdata=${osm_area}.osm.pbf testdata=${osm_area}.osm.pbf
## Min versions ...
##
## OpenMapTiles quickstart.sh for x86_64 linux
##
MIN_COMPOSE_VER=1.7.1 MIN_COMPOSE_VER=1.7.1
MIN_DOCKER_VER=1.10.0 MIN_DOCKER_VER=1.10.0
STARTTIME=$(date +%s) STARTTIME=$(date +%s)
STARTDATE=$(date -Iminutes) STARTDATE=$(date -Iminutes)
githash=$( git rev-parse HEAD ) githash=$( git rev-parse HEAD )
log_file=quickstart.log log_file=./quickstart.log
rm -f $log_file rm -f $log_file
exec &> >(tee -a "$log_file") exec &> >(tee -a "$log_file")
@ -94,6 +93,10 @@ if [ $DOCKER_VER "<" $MIN_DOCKER_VER ]; then
exit 1 exit 1
fi fi
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Pulling or Refreshing OpenMapTiles docker images "
#make refresh-docker-images
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
@ -103,18 +106,12 @@ docker images | grep openmaptiles
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo "====> : Stopping running services & removing old containers " echo "====> : Stopping running services & removing old containers "
docker-compose down make clean-docker
docker-compose kill
docker-compose rm -fv
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo "====> : For a clean start, we are removing old postgresql data volume ( if it exists )" echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata ) "
docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true mkdir -p pgdata
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Making directories - if they don't exist ( ./build ./data ) "
mkdir -p build mkdir -p build
mkdir -p data mkdir -p data
@ -123,6 +120,12 @@ echo "--------------------------------------------------------------------------
echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) "
rm -f ./data/*.mbtiles rm -f ./data/*.mbtiles
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Removing pgdata "
# rm -rf ./pgdata/*
if [ ! -f ./data/${testdata} ]; then if [ ! -f ./data/${testdata} ]; then
echo " " echo " "
@ -131,22 +134,28 @@ if [ ! -f ./data/${testdata} ]; then
rm -f ./data/* rm -f ./data/*
#wget $testdataurl -P ./data #wget $testdataurl -P ./data
docker-compose run --rm import-osm ./download-geofabrik.sh ${osm_area} docker-compose run --rm import-osm ./download-geofabrik.sh ${osm_area}
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Osm metadata : $testdata "
cat ./data/osmstat.txt
echo " "
echo "-------------------------------------------------------------------------------------"
echo "====> : Generated docker-compose config "
cat ./data/docker-compose-config.yml
else else
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo "====> : The testdata ./data/$testdata exists, we don't need to download! " echo "====> : The testdata ./data/$testdata exists, we don't need to download! "
fi fi
if [ ! -f ./data/${testdata} ]; then if [ ! -f ./data/${testdata} ]; then
echo " " echo " "
echo "Missing ./data/$testdata , Download error? " echo "Missing ./data/$testdata , Download or Parameter error? "
exit 404 exit 404
fi fi
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) " echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) "
@ -165,7 +174,9 @@ echo "====> : Start PostgreSQL service ; create PostgreSQL data volume "
echo " : Source code: https://github.com/openmaptiles/postgis " echo " : Source code: https://github.com/openmaptiles/postgis "
echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !"
docker-compose up -d postgres docker-compose up -d postgres
sleep 30
# Drop all PostgreSQL tables ...
make forced-clean-sql
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
@ -198,7 +209,7 @@ echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/inde
echo " : Thank you Omniscale! " echo " : Thank you Omniscale! "
echo " : Source code: https://github.com/openmaptiles/import-osm " echo " : Source code: https://github.com/openmaptiles/import-osm "
echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) "
echo " : Thank you OpenStreetMap Contributors ! " echo " : Thank you OpenStreetMap Contributors ! "
docker-compose run --rm import-osm docker-compose run --rm import-osm
echo " " echo " "
@ -230,16 +241,17 @@ docker-compose stop postgres
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo "====> : Inputs - Outputs md5sum for debugging " echo "====> : Inputs - Outputs md5sum for debugging "
rm -f quickstart_checklist.chk rm -f ./data/quickstart_checklist.chk
md5sum build/mapping.yaml >> quickstart_checklist.chk md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk
md5sum build/tileset.sql >> quickstart_checklist.chk md5sum build/tileset.sql >> ./data/quickstart_checklist.chk
md5sum build/openmaptiles.tm2source/data.yml >> quickstart_checklist.chk md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk
md5sum ./data/${testdata} >> quickstart_checklist.chk md5sum ./data/${testdata} >> ./data/quickstart_checklist.chk
md5sum ./data/tiles.mbtiles >> quickstart_checklist.chk md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk
md5sum ./data/docker-compose-config.yml >> quickstart_checklist.chk md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk
cat quickstart_checklist.chk md5sum ./data/osmstat.txt >> ./data/quickstart_checklist.chk
cat ./data/quickstart_checklist.chk
ENDTIME=$(date +%s) ENDTIME=$(date +%s)
ENDDATE=$(date -Iminutes) ENDDATE=$(date -Iminutes)
MODDATE=$(stat -c %y ./data/${testdata} ) MODDATE=$(stat -c %y ./data/${testdata} )
@ -254,11 +266,12 @@ echo "====> : (disk space) We have created a lot of docker images: "
echo " : Hint: you can remove with: docker rmi IMAGE " echo " : Hint: you can remove with: docker rmi IMAGE "
docker images | grep openmaptiles docker images | grep openmaptiles
echo " " #
echo "-------------------------------------------------------------------------------------" # echo " "
echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" # echo "-------------------------------------------------------------------------------------"
echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " # echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:"
docker volume ls -q | grep openmaptiles # echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata "
# docker volume ls -q | grep openmaptiles
echo " " echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
@ -273,7 +286,7 @@ echo "--------------------------------------------------------------------------
echo "The ./quickstart.sh $osm_area is finished! " echo "The ./quickstart.sh $osm_area is finished! "
echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete"
echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !" echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !"
echo " " echo " "
echo "Start experimenting ! And check the QUICKSTART.MD file !" echo "Start experimenting ! And check the QUICKSTART.MD file !"
echo " " echo " "
echo "Hints for testing other areas" echo "Hints for testing other areas"
@ -283,17 +296,23 @@ echo " "
echo "Hints for designers:" echo "Hints for designers:"
echo " .... # start Maputnik " echo " .... # start Maputnik "
echo " .... # start Tileserver-gl-light" echo " .... # start Tileserver-gl-light"
echo " make start-mapbox-studio # start Mapbox Studio " echo " make start-mapbox-studio # start Mapbox Studio"
echo " " echo " "
echo "Hints for developers:" echo "Hints for developers:"
echo " make download-geofabrik area=albania # download OSM data from geofabrik area=albania" echo " make download-geofabrik area=albania # download OSM data from geofabrik area=albania"
echo " make psql # start PostgreSQL console " echo " make psql # start PostgreSQL console "
echo " make psql-list-tables # list all PostgreSQL tables "
echo " make import-sql-dev # start import-sql /bin/bash terminal " echo " make import-sql-dev # start import-sql /bin/bash terminal "
echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)"
echo " .... # start lukasmartinelli/postgis-editor" echo " make clean-docker # remove docker containers, PG data volume "
echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment "
echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB"
echo " make remove-docker-images # remove openmaptiles docker images"
echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations"
echo " cat quickstart.log # backup ./quickstart.log for Technical Support :) " echo " cat ./quickstart.log # backup of the last ./quickstart.sh "
echo " .... # start lukasmartinelli/postgis-editor"
echo " "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"
echo " Acknowledgments " echo " Acknowledgments "
echo " Thanks to all free, open source software developers and Open Data Contributors ! " echo " Thanks to all free, open source software developers and Open Data Contributors ! "
echo "-------------------------------------------------------------------------------------" echo "-------------------------------------------------------------------------------------"