diff --git a/.env b/.env index 6a65572..11b9f68 100644 --- a/.env +++ b/.env @@ -3,3 +3,6 @@ POSTGRES_USER=openmaptiles POSTGRES_PASSWORD=openmaptiles POSTGRES_HOST=postgres POSTGRES_PORT=5432 +QUICKSTART_MIN_ZOOM=0 +QUICKSTART_MAX_ZOOM=7 +DIFF_MODE=false \ No newline at end of file diff --git a/.gitignore b/.gitignore index e5d43ba..d0244cc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,13 @@ data.xml *.thumb.png *.swp *.pbf + +# quickstart +quickstart.log + + +# imput / output data +data/* + +# generated source files +build/* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 8bb8d4d..aa9894c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,9 @@ before_install: - sudo apt-get clean - sudo apt-get update - sudo apt-key update - - sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.12.0-0~precise + - sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.10.0-0~precise - sudo rm -f /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/1.8.0/docker-compose-`uname -s`-`uname -m` > docker-compose + - curl -L https://github.com/docker/compose/releases/download/1.7.1/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - sudo service docker restart diff --git a/Makefile b/Makefile index 8ff02da..937060b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,40 @@ all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql +help: + @echo "==============================================================================" + @echo " OpenMapTiles https://github.com/openmaptiles/openmaptiles " + @echo "Hints for testing areas " + @echo " make download-geofabrik-list # list actual geofabrik OSM extracts for download -> <> " + @echo " make list # list actual geofabrik OSM extracts for download -> <> " + @echo " ./quickstart.sh <> # example: ./quickstart.sh madagascar " + @echo " " + @echo "Hints for designers:" + @echo " ....TODO.... # start Maputnik " + @echo " ....TODO.... # start Tileserver-gl-light" + @echo " make start-mapbox-studio # start Mapbox Studio" + @echo " " + @echo "Hints for developers:" + @echo " make # build source code " + @echo " make download-geofabrik area=albania # download OSM data from geofabrik, and create config file" + @echo " make psql # start PostgreSQL console " + @echo " make psql-list-tables # list all PostgreSQL tables " + @echo " make generate-qareports # generate reports [./build/qareports]" + @echo " make generate-devdoc # generate devdoc [./build/devdoc]" + @echo " make import-sql-dev # start import-sql /bin/bash terminal " + @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" + @echo " make clean-docker # remove docker containers, PG data volume " + @echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment " + @echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)" + @echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" + @echo " make remove-docker-images # remove openmaptiles docker images" + @echo " make pgclimb-list-views # list PostgreSQL public schema views" + @echo " make pgclimb-list-tables # list PostgreSQL public schema tabless" + @echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" + @echo " cat ./quickstart.log # backup of the last ./quickstart.sh " + @echo " ....TODO.... # start lukasmartinelli/postgis-editor" + @echo " make help # help about avaialable commands" + @echo "==============================================================================" + build/openmaptiles.tm2source/data.yml: mkdir -p build/openmaptiles.tm2source && generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > build/openmaptiles.tm2source/data.yml @@ -11,3 +46,97 @@ build/tileset.sql: clean: rm -f build/openmaptiles.tm2source/data.yml && rm -f build/mapping.yaml && rm -f build/tileset.sql + +clean-docker: + docker-compose down -v --remove-orphans + docker-compose rm -fv + docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true + +list-docker-images: + docker images | grep openmaptiles + +refresh-docker-images: + docker-compose pull --ignore-pull-failures + +remove-docker-images: + @echo "Deleting all openmaptiles related docker image(s)..." + @docker-compose down + @docker images | grep "" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + @docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + +docker-unnecessary-clean: + @echo "Deleting unnecessary container(s)..." + @docker ps -a | grep Exited | awk -F" " '{print $$1}' | xargs --no-run-if-empty docker rm + @echo "Deleting unnecessary image(s)..." + @docker images | grep \ | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + +psql: + docker-compose run --rm import-osm /usr/src/app/psql.sh + +psql-list-tables: + docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c "\d+" + +psql-pg-stat-reset: + docker-compose run --rm import-osm /usr/src/app/psql.sh -P pager=off -c 'SELECT pg_stat_statements_reset();' + +forced-clean-sql: + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "DROP SCHEMA IF EXISTS public CASCADE ; CREATE SCHEMA IF NOT EXISTS public; " + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION pg_stat_statements;" + docker-compose run --rm import-osm /usr/src/app/psql.sh -c "GRANT ALL ON SCHEMA public TO public;COMMENT ON SCHEMA public IS 'standard public schema';" + +pgclimb-list-views: + docker-compose run --rm import-osm /usr/src/app/pgclimb.sh -c "select schemaname,viewname from pg_views where schemaname='public' order by viewname;" csv + +pgclimb-list-tables: + docker-compose run --rm import-osm /usr/src/app/pgclimb.sh -c "select schemaname,tablename from pg_tables where schemaname='public' order by tablename;" csv + +import-sql-dev: + docker-compose run --rm import-sql /bin/bash + +import-osm-dev: + docker-compose run --rm import-osm /bin/bash + +download-geofabrik: + @echo =============== download-geofabrik ======================= + @echo Download area : $(area) + @echo [[ example: make download-geofabrik area=albania ]] + @echo [[ list areas: make download-geofabrik-list ]] + docker-compose run --rm import-osm ./download-geofabrik.sh $(area) + ls -la ./data/$(area).* + @echo "Generated config file: ./data/docker-compose-config.yml" + @echo " " + cat ./data/docker-compose-config.yml + @echo " " + +# the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command +list: + docker-compose run --rm import-osm ./download-geofabrik-list.sh + +# same as a `make list` +download-geofabrik-list: + docker-compose run --rm import-osm ./download-geofabrik-list.sh + +start-mapbox-studio: + docker-compose up mapbox-studio + +generate-qareports: + ./qa/run.sh + +# work in progress - please don't remove +generate-devdoc: + mkdir -p ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/aeroway/aeroway.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/boundary/boundary.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/building/building.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/housenumber/housenumber.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landcover/landcover.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/landuse/landuse.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/park/park.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/place/place.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/poi/poi.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation/transportation.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/transportation_name/transportation_name.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water/water.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/water_name/water_name.yaml ./build/devdoc + docker run --rm -v $$(pwd):/tileset openmaptiles/openmaptiles-tools generate-etlgraph layers/waterway/waterway.yaml ./build/devdoc diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000..36abe18 --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,294 @@ + + +## Quickstart - for small extracts + +### Req: +* CPU: AMD64 ( = Intel 64 bit) + * The base docker debian images are x86_64 based, so the ARM,MIPS currently not supported! +* Operating system + * Linux is suggested + * The development and the testing platform is Linux. + * If you are using FreeBSD, Solaris, Windows, ... + * Please give a feedback, share your experience, write a tutorial +* bash +* git +* make +* docker >=1.10 + * https://www.docker.com/products/overview +* docker-compose >=1.7.1 + * https://docs.docker.com/compose/install/ +* disk space ( >= ~15Gb ) + * for small extracts >= ~15Gb + * for big extracts ( continents, planet) > 20 ... 1000 Gb + * And depends on + * OpenStreetMap data size + * Zoom level +* memory ( >= 3Gb ) + * for small extracts 3Gb-8Gb RAM + * for big extracts ( Europe, Planet) > 8-32 Gb +* internet connections + * for downloading docker images + * for downloading OpenStreetMap data from Geofabrik + +Important: The ./quickstart.sh is for small extracts - not optimal for a Planet rendering !! + +### First experiment - with `albania` ( small extracts! ) + +```bash +git clone https://github.com/openmaptiles/openmaptiles.git +cd openmaptiles +./quickstart.sh +``` + +If you have problems with the quickstart +* check the ./quickstart.log! +* check again the system requirements! +* check the current issues : https://github.com/openmaptiles/openmaptiles/issues +* create new issues: + * create a new gist https://gist.github.com/ from your ./quickstart.log + * double check: if this is not contain any sensitive informations about your system + * create a new issues: https://github.com/openmaptiles/openmaptiles/issues + * describe the problems + * add any important informations your environment + * and link your (quickstart.log) gist ! + +### Check other extracts + +IF the previous step is working, +THAN you can test other available quickstart extracts ( based on geofabrik extracts) ! + * We are using https://github.com/julien-noblet/download-geofabrik tool + * The current extract list, and more informations -> `make list` + +This is generating `.mbtiles` for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ] + +```bash +./quickstart.sh africa +./quickstart.sh alabama +./quickstart.sh alaska +./quickstart.sh albania +./quickstart.sh alberta +./quickstart.sh alps +./quickstart.sh alsace +./quickstart.sh andorra +./quickstart.sh antarctica +./quickstart.sh aquitaine +./quickstart.sh argentina +./quickstart.sh arizona +./quickstart.sh arkansas +./quickstart.sh asia +./quickstart.sh australia +./quickstart.sh austria +./quickstart.sh auvergne +./quickstart.sh azerbaijan +./quickstart.sh azores +./quickstart.sh bangladesh +./quickstart.sh bayern +./quickstart.sh belarus +./quickstart.sh belgium +./quickstart.sh belize +./quickstart.sh berlin +./quickstart.sh bolivia +./quickstart.sh botswana +./quickstart.sh bourgogne +./quickstart.sh brandenburg +./quickstart.sh brazil +./quickstart.sh bremen +./quickstart.sh bretagne +./quickstart.sh buckinghamshire +./quickstart.sh bulgaria +./quickstart.sh california +./quickstart.sh cambridgeshire +./quickstart.sh cameroon +./quickstart.sh canada +./quickstart.sh centre +./quickstart.sh cheshire +./quickstart.sh chile +./quickstart.sh china +./quickstart.sh colombia +./quickstart.sh colorado +./quickstart.sh connecticut +./quickstart.sh cornwall +./quickstart.sh corse +./quickstart.sh croatia +./quickstart.sh cuba +./quickstart.sh cumbria +./quickstart.sh cyprus +./quickstart.sh dach +./quickstart.sh delaware +./quickstart.sh denmark +./quickstart.sh derbyshire +./quickstart.sh devon +./quickstart.sh dorset +./quickstart.sh ecuador +./quickstart.sh egypt +./quickstart.sh england +./quickstart.sh essex +./quickstart.sh estonia +./quickstart.sh ethiopia +./quickstart.sh europe +./quickstart.sh fiji +./quickstart.sh finland +./quickstart.sh florida +./quickstart.sh france +./quickstart.sh germany +./quickstart.sh gloucestershire +./quickstart.sh greece +./quickstart.sh greenland +./quickstart.sh guadeloupe +./quickstart.sh guatemala +./quickstart.sh guinea +./quickstart.sh guyane +./quickstart.sh hamburg +./quickstart.sh hampshire +./quickstart.sh hawaii +./quickstart.sh herefordshire +./quickstart.sh hertfordshire +./quickstart.sh hessen +./quickstart.sh hungary +./quickstart.sh iceland +./quickstart.sh idaho +./quickstart.sh illinois +./quickstart.sh india +./quickstart.sh indiana +./quickstart.sh indonesia +./quickstart.sh iowa +./quickstart.sh irak +./quickstart.sh iran +./quickstart.sh italy +./quickstart.sh japan +./quickstart.sh jordan +./quickstart.sh kansas +./quickstart.sh kazakhstan +./quickstart.sh kent +./quickstart.sh kentucky +./quickstart.sh kenya +./quickstart.sh kosovo +./quickstart.sh kyrgyzstan +./quickstart.sh lancashire +./quickstart.sh latvia +./quickstart.sh lebanon +./quickstart.sh leicestershire +./quickstart.sh lesotho +./quickstart.sh liberia +./quickstart.sh libya +./quickstart.sh liechtenstein +./quickstart.sh limousin +./quickstart.sh lithuania +./quickstart.sh lorraine +./quickstart.sh louisiana +./quickstart.sh luxembourg +./quickstart.sh macedonia +./quickstart.sh madagascar +./quickstart.sh maine +./quickstart.sh malta +./quickstart.sh manitoba +./quickstart.sh martinique +./quickstart.sh maryland +./quickstart.sh massachusetts +./quickstart.sh mayotte +./quickstart.sh mexico +./quickstart.sh michigan +./quickstart.sh minnesota +./quickstart.sh mississippi +./quickstart.sh missouri +./quickstart.sh mittelfranken +./quickstart.sh moldova +./quickstart.sh monaco +./quickstart.sh mongolia +./quickstart.sh montana +./quickstart.sh montenegro +./quickstart.sh morocco +./quickstart.sh nebraska +./quickstart.sh nepal +./quickstart.sh netherlands +./quickstart.sh nevada +./quickstart.sh niederbayern +./quickstart.sh niedersachsen +./quickstart.sh nigeria +./quickstart.sh norfolk +./quickstart.sh norway +./quickstart.sh nottinghamshire +./quickstart.sh nunavut +./quickstart.sh oberbayern +./quickstart.sh oberfranken +./quickstart.sh oberpfalz +./quickstart.sh ohio +./quickstart.sh oklahoma +./quickstart.sh ontario +./quickstart.sh oregon +./quickstart.sh oxfordshire +./quickstart.sh pakistan +./quickstart.sh paraguay +./quickstart.sh pennsylvania +./quickstart.sh peru +./quickstart.sh philippines +./quickstart.sh picardie +./quickstart.sh poland +./quickstart.sh portugal +./quickstart.sh quebec +./quickstart.sh reunion +./quickstart.sh romania +./quickstart.sh saarland +./quickstart.sh sachsen +./quickstart.sh saskatchewan +./quickstart.sh schwaben +./quickstart.sh scotland +./quickstart.sh serbia +./quickstart.sh shropshire +./quickstart.sh slovakia +./quickstart.sh slovenia +./quickstart.sh somalia +./quickstart.sh somerset +./quickstart.sh spain +./quickstart.sh staffordshire +./quickstart.sh suffolk +./quickstart.sh surrey +./quickstart.sh sweden +./quickstart.sh switzerland +./quickstart.sh syria +./quickstart.sh taiwan +./quickstart.sh tajikistan +./quickstart.sh tanzania +./quickstart.sh tennessee +./quickstart.sh texas +./quickstart.sh thailand +./quickstart.sh thueringen +./quickstart.sh turkey +./quickstart.sh turkmenistan +./quickstart.sh ukraine +./quickstart.sh unterfranken +./quickstart.sh uruguay +./quickstart.sh us +./quickstart.sh utah +./quickstart.sh uzbekistan +./quickstart.sh vermont +./quickstart.sh vietnam +./quickstart.sh virginia +./quickstart.sh wales +./quickstart.sh washington +./quickstart.sh wiltshire +./quickstart.sh wisconsin +./quickstart.sh wyoming +./quickstart.sh yukon +``` + +### Check other commands + +`make help` + + +### Change MIN_ZOOM and MAX_ZOOM + +modify the settings in the `.env` file +* QUICKSTART_MIN_ZOOM=0 +* QUICKSTART_MAX_ZOOM=7 + +and re-start `./quickstart.sh ` +* the new config file re-generating to here ./data/docker-compose-config.yml +* Known problems: + * If you use same area - then the ./data/docker-compose-config.yml not re-generating, so you have to modify by hand! + +Hints: +* Small increments! Never starts with the MAX_ZOOM = 14 +* The suggested MAX_ZOOM = 14 - use only with small extracts + diff --git a/docker-compose-test-override.yml b/docker-compose-test-override.yml deleted file mode 100644 index 0ae9e4a..0000000 --- a/docker-compose-test-override.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: "2" - -services: - generate-vectortiles: - environment: - BBOX: "8.25,46.97,9.58,47.52" - MIN_ZOOM: "0" - MAX_ZOOM: "7" diff --git a/docker-compose.yml b/docker-compose.yml index 1c21b4a..e6d7970 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: links: - postgres import-osm: - image: "openmaptiles/import-osm:0.1" + image: "openmaptiles/import-osm:latest" env_file: .env environment: DIFF_MODE: ${DIFF_MODE} @@ -43,6 +43,14 @@ services: - postgres volumes: - ./build:/sql + openmaptiles-tools: + image: "openmaptiles/openmaptiles-tools:latest" + env_file: .env + links: + - postgres + volumes: + - .:/tileset + - ./build:/sql mapbox-studio: image: "osm2vectortiles/mapbox-studio" volumes: diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh new file mode 100755 index 0000000..1ba1a90 --- /dev/null +++ b/qa/layer_freq.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +layerid=$1 +classvars=$2 + +echo "# FREQ - $layerid group by : $classvars " + +for z in {0..15} +do +echo " " +echo "## $layerid z$z - freq" + +SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) + +SQLCODE=$(cat <<-END +select $classvars , count(*) as _count_ from +( $SQL ) as t +GROUP BY $classvars +ORDER BY $classvars +; +END +) + +#echo "\`\`\`sql" +#echo "$SQLCODE" +#echo "\`\`\`" + +docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ + | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' + +done + diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh new file mode 100755 index 0000000..e786724 --- /dev/null +++ b/qa/layer_numvar_analyze.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +layerid=$1 +var=$2 + +echo "# NUMVAR-ANALYZE - $layerid - $var " + +for z in {0..15} +do +echo " " +echo "## $layerid z$z - $var " + +SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) + +SQLCODE=$(cat <<-END +SELECT + count($var) as count + ,min($var) as min + ,max($var) as max + ,avg($var) as avg + ,stddev($var) as stddev + ,variance($var) as variance +FROM +( $SQL ) as t +; +END +) + + +#echo "\`\`\`sql" +#echo "$SQLCODE" +#echo "\`\`\`" + +docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ + | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' + +done diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh new file mode 100755 index 0000000..a72e902 --- /dev/null +++ b/qa/layer_toplength.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +layerid=$1 +classvar=$2 + +for z in {0..15} +do +echo " " +echo "## $layerid z$z max length ($classvar)" + +SQL=$(docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) + +SQLCODE=$(cat <<-END +SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from +( $SQL ) as t +WHERE length( $classvar ) > 0 +ORDER BY length( $classvar ) DESC NULLS LAST +LIMIT 30 +; +END +) + +#echo "\`\`\`sql" +#echo "$SQLCODE" +#echo "\`\`\`" + +docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ + | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' + +done + diff --git a/qa/run.sh b/qa/run.sh new file mode 100755 index 0000000..ea9f573 --- /dev/null +++ b/qa/run.sh @@ -0,0 +1,83 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +# example call from the parent folder : ./qa/run.sh +# or make generate-qareports + +# ---- freq --------------------------------------------------------------- + +mkdir -p ./build/qareports +rm -f ./build/qareports/*.md + +# ----- + +./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md + +###Todo:./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md +./qa/layer_freq.sh boundary "admin_level" > ./build/qareports/freq_boundary__admin_level.md +###Todo: ./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md + +./qa/layer_freq.sh building "render_min_height" > ./build/qareports/freq_building__render_min_height.md + +#./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md +./qa/layer_freq.sh landcover "class, subclass" > ./build/qareports/freq_landcover__class_subclass.md + +./qa/layer_freq.sh landuse "class " > ./build/qareports/freq_landuse__class.md + +./qa/layer_freq.sh park "class " > ./build/qareports/freq_park__class.md + +./qa/layer_freq.sh place "class " > ./build/qareports/freq_place__class.md +./qa/layer_freq.sh place "class,capital " > ./build/qareports/freq_place__class_capital.md +./qa/layer_freq.sh place "capital " > ./build/qareports/freq_place__capital.md +./qa/layer_freq.sh place "class,capital,rank " > ./build/qareports/freq_place__class_capital_rank.md +./qa/layer_freq.sh place "rank " > ./build/qareports/freq_place__rank.md + +./qa/layer_freq.sh poi "class " > ./build/qareports/freq_poi__class.md +./qa/layer_freq.sh poi "subclass " > ./build/qareports/freq_poi__subclass.md +./qa/layer_freq.sh poi "rank " > ./build/qareports/freq_poi__rank.md +./qa/layer_freq.sh poi "class,subclass " > ./build/qareports/freq_poi__class_subclass.md +./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qareports/freq_poi__class_subclass_rank.md +./qa/layer_freq.sh poi "class,rank " > ./build/qareports/freq_poi__class_rank.md + +./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qareports/freq_transportation__class_oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qareports/freq_transportation__oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "class " > ./build/qareports/freq_transportation__class.md + +./qa/layer_freq.sh transportation_name "class " > ./build/qareports/freq_transportation_name__class.md +./qa/layer_freq.sh transportation_name "ref_length" > ./build/qareports/freq_transportation_name__ref_length.md + +./qa/layer_freq.sh water "class " > ./build/qareports/freq_water__class.md + +./qa/layer_freq.sh water_name "class " > ./build/qareports/freq_water_name__class.md + +./qa/layer_freq.sh waterway "class " > ./build/qareports/freq_waterway__class.md + +# ---- toplength ------------------------------------------- + +./qa/layer_toplength.sh housenumber "housenumber" > ./build/qareports/toplength_housenumber__housenumber.md + +./qa/layer_toplength.sh place "name" > ./build/qareports/toplength_place__name.md +./qa/layer_toplength.sh place "name_en" > ./build/qareports/toplength_place__name_en.md + +./qa/layer_toplength.sh poi "name" > ./build/qareports/toplength_poi__name.md +./qa/layer_toplength.sh poi "name_en" > ./build/qareports/toplength_poi__name_en.md + +./qa/layer_toplength.sh transportation_name "name" > ./build/qareports/toplength_transportation_name__name.md +./qa/layer_toplength.sh transportation_name "ref" > ./build/qareports/toplength_transportation_name__ref.md +###Todo: ./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md + +./qa/layer_toplength.sh water_name "name" > ./build/qareports/toplength_water_name__name.md +./qa/layer_toplength.sh water_name "name_en" > ./build/qareports/toplength_water_name__name_en.md + +./qa/layer_toplength.sh waterway "name" > ./build/qareports/toplength_waterway__name.md + + +# ---- numvar analyze ------------------------------------- + +./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qareports/numvara_building__render_min_height.md +./qa/layer_numvar_analyze.sh building "render_height" > ./build/qareports/numvara_building__render_height.md + +./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qareports/numvara_transportation_name__ref_length.md + diff --git a/quickstart.sh b/quickstart.sh index 8dda212..404478a 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -3,44 +3,306 @@ set -o errexit set -o pipefail set -o nounset -DOCKER_EXEC=docker -DC_EXEC=docker-compose -#Check installed versions -echo "This requires a docker engine version 1.10.0+ and docker-compose 1.6.0+. If not, it is expected to fail. See https://docs.docker.com/engine/installation/ and https://docs.docker.com/compose/install/" -$DOCKER_EXEC --version -$DC_EXEC --version +########################################### +# OpenMapTiles quickstart.sh for x86_64 linux +# +# Example calls ... +# ./quickstart.sh +# ./quickstart.sh africa +# ./quickstart.sh alabama +# ./quickstart.sh alaska +# ./quickstart.sh albania +# ./quickstart.sh alberta +# ./quickstart.sh alps +# .... +# +# to list areas : make download-geofabrik-list +# see more QUICKSTART.md +# -#Remove -$DC_EXEC down -$DC_EXEC rm -fv -echo "Remove old volume" -$DOCKER_EXEC volume ls -q | grep openmaptiles | xargs -r docker volume rm || true +if [ $# -eq 0 ]; then + osm_area=albania # default test country + echo "No parameter - set area=$osm_area " +else + osm_area=$1 +fi +testdata=${osm_area}.osm.pbf -echo "Make directories " +## Min versions ... +MIN_COMPOSE_VER=1.7.1 +MIN_DOCKER_VER=1.10.0 +STARTTIME=$(date +%s) +STARTDATE=$(date -Iminutes) +githash=$( git rev-parse HEAD ) + +log_file=./quickstart.log +rm -f $log_file +echo " " +echo "=====================================================================================" +echo " Docker check & Download images " +echo "-------------------------------------------------------------------------------------" +echo "====> : Please check the docker and docker-compose version!" +echo " : We are using docker-compose V2 file format ! see more: https://docs.docker.com/" +echo " : (theoretically;not tested) minumum Docker version is $MIN_DOCKER_VER+." +echo " : (theoretically;not tested) minimum Docker-compose version is $MIN_COMPOSE_VER+." +echo " : See the .travis testfile for the current supported versions " +echo " : Your docker systems is:" +docker --version +docker-compose --version + +COMPOSE_VER=$(docker-compose version --short) +if [ $COMPOSE_VER "<" $MIN_COMPOSE_VER ]; then + echo "ERR: Your Docker-compose version is Known to have bugs , Please Update docker-compose!" + exit 1 +fi + +DOCKER_VER="$(docker -v | awk -F '[ ,]+' '{ print $3 }')" +if [ $DOCKER_VER "<" $MIN_DOCKER_VER ]; then + echo "ERR: Your Docker version is not compatible , Please Update docker!" + exit 1 +fi + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Pulling or Refreshing OpenMapTiles docker images " +make refresh-docker-images + + +##### backup log from here ... +exec &> >(tee -a "$log_file") + +echo " " +echo "=====================================================================================" +echo " Start processing " +echo "-------------------------------------------------------------------------------------" +echo "====> : OpenMapTiles quickstart! [ https://github.com/openmaptiles/openmaptiles ] " +echo " : This will be logged to the $log_file file ( for debugging ) and to the screen" +echo " : Area : $osm_area " +echo " : Git version : $githash " +echo " : Started : $STARTDATE " +echo " : Your bash version: $BASH_VERSION" +echo " : Your OS : $OSTYPE" +docker --version +docker-compose --version + +if [[ "$OSTYPE" == "linux-gnu" ]]; then + + echo " : Your system is:" + lsb_release -a + echo " " + echo "-------------------------------------------------------------------------------------" + echo " : This is working on x86_64 ; Your kernel is:" + uname -r + uname -m + + KERNEL_CPU_VER=$(uname -m) + if [ "$KERNEL_CPU_VER" != "x86_64" ]; then + echo "ERR: Sorry this is working only on x86_64!" + exit 1 + fi + echo " : --- Memory, CPU info ---- " + mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) + echo "system memory (GB): ${mem} " + grep SwapTotal /proc/meminfo + echo cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1) + cat /proc/meminfo | grep Free +else + echo " " + echo "Warning : This is not a Linux ... ( Less tested ... ) " + echo " " +fi + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Stopping running services & removing old containers " +make clean-docker + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Checking OpenMapTiles docker images " +docker images | grep openmaptiles + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Making directories - if they don't exist ( ./build ./data ./pgdata ) " +mkdir -p pgdata mkdir -p build mkdir -p data -testdata=zurich_switzerland.osm.pbf +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Removing old MBTILES if exists ( ./data/*.mbtiles ) " +rm -f ./data/*.mbtiles + if [ ! -f ./data/${testdata} ]; then - echo "Download $testdata " + echo " " + echo "-------------------------------------------------------------------------------------" + echo "====> : Downloading testdata $testdata " rm -f ./data/* - wget https://s3.amazonaws.com/metro-extracts.mapzen.com/zurich_switzerland.osm.pbf -P ./data + #wget $testdataurl -P ./data + make download-geofabrik area=${osm_area} + echo " " + echo "-------------------------------------------------------------------------------------" + echo "====> : Osm metadata : $testdata " + cat ./data/osmstat.txt + echo " " + echo "-------------------------------------------------------------------------------------" + echo "====> : Generated docker-compose config " + cat ./data/docker-compose-config.yml +else + echo " " + echo "-------------------------------------------------------------------------------------" + echo "====> : The testdata ./data/$testdata exists, we don't need to download! " fi -$DOCKER_EXEC run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make -$DC_EXEC up -d postgres -sleep 30 -$DC_EXEC run --rm import-water -$DC_EXEC run --rm import-natural-earth -$DC_EXEC run --rm import-lakelines -$DC_EXEC run --rm import-osm -$DC_EXEC run --rm import-sql +if [ ! -f ./data/${testdata} ]; then + echo " " + echo "Missing ./data/$testdata , Download or Parameter error? " + exit 404 +fi -$DC_EXEC -f docker-compose.yml -f docker-compose-test-override.yml run --rm generate-vectortiles +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Remove old generated source files ( ./build/* ) ( if they exist ) " +docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make clean -$DC_EXEC stop postgres -echo "The vectortiles created from $testdata " +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )" +echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools " +echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! " +docker run --rm -v $(pwd):/tileset openmaptiles/openmaptiles-tools make + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start PostgreSQL service ; create PostgreSQL data volume " +echo " : Source code: https://github.com/openmaptiles/postgis " +echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" +docker-compose up -d postgres + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Drop and Recreate PostgreSQL public schema " +# Drop all PostgreSQL tables +# This is add an extra safe belt , if the user modify the docker volume seetings +make forced-clean-sql + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing water data from http://openstreetmapdata.com into PostgreSQL " +echo " : Source code: https://github.com/openmaptiles/import-water " +echo " : Data license: http://openstreetmapdata.com/info/license " +echo " : Thank you: http://openstreetmapdata.com/info/supporting " +docker-compose run --rm import-water + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL " +echo " : Source code: https://github.com/openmaptiles/import-natural-earth " +echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use " +echo " : Thank you: Natural Earth Contributors! " +docker-compose run --rm import-natural-earth + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing OpenStreetMap Lakelines data " +echo " : Source code: https://github.com/openmaptiles/import-lakelines " +echo " : https://github.com/lukasmartinelli/osm-lakelines " +echo " : Data license: .. " +docker-compose run --rm import-lakelines + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing OpenStreetMap data: ./data/${testdata} -> imposm3[./build/mapping.yaml] -> PostgreSQL" +echo " : Imposm3 documentation: https://imposm.org/docs/imposm3/latest/index.html " +echo " : Thank you Omniscale! " +echo " : Source code: https://github.com/openmaptiles/import-osm " +echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copyright (ODBL) " +echo " : Thank you OpenStreetMap Contributors ! " +docker-compose run --rm import-osm + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start SQL postprocessing: ./build/tileset.sql -> PostgreSQL " +echo " : Source code: https://github.com/openmaptiles/import-sql " +docker-compose run --rm import-sql + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start generating MBTiles (containing gzipped MVT PBF) from a TM2Source project. " +echo " : TM2Source project definitions : ./build/openmaptiles.tm2source/data.yml " +echo " : Output MBTiles: ./data/tiles.mbtiles " +echo " : Source code: https://github.com/openmaptiles/generate-vectortiles " +echo " : We are using a lot of Mapbox Open Source tools! : https://github.com/mapbox " +echo " : Thank you https://www.mapbox.com !" +echo " : See other MVT tools : https://github.com/mapbox/awesome-vector-tiles " +echo " : " +echo " : You will see a lot of deprecated warning in the log! This is normal! " +echo " : like : Mapnik LOG> ... is deprecated and will be removed in Mapnik 4.x ... " + +docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run --rm generate-vectortiles + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" +docker-compose stop postgres + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Inputs - Outputs md5sum for debugging " +rm -f ./data/quickstart_checklist.chk +md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk +md5sum build/tileset.sql >> ./data/quickstart_checklist.chk +md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk +md5sum ./data/${testdata} >> ./data/quickstart_checklist.chk +md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk +md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk +md5sum ./data/osmstat.txt >> ./data/quickstart_checklist.chk +cat ./data/quickstart_checklist.chk + +ENDTIME=$(date +%s) +ENDDATE=$(date -Iminutes) +MODDATE=$(stat -c %y ./data/${testdata} ) + +echo " " +echo " " +echo "-------------------------------------------------------------------------------------" +echo "-- S u m m a r y --" +echo "-------------------------------------------------------------------------------------" +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : (disk space) We have created a lot of docker images: " +echo " : Hint: you can remove with: docker rmi IMAGE " +docker images | grep openmaptiles + + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : (disk space) We have created this new docker volume for PostgreSQL data:" +echo " : Hint: you can remove with : docker volume rm openmaptiles_pgdata " +docker volume ls -q | grep openmaptiles + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : (disk space) We have created the new vectortiles ( ./data/tiles.mbtiles ) " +echo " : The OpenMapTiles MBTILES license: ..... " +echo " : We created from $testdata ( file moddate: $MODDATE ) " +echo " : Size: " ls ./data/*.mbtiles -la -echo "Hello ... start experimenting - see docs ! " + +echo " " +echo "-------------------------------------------------------------------------------------" +echo "The ./quickstart.sh $osm_area is finished! " +echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete" +echo "We saved the log file to $log_file ( for debugging ) You can compare with the travis log !" +echo " " +echo "Start experimenting ! And check the QUICKSTART.MD file !" +echo "Avaialable help commands ( make help ) " +make help + +echo "-------------------------------------------------------------------------------------" +echo " Acknowledgments " +echo " Thanks to all free, open source software developers and Open Data Contributors ! " +echo "-------------------------------------------------------------------------------------"