From c38c7af17d23b686237baf91372ceeb862cd5ecc Mon Sep 17 00:00:00 2001 From: Miel Truyen Date: Tue, 10 Jun 2025 20:34:00 +0200 Subject: [PATCH] feat: pre-2025 state --- .env | 17 ++++++------- cerxes-maps-belgium.sh | 46 ++++++++++++++++++++++++++++++++++++ cerxes-maps.sh | 46 ++++++++++++++++++++++++++++++++++++ docker-compose.yml | 2 +- layers/boundary/mapping.yaml | 45 +++++++++++++++++++++++++++++++++++ layers/landuse/landuse.sql | 42 +++++++++++++++++++++----------- test.sh | 5 ++-- 7 files changed, 177 insertions(+), 26 deletions(-) create mode 100755 cerxes-maps-belgium.sh create mode 100755 cerxes-maps.sh diff --git a/.env b/.env index 55c3f32..4d1def1 100644 --- a/.env +++ b/.env @@ -16,13 +16,13 @@ PGPORT=5432 # BBOX may get overwritten by the computed bbox of the specific area: # make generate-bbox-file # By default, the Makefile will use the content of data/$(area).bbox file if it exists. -#BBOX=-180.0,-85.0511,180.0,85.0511 -BBOX=4.827919,50.740472,5.798035,51.095291 +#BBOX=2.255544,49.857797,6.586904,51.389246 +#BBOX=2.255544,49.857797,6.586904,51.389246 # Which zooms to generate with make generate-tiles-pg MIN_ZOOM=0 -MAX_ZOOM=14 +MAX_ZOOM=16 # `MID_ZOOM` setting only works with `make generate-tiles-pg` command. Make sure MID_ZOOM < MAX_ZOOM. # See https://github.com/openmaptiles/openmaptiles-tools/pull/383 @@ -47,8 +47,8 @@ BORDERS_PBF_FILE=data/borders/filtered.pbf BORDERS_CSV_FILE=data/borders/lines.csv # Number of parallel processes to use when importing sql files -#MAX_PARALLEL_PSQL=5 -MAX_PARALLEL_PSQL=16 +MAX_PARALLEL_PSQL=5 +#MAX_PARALLEL_PSQL=16 # Number of concurrent IO ops to use when generating vector map tiles # Set to ridiculous high values for SSD/NVME @@ -61,13 +61,14 @@ MAX_PARALLEL_PSQL=16 # Postgres is mainly idle during all of this (coarse dataset), using only half a core # To be tested if 1 core is dedicated to postgres and the others to tilelive(UV_THREADPOOL_SIZE) # will prove to be more efficient on average. Or just UV_THREADPOOL_SIZE==CPU_THREADS. -#COPY_CONCURRENCY=64 +#COPY_CONCURRENCY=10 +COPY_CONCURRENCY=64 #COPY_CONCURRENCY=256 #COPY_CONCURRENCY=512 -COPY_CONCURRENCY=1024 +#COPY_CONCURRENCY=1024 #COPY_CONCURRENCY=4096 -UV_THREADPOOL_SIZE=16 +#UV_THREADPOOL_SIZE=16 #UV_THREADPOOL_SIZE=24 # Variables for generate tiles using tilelive-pgquery diff --git a/cerxes-maps-belgium.sh b/cerxes-maps-belgium.sh new file mode 100755 index 0000000..11492f9 --- /dev/null +++ b/cerxes-maps-belgium.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -e + +# Test script for generating tiles for Lummen and region (e.g. including Hasselt) +export SOURCE=osmfr +export AREA=europe/belgium +export BBOX=2.255544,49.857797,6.586904,51.389246 +# Which zooms to generate in make generate-tiles +export MIN_ZOOM=0 +export MAX_ZOOM=16 + +# Update the .env to match +sed -i "s/MIN_ZOOM=.*/MIN_ZOOM=${MIN_ZOOM}/" .env +sed -i "s/MAX_ZOOM=.*/MAX_ZOOM=${MAX_ZOOM}/" .env +sed -i "s/BBOX=.*/BBOX=${BBOX}/" .env + + +# Setup +make clean +make DC_OPTS=--rm + +# Start from a clean db +make start-db +make destroy-db +make import-data +# ALTERNATIVE would be this +#make start-db-preloaded + +# Download +make download-${SOURCE} area="${AREA}" + +# Import (yes we're doing import borders twice, it crashes on first run, oddly enough) +make import-osm +make import-wikidata +make import-borders || true +make import-sql +make analyze-db + +# (This potentially screws stuff up?!) +#rm -rf data/${AREA}.dc-config.yml +#make generate-dc-config + +# Generate +make generate-tiles-pg + +make stop-db \ No newline at end of file diff --git a/cerxes-maps.sh b/cerxes-maps.sh new file mode 100755 index 0000000..12700b8 --- /dev/null +++ b/cerxes-maps.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -e + +# Script for generating tiles for what cerxes server should host +export SOURCE=geofabrik +export AREA=europe +export BBOX=-4.382176,47.705913,13.053127,53.803762 +# Which zooms to generate in make generate-tiles +export MIN_ZOOM=0 +export MAX_ZOOM=16 + +# Update the .env to match +sed -i "s/MIN_ZOOM=.*/MIN_ZOOM=${MIN_ZOOM}/" .env +sed -i "s/MAX_ZOOM=.*/MAX_ZOOM=${MAX_ZOOM}/" .env +sed -i "s/BBOX=.*/BBOX=${BBOX}/" .env + + +# Setup +make clean +make DC_OPTS=--rm + +# Start from a clean db +make start-db +make destroy-db +make import-data +# ALTERNATIVE would be this +#make start-db-preloaded + +# Download +make download-${SOURCE} area="${AREA}" + +# Import (yes we're doing import borders twice, it crashes on first run, oddly enough) +make import-osm +make import-wikidata +make import-borders || true +make import-sql +make analyze-db + +# (This potentially screws stuff up?!) +#rm -rf data/${AREA}.dc-config.yml +#make generate-dc-config + +# Generate +make generate-tiles-pg + +make stop-db \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 2c78c8e..c8b3ab5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: POSTGRES_USER: ${PGUSER:-openmaptiles} POSTGRES_PASSWORD: ${PGPASSWORD:-openmaptiles} PGPORT: ${PGPORT:-5432} - shm_size: 4g + shm_size: 16384MB #command: /docker-entrypoint-initdb.d/01_tune-postgis.sh #command: /docker-entrypoint-initdb.d/01_tune-postgis.sh && echo 'tuned' && postgres -c 'config_file=/etc/postgresql/postgresql.conf' diff --git a/layers/boundary/mapping.yaml b/layers/boundary/mapping.yaml index abf53e8..ad13b67 100644 --- a/layers/boundary/mapping.yaml +++ b/layers/boundary/mapping.yaml @@ -78,6 +78,51 @@ tables: boundary_type: - maritime + + # etldoc: imposm3 -> osm_border_disp_relation + border_disp_relation: + type: relation_member + columns: + - name: relation_id + type: id + - name: osm_id + type: id + from_member: true + - name: geometry + type: geometry + - key: name + name: name + type: string + - key: boundary + name: boundary + type: string + - key: admin_level + name: admin_level + type: integer + - key: claimed_by + name: claimed_by + type: string + - key: disputed_by + name: disputed_by + type: string + - key: maritime + name: maritime + type: bool + from_member: true + - name: index + type: member_index + - name: role + type: member_role + - name: type + type: member_type + mapping: + type: [boundary] + filters: + require: + #admin_level: ['2'] # this used to be specified, re-enable if bugs show up with country borders + admin_level: [__any__] + boundary: ['administrative'] # Filters out boundary administrative_fraction and religious_administration + # FOr NUTS in linestring version administrative_relation: type: relation diff --git a/layers/landuse/landuse.sql b/layers/landuse/landuse.sql index 35fa2f2..889e482 100644 --- a/layers/landuse/landuse.sql +++ b/layers/landuse/landuse.sql @@ -48,7 +48,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z6_union AS leisure, tourism, place, - waterway + waterway, + man_made FROM osm_landuse_polygon_gen_z6 WHERE landuse <> 'residential' UNION ALL @@ -59,7 +60,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z6_union AS '' AS leisure, '' AS tourism, '' AS place, - '' AS waterway + '' AS waterway, + '' AS man_made FROM osm_residential_gen_z6 ); @@ -74,7 +76,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z7_union AS leisure, tourism, place, - waterway + waterway, + man_made FROM osm_landuse_polygon_gen_z7 WHERE landuse <> 'residential' UNION ALL @@ -85,7 +88,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z7_union AS '' AS leisure, '' AS tourism, '' AS place, - '' AS waterway + '' AS waterway, + '' AS man_made FROM osm_residential_gen_z7 ); @@ -100,7 +104,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z8_union AS leisure, tourism, place, - waterway + waterway, + man_made FROM osm_landuse_polygon_gen_z8 WHERE landuse <> 'residential' UNION ALL @@ -111,7 +116,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z8_union AS '' AS leisure, '' AS tourism, '' AS place, - '' AS waterway + '' AS waterway, + '' AS man_made FROM osm_residential_gen_z8 ); @@ -126,7 +132,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z9_union AS leisure, tourism, place, - waterway + waterway, + man_made FROM osm_landuse_polygon_gen_z9 WHERE landuse <> 'residential' UNION ALL @@ -137,7 +144,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z9_union AS '' AS leisure, '' AS tourism, '' AS place, - '' AS waterway + '' AS waterway, + '' AS man_made FROM osm_residential_gen_z9 ); @@ -152,7 +160,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z10_union AS leisure, tourism, place, - waterway + waterway, + man_made FROM osm_landuse_polygon_gen_z10 WHERE landuse <> 'residential' UNION ALL @@ -163,7 +172,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z10_union AS '' AS leisure, '' AS tourism, '' AS place, - '' AS waterway + '' AS waterway, + '' AS man_made FROM osm_residential_gen_z10 ); @@ -178,7 +188,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z11_union AS leisure, tourism, place, - waterway + waterway, + man_made FROM osm_landuse_polygon_gen_z11 WHERE landuse <> 'residential' UNION ALL @@ -189,7 +200,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z11_union AS '' AS leisure, '' AS tourism, '' AS place, - '' AS waterway + '' AS waterway, + '' AS man_made FROM osm_residential_gen_z11 ); @@ -204,7 +216,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z12_union AS leisure, tourism, place, - waterway + waterway, + man_made FROM osm_landuse_polygon_gen_z12 WHERE landuse <> 'residential' UNION ALL @@ -215,7 +228,8 @@ CREATE OR REPLACE VIEW osm_landuse_polygon_gen_z12_union AS '' AS leisure, '' AS tourism, '' AS place, - '' AS waterway + '' AS waterway, + '' AS man_made FROM osm_residential_gen_z12 ); diff --git a/test.sh b/test.sh index 0452440..890b882 100755 --- a/test.sh +++ b/test.sh @@ -31,9 +31,8 @@ make download-${SOURCE} area="${AREA}" # Import (yes we're doing import borders twice, it crashes on first run, oddly enough) make import-osm -make import-borders || true -make import-borders make import-wikidata +make import-borders || true make import-sql make analyze-db @@ -42,6 +41,6 @@ make analyze-db #make generate-dc-config # Generate -make generate-tiles +make generate-tiles-pg make stop-db \ No newline at end of file