diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 28fcbfe..98f0c7e 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -153,13 +153,12 @@ jobs: make start-db profile 1_data make import-data profile 2_osm make import-osm - profile 3_borders make import-borders if [ -f ../ci_cache/wikidata-cache.json ]; then cp ../ci_cache/wikidata-cache.json cache/wikidata-cache.json fi - profile 4_wikidata make import-wikidata - profile 5_sql make import-sql + profile 3_wikidata make import-wikidata + profile 4_sql make import-sql # Get database total size, in MB # Once Makefile has a few more improvements, we can use this approach instead: diff --git a/Makefile b/Makefile index 26c169a..510bc51 100644 --- a/Makefile +++ b/Makefile @@ -143,11 +143,6 @@ else DOWNLOAD_AREA := $(area) endif -# import-borders uses these temp files during border parsing/import -export BORDERS_CLEANUP_FILE ?= data/borders/$(area).cleanup.pbf -export BORDERS_PBF_FILE ?= data/borders/$(area).filtered.pbf -export BORDERS_CSV_FILE ?= data/borders/$(area).lines.csv - # The file is placed into the $EXPORT_DIR=/export (mapped to ./data) export MBTILES_FILE ?= $(area).mbtiles MBTILES_LOCAL_FILE = data/$(MBTILES_FILE) @@ -202,7 +197,6 @@ Hints for downloading & importing data: make download-bbbike area=Amsterdam # download OSM data from bbbike.org and create config file make import-data # Import data from OpenStreetMapData, Natural Earth and OSM Lake Labels. make import-osm # Import OSM data with the mapping rules from build/mapping.yaml - make import-borders # Create borders table using extra processing with osmborder tool make import-wikidata # Import labels from Wikidata make import-sql # Import layers (run this after modifying layer SQL) @@ -254,7 +248,7 @@ endef init-dirs: @mkdir -p build/sql/parallel @mkdir -p build/openmaptiles.tm2source - @mkdir -p data/borders + @mkdir -p data @mkdir -p cache @ ! ($(DOCKER_COMPOSE) 2>/dev/null run $(DC_OPTS) openmaptiles-tools df --output=fstype /tileset| grep -q 9p) || ($(win_fs_error)) @@ -406,17 +400,6 @@ import-diff: all start-db-nowait import-data: start-db $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) import-data -.PHONY: import-borders -import-borders: start-db-nowait -ifeq (,$(wildcard $(BORDERS_CSV_FILE))) - @$(assert_area_is_given) - @echo "Generating borders out of $(PBF_FILE)" -else - @echo "Borders already exists. Useing $(BORDERS_CSV_FILE) to import borders" -endif - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \ - 'pgwait && import-borders $$([ -f "$(BORDERS_CSV_FILE)" ] && echo load $(BORDERS_CSV_FILE) || echo import $(PBF_FILE))' - .PHONY: import-sql import-sql: all start-db-nowait $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \ diff --git a/docker-compose.yml b/docker-compose.yml index def4165..ece9961 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,10 +42,6 @@ services: BBOX: ${BBOX} # Imposm configuration file describes how to load updates when enabled IMPOSM_CONFIG_FILE: ${IMPOSM_CONFIG_FILE} - # Which files to use during import-borders processing - BORDERS_CLEANUP_FILE: ${BORDERS_CLEANUP_FILE} - BORDERS_PBF_FILE: ${BORDERS_PBF_FILE} - BORDERS_CSV_FILE: ${BORDERS_CSV_FILE} # Control import-sql processes MAX_PARALLEL_PSQL: ${MAX_PARALLEL_PSQL} networks: diff --git a/layers/boundary/boundary.sql b/layers/boundary/boundary.sql index 0a38092..fa73bd1 100644 --- a/layers/boundary/boundary.sql +++ b/layers/boundary/boundary.sql @@ -1,207 +1,187 @@ --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z13 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring -> osm_border_linestring_gen_z13 -- etldoc: osm_border_linestring_adm -> osm_border_linestring_gen_z13 +-- etldoc: osm_border_disp_linestring -> osm_border_linestring_gen_z13 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z13 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z13 AS ( -SELECT ST_Simplify(geometry, ZRes(14)) AS geometry, NULL::text AS adm0_l, NULL::text AS adm0_r, admin_level, disputed, maritime -FROM osm_border_linestring -WHERE admin_level BETWEEN 3 AND 10 -UNION ALL -SELECT ST_Simplify(geometry, ZRes(14)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime -FROM osm_border_linestring_adm - ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; -CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z13_idx ON osm_border_linestring_gen_z13 USING gist (geometry); +SELECT ST_Simplify(ST_Collect(geometry), ZRes(14)) AS geometry, + MAX(adm0_l) AS adm0_l, + MAX(adm0_r) AS adm0_r, + MIN(admin_level) AS admin_level, + BOOL_OR(disputed) AS disputed, + MAX(name) AS name, + MAX(claimed_by) AS claimed_by, + BOOL_OR(maritime) AS maritime +FROM ( + -- All admin 3-10 boundaries + SELECT osm_id, + geometry, + NULL::text AS adm0_l, + NULL::text AS adm0_r, + MIN(admin_level) AS admin_level, + BOOL_OR(disputed) + OR BOOL_OR(dispute) + OR BOOL_OR(border_status = 'disputed') + OR BOOL_OR(disputed_by <> '') AS disputed, + NULLIF(name, '') AS name, + NULLIF(claimed_by, '') AS claimed_by, + BOOL_OR(maritime) AS maritime + FROM osm_border_linestring + WHERE admin_level BETWEEN 3 AND 10 + AND type = 1 -- ways only + GROUP BY osm_id, geometry, name, claimed_by --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z12 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; + UNION ALL + + -- All non-disputed admin 2 boundaries + SELECT osm_id, + geometry, + adm0_l, + adm0_r, + admin_level, + FALSE AS disputed, + NULL::text AS name, + NULL::text AS claimed_by, + maritime + FROM osm_border_linestring_adm + + UNION ALL + + -- All disputed admin 2 boundaries + SELECT osm_id, + geometry, + NULL::text AS adm0_l, + NULL::text AS adm0_r, + 2::int AS admin_level, + TRUE AS disputed, + NULLIF(name, '') AS name, + NULLIF(claimed_by, '') AS claimed_by, + maritime + FROM osm_border_disp_linestring + GROUP BY osm_id, geometry, name, claimed_by, maritime + ) AS merged_boundary +GROUP by osm_id +)/* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z13_idx ON osm_border_linestring_gen_z13 USING gist (geometry); -- etldoc: osm_border_linestring_gen_z13 -> osm_border_linestring_gen_z12 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z12 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z12 AS ( -SELECT ST_Simplify(geometry, ZRes(13)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(13)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z13 WHERE admin_level BETWEEN 2 AND 10 ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z12_idx ON osm_border_linestring_gen_z12 USING gist (geometry); --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z11 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring_gen_z12 -> osm_border_linestring_gen_z11 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z11 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z11 AS ( -SELECT ST_Simplify(geometry, ZRes(12)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(12)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z12 WHERE admin_level BETWEEN 2 AND 8 ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z11_idx ON osm_border_linestring_gen_z11 USING gist (geometry); --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z10 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring_gen_z11 -> osm_border_linestring_gen_z10 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z10 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z10 AS ( -SELECT ST_Simplify(geometry, ZRes(11)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(11)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z11 WHERE admin_level BETWEEN 2 AND 6 ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z10_idx ON osm_border_linestring_gen_z10 USING gist (geometry); --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z9 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring_gen_z10 -> osm_border_linestring_gen_z9 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z9 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z9 AS ( -SELECT ST_Simplify(geometry, ZRes(10)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(10)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z10 -WHERE admin_level BETWEEN 2 AND 6 +-- WHERE admin_level BETWEEN 2 AND 6 ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z9_idx ON osm_border_linestring_gen_z9 USING gist (geometry); --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z8 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring_gen_z9 -> osm_border_linestring_gen_z8 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z8 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z8 AS ( -SELECT ST_Simplify(geometry, ZRes(9)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(9)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z9 WHERE admin_level BETWEEN 2 AND 4 ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z8_idx ON osm_border_linestring_gen_z8 USING gist (geometry); --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z7 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring_gen_z8 -> osm_border_linestring_gen_z7 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z7 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z7 AS ( -SELECT ST_Simplify(geometry, ZRes(8)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(8)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z8 -WHERE admin_level BETWEEN 2 AND 4 +-- WHERE admin_level BETWEEN 2 AND 4 ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z7_idx ON osm_border_linestring_gen_z7 USING gist (geometry); --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z6 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring_gen_z7 -> osm_border_linestring_gen_z6 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z6 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z6 AS ( -SELECT ST_Simplify(geometry, ZRes(7)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(7)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z7 -WHERE admin_level BETWEEN 2 AND 4 +-- WHERE admin_level BETWEEN 2 AND 4 ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z6_idx ON osm_border_linestring_gen_z6 USING gist (geometry); --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z5 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring_gen_z6 -> osm_border_linestring_gen_z5 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z5 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z5 AS ( -SELECT ST_Simplify(geometry, ZRes(6)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(6)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z6 -WHERE admin_level BETWEEN 2 AND 4 +-- WHERE admin_level BETWEEN 2 AND 4 ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z5_idx ON osm_border_linestring_gen_z5 USING gist (geometry); --- This statement can be deleted after the border importer image stops creating this object as a table -DO -$$ - BEGIN - DROP TABLE IF EXISTS osm_border_linestring_gen_z4 CASCADE; - EXCEPTION - WHEN wrong_object_type THEN - END; -$$ LANGUAGE plpgsql; - -- etldoc: osm_border_linestring_gen_z5 -> osm_border_linestring_gen_z4 DROP MATERIALIZED VIEW IF EXISTS osm_border_linestring_gen_z4 CASCADE; CREATE MATERIALIZED VIEW osm_border_linestring_gen_z4 AS ( -SELECT ST_Simplify(geometry, ZRes(5)) AS geometry, adm0_l, adm0_r, admin_level, disputed, maritime +SELECT ST_Simplify(geometry, ZRes(5)) AS geometry, adm0_l, adm0_r, admin_level, disputed, name, claimed_by, maritime FROM osm_border_linestring_gen_z5 -WHERE admin_level = 2 +WHERE admin_level = 2 AND (maritime OR disputed) ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; CREATE INDEX IF NOT EXISTS osm_border_linestring_gen_z4_idx ON osm_border_linestring_gen_z4 USING gist (geometry); +-- etldoc: osm_border_linestring_gen_z4 -> osm_border_disp_linestring_gen_z3 +DROP MATERIALIZED VIEW IF EXISTS osm_border_disp_linestring_gen_z3 CASCADE; +CREATE MATERIALIZED VIEW osm_border_disp_linestring_gen_z3 AS +( +SELECT ST_Simplify(geometry, ZRes(4)) AS geometry, adm0_l, adm0_r, admin_level, TRUE AS disputed, name, claimed_by, maritime +FROM osm_border_linestring_gen_z4 +WHERE disputed -- AND admin_level = 2 + ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_disp_linestring_gen_z3_idx ON osm_border_disp_linestring_gen_z3 USING gist (geometry); + +-- etldoc: osm_border_disp_linestring_gen_z3 -> osm_border_disp_linestring_gen_z2 +DROP MATERIALIZED VIEW IF EXISTS osm_border_disp_linestring_gen_z2 CASCADE; +CREATE MATERIALIZED VIEW osm_border_disp_linestring_gen_z2 AS +( +SELECT ST_Simplify(geometry, ZRes(3)) AS geometry, adm0_l, adm0_r, admin_level, TRUE AS disputed, name, claimed_by, maritime +FROM osm_border_disp_linestring_gen_z3 + ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_disp_linestring_gen_z2_idx ON osm_border_disp_linestring_gen_z2 USING gist (geometry); + +-- etldoc: osm_border_disp_linestring_gen_z2 -> osm_border_disp_linestring_gen_z1 +DROP MATERIALIZED VIEW IF EXISTS osm_border_disp_linestring_gen_z1 CASCADE; +CREATE MATERIALIZED VIEW osm_border_disp_linestring_gen_z1 AS +( +SELECT ST_Simplify(geometry, ZRes(2)) AS geometry, adm0_l, adm0_r, admin_level, TRUE AS disputed, name, claimed_by, maritime +FROM osm_border_disp_linestring_gen_z2 + ) /* DELAY_MATERIALIZED_VIEW_CREATION */ ; +CREATE INDEX IF NOT EXISTS osm_border_disp_linestring_gen_z2_idx ON osm_border_disp_linestring_gen_z2 USING gist (geometry); + -- ne_10m_admin_0_boundary_lines_land -- etldoc: ne_10m_admin_0_boundary_lines_land -> ne_10m_admin_0_boundary_lines_land_gen_z4 DROP MATERIALIZED VIEW IF EXISTS ne_10m_admin_0_boundary_lines_land_gen_z4 CASCADE; @@ -477,7 +457,6 @@ FROM osm_border_disp_linestring_gen_z3 -- etldoc: ne_10m_admin_0_boundary_lines_land_gen_z4 -> boundary_z4 -- etldoc: ne_10m_admin_1_states_provinces_lines_gen_z4 -> boundary_z4 -- etldoc: osm_border_linestring_gen_z4 -> boundary_z4 --- etldoc: osm_border_disp_linestring_gen_z4 -> boundary_z4 CREATE OR REPLACE VIEW boundary_z4 AS ( SELECT geometry, @@ -505,26 +484,13 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, - maritime -FROM osm_border_linestring_gen_z4 -WHERE maritime = TRUE - AND admin_level <= 2 -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, claimed_by, maritime -FROM osm_border_disp_linestring_gen_z4 +FROM osm_border_linestring_gen_z4 ); -- etldoc: osm_border_linestring_gen_z5 -> boundary_z5 --- etldoc: osm_border_disp_linestring_gen_z5 -> boundary_z5 CREATE OR REPLACE VIEW boundary_z5 AS ( SELECT geometry, @@ -532,27 +498,14 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, + claimed_by, maritime FROM osm_border_linestring_gen_z5 WHERE admin_level <= 4 --- already not included in osm_border_linestring_adm --- AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z5) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, - claimed_by, - maritime -FROM osm_border_disp_linestring_gen_z5 ); -- etldoc: osm_border_linestring_gen_z6 -> boundary_z6 --- etldoc: osm_border_disp_linestring_gen_z6 -> boundary_z6 CREATE OR REPLACE VIEW boundary_z6 AS ( SELECT geometry, @@ -560,26 +513,14 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, + claimed_by, maritime FROM osm_border_linestring_gen_z6 WHERE admin_level <= 4 --- AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z6) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, - claimed_by, - maritime -FROM osm_border_disp_linestring_gen_z6 ); -- etldoc: osm_border_linestring_gen_z7 -> boundary_z7 --- etldoc: osm_border_disp_linestring_gen_z7 -> boundary_z7 CREATE OR REPLACE VIEW boundary_z7 AS ( SELECT geometry, @@ -587,26 +528,14 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, + claimed_by, maritime FROM osm_border_linestring_gen_z7 WHERE admin_level <= 6 --- AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z7) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, - claimed_by, - maritime -FROM osm_border_disp_linestring_gen_z7 ); -- etldoc: osm_border_linestring_gen_z8 -> boundary_z8 --- etldoc: osm_border_disp_linestring_gen_z8 -> boundary_z8 CREATE OR REPLACE VIEW boundary_z8 AS ( SELECT geometry, @@ -614,26 +543,14 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, + claimed_by, maritime FROM osm_border_linestring_gen_z8 WHERE admin_level <= 6 --- AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z8) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, - claimed_by, - maritime -FROM osm_border_disp_linestring_gen_z8 ); -- etldoc: osm_border_linestring_gen_z9 -> boundary_z9 --- etldoc: osm_border_disp_linestring_gen_z9 -> boundary_z9 CREATE OR REPLACE VIEW boundary_z9 AS ( SELECT geometry, @@ -641,26 +558,14 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, + claimed_by, maritime FROM osm_border_linestring_gen_z9 WHERE admin_level <= 6 --- AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z9) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, - claimed_by, - maritime -FROM osm_border_disp_linestring_gen_z9 ); -- etldoc: osm_border_linestring_gen_z10 -> boundary_z10 --- etldoc: osm_border_disp_linestring_gen_z10 -> boundary_z10 CREATE OR REPLACE VIEW boundary_z10 AS ( SELECT geometry, @@ -668,26 +573,14 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, + claimed_by, maritime FROM osm_border_linestring_gen_z10 WHERE admin_level <= 6 --- AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z10) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, - claimed_by, - maritime -FROM osm_border_disp_linestring_gen_z10 ); -- etldoc: osm_border_linestring_gen_z11 -> boundary_z11 --- etldoc: osm_border_disp_linestring_gen_z11 -> boundary_z11 CREATE OR REPLACE VIEW boundary_z11 AS ( SELECT geometry, @@ -695,26 +588,14 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, + claimed_by, maritime FROM osm_border_linestring_gen_z11 WHERE admin_level <= 8 --- AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z11) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, - claimed_by, - maritime -FROM osm_border_disp_linestring_gen_z11 ); -- etldoc: osm_border_linestring_gen_z12 -> boundary_z12 --- etldoc: osm_border_disp_linestring_gen_z12 -> boundary_z12 CREATE OR REPLACE VIEW boundary_z12 AS ( SELECT geometry, @@ -722,25 +603,13 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, - maritime -FROM osm_border_linestring_gen_z12 ---WHERE osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z12) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, claimed_by, maritime -FROM osm_border_disp_linestring_gen_z12 +FROM osm_border_linestring_gen_z12 ); -- etldoc: osm_border_linestring_gen_z13 -> boundary_z13 --- etldoc: osm_border_disp_linestring_gen_z13 -> boundary_z13 CREATE OR REPLACE VIEW boundary_z13 AS ( SELECT geometry, @@ -748,21 +617,10 @@ SELECT geometry, adm0_l, adm0_r, disputed, - NULL::text AS disputed_name, - NULL::text AS claimed_by, - maritime -FROM osm_border_linestring_gen_z13 ---WHERE osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring_gen_z13) -UNION ALL -SELECT geometry, - admin_level, - NULL::text AS adm0_l, - NULL::text AS adm0_r, - TRUE AS disputed, - edit_name(name) AS disputed_name, + CASE WHEN disputed THEN edit_name(name) END AS disputed_name, claimed_by, maritime -FROM osm_border_disp_linestring_gen_z13 +FROM osm_border_linestring_gen_z13 ); -- etldoc: layer_boundary[shape=record fillcolor=lightpink, style="rounded,filled", diff --git a/layers/boundary/boundary_name.sql b/layers/boundary/boundary_name.sql index 56e1296..4be7dac 100644 --- a/layers/boundary/boundary_name.sql +++ b/layers/boundary/boundary_name.sql @@ -1,23 +1,25 @@ DROP TABLE IF EXISTS osm_border_linestring_adm CASCADE; -- etldoc: osm_border_linestring -> osm_border_linestring_adm +-- etldoc: osm_border_disp_linestring -> osm_border_linestring_adm +-- etldoc: ne_10m_admin_0_countries -> osm_border_linestring_adm CREATE TABLE IF NOT EXISTS osm_border_linestring_adm AS ( WITH -- Prepare lines from osm to be merged multiline AS ( - SELECT ST_Node(ST_Collect(geometry)) AS geometry, - maritime, - disputed + SELECT osm_id, + ST_Node(ST_Collect(geometry)) AS geometry, + BOOL_OR(maritime) AS maritime, + FALSE AS disputed FROM osm_border_linestring - WHERE admin_level = 2 + WHERE admin_level = 2 AND ST_Dimension(geometry) = 1 AND osm_id NOT IN (SELECT DISTINCT osm_id FROM osm_border_disp_linestring) - GROUP BY maritime, - disputed + GROUP BY osm_id ), mergedline AS ( - SELECT (ST_Dump( - ST_LineMerge(geometry))).geom AS geometry, + SELECT osm_id, + (ST_Dump(ST_LineMerge(geometry))).geom AS geometry, maritime, disputed FROM multiline @@ -32,7 +34,7 @@ CREATE TABLE IF NOT EXISTS osm_border_linestring_adm AS ( FROM (SELECT ST_Node( ST_Collect(geometry)) AS geometry FROM osm_border_linestring - WHERE admin_level = 2 + WHERE admin_level = 2 AND ST_Dimension(geometry) = 1 ) nodes ) linemerge ), @@ -55,12 +57,14 @@ CREATE TABLE IF NOT EXISTS osm_border_linestring_adm AS ( ), rights AS ( - SELECT adm0_r, + SELECT osm_id, + adm0_r, geometry, maritime, disputed FROM ( - SELECT b.adm0_a3 AS adm0_r, + SELECT a.osm_id AS osm_id, + b.adm0_a3 AS adm0_r, a.geometry, a.maritime, a.disputed @@ -73,14 +77,16 @@ CREATE TABLE IF NOT EXISTS osm_border_linestring_adm AS ( ) line_rights ) - SELECT adm0_l, + SELECT osm_id, + adm0_l, adm0_r, geometry, maritime, 2::integer AS admin_level, disputed FROM ( - SELECT b.adm0_a3 AS adm0_l, + SELECT r.osm_id AS osm_id, + b.adm0_a3 AS adm0_l, r.adm0_r AS adm0_r, r.geometry, r.maritime, @@ -96,4 +102,4 @@ CREATE TABLE IF NOT EXISTS osm_border_linestring_adm AS ( CREATE INDEX IF NOT EXISTS osm_border_linestring_adm_geom_idx ON osm_border_linestring_adm - USING GIST (geometry); \ No newline at end of file + USING GIST (geometry); diff --git a/layers/boundary/etl_diagram.png b/layers/boundary/etl_diagram.png index a4841c2..c571efd 100644 Binary files a/layers/boundary/etl_diagram.png and b/layers/boundary/etl_diagram.png differ diff --git a/layers/boundary/mapping.yaml b/layers/boundary/mapping.yaml index 76c0f93..dda9fa4 100644 --- a/layers/boundary/mapping.yaml +++ b/layers/boundary/mapping.yaml @@ -1,107 +1,34 @@ generalized_tables: - - # etldoc: osm_border_disp_linestring_gen_z2 -> osm_border_disp_linestring_gen_z1 - border_disp_linestring_gen_z1: - source: border_disp_linestring_gen_z2 - sql_filter: admin_level = 2 - tolerance: ZRES2 - - # etldoc: osm_border_disp_linestring_gen_z3 -> osm_border_disp_linestring_gen_z2 - border_disp_linestring_gen_z2: - source: border_disp_linestring_gen_z3 - sql_filter: admin_level = 2 - tolerance: ZRES3 - - # etldoc: osm_border_disp_linestring_gen_z4 -> osm_border_disp_linestring_gen_z3 - border_disp_linestring_gen_z3: - source: border_disp_linestring_gen_z4 - sql_filter: admin_level = 2 - tolerance: ZRES4 - - # etldoc: osm_border_disp_linestring_gen_z5 -> osm_border_disp_linestring_gen_z4 - border_disp_linestring_gen_z4: - source: border_disp_linestring_gen_z5 - sql_filter: admin_level = 2 - tolerance: ZRES5 - - # etldoc: osm_border_disp_linestring_gen_z6 -> osm_border_disp_linestring_gen_z5 - border_disp_linestring_gen_z5: - source: border_disp_linestring_gen_z6 - sql_filter: admin_level = 2 - tolerance: ZRES6 - - # etldoc: osm_border_disp_linestring_gen_z7 -> osm_border_disp_linestring_gen_z6 - border_disp_linestring_gen_z6: - source: border_disp_linestring_gen_z7 - sql_filter: admin_level = 2 - tolerance: ZRES7 - - # etldoc: osm_border_disp_linestring_gen_z8 -> osm_border_disp_linestring_gen_z7 - border_disp_linestring_gen_z7: - source: border_disp_linestring_gen_z8 - sql_filter: admin_level = 2 - tolerance: ZRES8 - - # etldoc: osm_border_disp_linestring_gen_z9 -> osm_border_disp_linestring_gen_z8 - border_disp_linestring_gen_z8: - source: border_disp_linestring_gen_z9 - sql_filter: admin_level = 2 - tolerance: ZRES9 - - # etldoc: osm_border_disp_linestring_gen_z10 -> osm_border_disp_linestring_gen_z9 - border_disp_linestring_gen_z9: - source: border_disp_linestring_gen_z10 - sql_filter: admin_level = 2 - tolerance: ZRES10 - - # etldoc: osm_border_disp_linestring_gen_z11 -> osm_border_disp_linestring_gen_z10 - border_disp_linestring_gen_z10: - source: border_disp_linestring_gen_z11 - sql_filter: admin_level = 2 - tolerance: ZRES11 - - # etldoc: osm_border_disp_linestring_gen_z12 -> osm_border_disp_linestring_gen_z11 - border_disp_linestring_gen_z11: - source: border_disp_linestring_gen_z12 - sql_filter: admin_level = 2 - tolerance: ZRES12 - - # etldoc: osm_border_disp_linestring_gen_z13 -> osm_border_disp_linestring_gen_z12 - border_disp_linestring_gen_z12: - source: border_disp_linestring_gen_z13 - sql_filter: admin_level = 2 - tolerance: ZRES13 - - # etldoc: osm_border_disp_linestring -> osm_border_disp_linestring_gen_z13 - border_disp_linestring_gen_z13: - source: border_disp_linestring - sql_filter: admin_level = 2 - tolerance: ZRES14 - - # etldoc: osm_border_disp_relation -> osm_border_disp_linestring + # etldoc: osm_border_linestring -> osm_border_disp_linestring border_disp_linestring: - source: border_disp_relation - sql_filter: ST_GeometryType(geometry) = 'ST_LineString' - + source: border_linestring + sql_filter: ST_GeometryType(geometry) = 'ST_LineString' AND (disputed OR dispute OR border_status = 'disputed' OR disputed_by <> '') AND admin_level = 2 tables: - # etldoc: imposm3 -> osm_border_disp_relation - border_disp_relation: + # etldoc: imposm3 -> osm_border_linestring + border_linestring: type: relation_member + filters: + require: + admin_level: [__any__] + boundary: [administrative] columns: - name: relation_id type: id - name: osm_id type: id from_member: true + - name: member + type: member_id + - name: type + type: member_type - name: geometry type: geometry - key: name name: name type: string - - key: boundary - name: boundary - type: string + # Used for disputed boundary, e.g. "Line of actual control" + from_member: true - key: admin_level name: admin_level type: integer @@ -111,14 +38,36 @@ tables: - key: disputed_by name: disputed_by type: string + from_member: true + - key: dispute + name: dispute + type: bool + from_member: true + - key: disputed + name: disputed + type: bool + from_member: true + - key: border_status + name: border_status + type: string + from_member: true - key: maritime name: maritime type: bool from_member: true + - key: boundary_type + name: boundary_type + type: string + from_member: true + - key: natural + name: natural + type: string + from_member: true + relation_types: [boundary] mapping: - type: [boundary] - filters: - require: - #admin_level: ['2'] - admin_level: [__any__] - claimed_by: [__any__] + boundary: + - administrative + border_status: + - dispute + boundary_type: + - maritime diff --git a/layers/boundary/mapping_diagram.png b/layers/boundary/mapping_diagram.png index 1eb7fb2..f75cc88 100644 Binary files a/layers/boundary/mapping_diagram.png and b/layers/boundary/mapping_diagram.png differ diff --git a/quickstart.sh b/quickstart.sh index e9ea546..1ea7385 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -234,14 +234,6 @@ echo " : The OpenstreetMap data license: https://www.openstreetmap.org/copy echo " : Thank you OpenStreetMap Contributors ! " make import-osm -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing border ${area} data into PostgreSQL using osmborder" -echo " : Source code: https://github.com/pnorman/osmborder" -echo " : Data license: http://www.openstreetmap.org/copyright" -echo " : Thank you Paul Norman" -make import-borders - echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Start importing Wikidata: Wikidata Query Service -> PostgreSQL"