From c64170d9bed9576b911130de9ee9d94b0ae69454 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Mon, 2 Jan 2017 18:31:17 -0500 Subject: [PATCH 01/54] Initial materialized views and triggers --- layers/transportation_name/merge_highways.sql | 39 ++++++++++++++----- layers/water_name/water_lakeline.sql | 21 ++++++++-- layers/water_name/water_point.sql | 21 ++++++++-- layers/waterway/merge_waterway.sql | 37 +++++++++++++----- 4 files changed, 90 insertions(+), 28 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index c834c17..18e92a9 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -1,16 +1,11 @@ -DROP TABLE IF EXISTS osm_transportation_name_linestring CASCADE; -DROP TABLE IF EXISTS osm_transportation_name_linestring_gen1 CASCADE; -DROP TABLE IF EXISTS osm_transportation_name_linestring_gen2 CASCADE; -DROP TABLE IF EXISTS osm_transportation_name_linestring_gen3 CASCADE; - -- Instead of using relations to find out the road names we -- stitch together the touching ways with the same name -- to allow for nice label rendering -- Because this works well for roads that do not have relations as well -- etldoc: osm_highway_linestring -> osm_transportation_name_linestring -CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring AS ( +CREATE MATERIALIZED VIEW osm_transportation_name_linestring AS ( SELECT (ST_Dump(geometry)).geom AS geometry, -- NOTE: The osm_id is no longer the original one which can make it difficult @@ -35,11 +30,10 @@ CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring AS ( GROUP BY name, highway, ref ) AS highway_union ); - CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_geometry_idx ON osm_transportation_name_linestring USING gist(geometry); -- etldoc: osm_transportation_name_linestring -> osm_transportation_name_linestring_gen1 -CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen1 AS ( +CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 50) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring WHERE highway IN ('motorway','trunk') AND ST_Length(geometry) > 8000 @@ -47,7 +41,7 @@ CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen1 AS ( CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_geometry_idx ON osm_transportation_name_linestring_gen1 USING gist(geometry); -- etldoc: osm_transportation_name_linestring_gen1 -> osm_transportation_name_linestring_gen2 -CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen2 AS ( +CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring_gen1 WHERE highway IN ('motorway','trunk') AND ST_Length(geometry) > 14000 @@ -55,9 +49,34 @@ CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen2 AS ( CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_geometry_idx ON osm_transportation_name_linestring_gen2 USING gist(geometry); -- etldoc: osm_transportation_name_linestring_gen2 -> osm_transportation_name_linestring_gen3 -CREATE TABLE IF NOT EXISTS osm_transportation_name_linestring_gen3 AS ( +CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring_gen2 WHERE highway = 'motorway' AND ST_Length(geometry) > 20000 ); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx ON osm_transportation_name_linestring_gen3 USING gist(geometry); + +--- Triggers + +CREATE OR REPLACE FUNCTION refresh_osm_transportation_name_linestring() RETURNS trigger AS + $BODY$ + BEGIN + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen1 CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen2 CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen3 CONCURRENTLY; + RETURN null; + END; + $BODY$ +language plpgsql + +CREATE TRIGGER trigger_refresh_osm_transportation_name_linestring + AFTER INSERT OR UPDATE OR DELETE ON osm_highway_linestring + FOR EACH STATEMENT + EXECUTE PROCEDURE refresh_osm_transportation_name_linestring + + + + + + diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index 16522e8..7110a09 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -1,9 +1,7 @@ -DROP TABLE IF EXISTS osm_water_lakeline CASCADE; - -- etldoc: osm_water_polygon -> osm_water_lakeline -- etldoc: lake_centerline -> osm_water_lakeline -CREATE TABLE IF NOT EXISTS osm_water_lakeline AS ( +CREATE MATERIALIZED VIEW osm_water_lakeline AS ( SELECT wp.osm_id, ll.wkb_geometry AS geometry, name, name_en, ST_Area(wp.geometry) AS area @@ -11,5 +9,20 @@ CREATE TABLE IF NOT EXISTS osm_water_lakeline AS ( INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id WHERE wp.name <> '' ); - CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline USING gist(geometry); + +-- Triggers + +CREATE OR REPLACE FUNCTION refresh_osm_water_lakeline() RETURNS trigger AS + $BODY$ + BEGIN + REFRESH MATERIALIZED VIEW osm_water_lakeline CONCURRENTLY; + RETURN null; + END; + $BODY$ +language plpgsql + +CREATE TRIGGER trigger_refresh_osm_water_lakeline + AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon + FOR EACH STATEMENT + EXECUTE PROCEDURE refresh_osm_water_lakeline diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 498a6e8..495aeab 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -1,9 +1,7 @@ -DROP TABLE IF EXISTS osm_water_point CASCADE; - -- etldoc: osm_water_polygon -> osm_water_point -- etldoc: lake_centerline -> osm_water_point -CREATE TABLE IF NOT EXISTS osm_water_point AS ( +CREATE MATERIALIZED VIEW osm_water_point AS ( SELECT wp.osm_id, topoint(wp.geometry) AS geometry, wp.name, wp.name_en, ST_Area(wp.geometry) AS area @@ -11,5 +9,20 @@ CREATE TABLE IF NOT EXISTS osm_water_point AS ( LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id WHERE ll.osm_id IS NULL AND wp.name <> '' ); - CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry); + +-- Triggers + +CREATE OR REPLACE FUNCTION refresh_osm_water_point() RETURNS trigger AS + $BODY$ + BEGIN + REFRESH MATERIALIZED VIEW osm_water_point CONCURRENTLY; + RETURN null; + END; + $BODY$ +language plpgsql + +CREATE TRIGGER trigger_refresh_osm_water_point + AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon + FOR EACH STATEMENT + EXECUTE PROCEDURE refresh_osm_water_point diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index 604e004..2d75f2c 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -1,15 +1,10 @@ -DROP TABLE IF EXISTS osm_important_waterway_linestring CASCADE; -DROP TABLE IF EXISTS osm_important_waterway_linestring_gen1 CASCADE; -DROP TABLE IF EXISTS osm_important_waterway_linestring_gen2 CASCADE; -DROP TABLE IF EXISTS osm_important_waterway_linestring_gen3 CASCADE; - -- We merge the waterways by name like the highways -- This helps to drop not important rivers (since they do not have a name) -- and also makes it possible to filter out too short rivers -- etldoc: osm_waterway_linestring -> osm_important_waterway_linestring -CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring AS ( +CREATE MATERIALIZED VIEW osm_important_waterway_linestring AS ( SELECT (ST_Dump(geometry)).geom AS geometry, name @@ -22,11 +17,10 @@ CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring AS ( GROUP BY name ) AS waterway_union ); - CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_geometry_idx ON osm_important_waterway_linestring USING gist(geometry); -- etldoc: osm_important_waterway_linestring -> osm_important_waterway_linestring_gen1 -CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_gen1 AS ( +CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 60) AS geometry, name FROM osm_important_waterway_linestring WHERE ST_Length(geometry) > 1000 @@ -34,7 +28,7 @@ CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_gen1 AS ( CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen1_geometry_idx ON osm_important_waterway_linestring_gen1 USING gist(geometry); -- etldoc: osm_important_waterway_linestring_gen1 -> osm_important_waterway_linestring_gen2 -CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_gen2 AS ( +CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 100) AS geometry, name FROM osm_important_waterway_linestring_gen1 WHERE ST_Length(geometry) > 4000 @@ -42,9 +36,32 @@ CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_gen2 AS ( CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen2_geometry_idx ON osm_important_waterway_linestring_gen2 USING gist(geometry); -- etldoc: osm_important_waterway_linestring_gen2 -> osm_important_waterway_linestring_gen3 -CREATE TABLE IF NOT EXISTS osm_important_waterway_linestring_gen3 AS ( +CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 200) AS geometry, name FROM osm_important_waterway_linestring_gen2 WHERE ST_Length(geometry) > 8000 ); CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen3_geometry_idx ON osm_important_waterway_linestring_gen3 USING gist(geometry); + +--- Triggers + +CREATE OR REPLACE FUNCTION refresh_osm_important_waterway_linestring() RETURNS trigger AS + $BODY$ + BEGIN + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen1 CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen2 CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen3 CONCURRENTLY; + RETURN null; + END; + $BODY$ +language plpgsql + +CREATE TRIGGER trigger_refresh_osm_important_waterway_linestring + AFTER INSERT OR UPDATE OR DELETE ON osm_waterway_linestring + FOR EACH STATEMENT + EXECUTE PROCEDURE refresh_osm_important_waterway_linestring + + + + From 53c29f65cd1c515c8fc8ce1b23ce29880fc68765 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Tue, 3 Jan 2017 23:21:52 -0500 Subject: [PATCH 02/54] fix semicolon --- layers/transportation_name/merge_highways.sql | 2 +- layers/water_name/water_lakeline.sql | 2 +- layers/water_name/water_point.sql | 2 +- layers/waterway/merge_waterway.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 18e92a9..05fa64b 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -73,7 +73,7 @@ language plpgsql CREATE TRIGGER trigger_refresh_osm_transportation_name_linestring AFTER INSERT OR UPDATE OR DELETE ON osm_highway_linestring FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_transportation_name_linestring + EXECUTE PROCEDURE refresh_osm_transportation_name_linestring; diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index 7110a09..d9cc9d7 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -25,4 +25,4 @@ language plpgsql CREATE TRIGGER trigger_refresh_osm_water_lakeline AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_water_lakeline + EXECUTE PROCEDURE refresh_osm_water_lakeline; diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 495aeab..e23ea9a 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -25,4 +25,4 @@ language plpgsql CREATE TRIGGER trigger_refresh_osm_water_point AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_water_point + EXECUTE PROCEDURE refresh_osm_water_point; diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index 2d75f2c..1c27559 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -60,7 +60,7 @@ language plpgsql CREATE TRIGGER trigger_refresh_osm_important_waterway_linestring AFTER INSERT OR UPDATE OR DELETE ON osm_waterway_linestring FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_important_waterway_linestring + EXECUTE PROCEDURE refresh_osm_important_waterway_linestring; From 0c1da272947b51a70fc466dfaa5178632e567d9a Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Tue, 3 Jan 2017 23:35:39 -0500 Subject: [PATCH 03/54] Fix pile of syntax issues --- layers/transportation_name/merge_highways.sql | 12 ++++++------ layers/water_name/water_lakeline.sql | 6 +++--- layers/water_name/water_point.sql | 6 +++--- layers/waterway/merge_waterway.sql | 12 ++++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 05fa64b..76debce 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -61,19 +61,19 @@ CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx CREATE OR REPLACE FUNCTION refresh_osm_transportation_name_linestring() RETURNS trigger AS $BODY$ BEGIN - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring CONCURRENTLY; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen1 CONCURRENTLY; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen2 CONCURRENTLY; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen3 CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen1; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen2; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen3; RETURN null; END; $BODY$ -language plpgsql +language plpgsql; CREATE TRIGGER trigger_refresh_osm_transportation_name_linestring AFTER INSERT OR UPDATE OR DELETE ON osm_highway_linestring FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_transportation_name_linestring; + EXECUTE PROCEDURE refresh_osm_transportation_name_linestring(); diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index d9cc9d7..dcc96e9 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -16,13 +16,13 @@ CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline CREATE OR REPLACE FUNCTION refresh_osm_water_lakeline() RETURNS trigger AS $BODY$ BEGIN - REFRESH MATERIALIZED VIEW osm_water_lakeline CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_water_lakeline; RETURN null; END; $BODY$ -language plpgsql +language plpgsql; CREATE TRIGGER trigger_refresh_osm_water_lakeline AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_water_lakeline; + EXECUTE PROCEDURE refresh_osm_water_lakeline(); diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index e23ea9a..f736dc9 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -16,13 +16,13 @@ CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING CREATE OR REPLACE FUNCTION refresh_osm_water_point() RETURNS trigger AS $BODY$ BEGIN - REFRESH MATERIALIZED VIEW osm_water_point CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_water_point; RETURN null; END; $BODY$ -language plpgsql +language plpgsql; CREATE TRIGGER trigger_refresh_osm_water_point AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_water_point; + EXECUTE PROCEDURE refresh_osm_water_point(); diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index 1c27559..7b3c5db 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -48,19 +48,19 @@ CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen3_geometry_idx O CREATE OR REPLACE FUNCTION refresh_osm_important_waterway_linestring() RETURNS trigger AS $BODY$ BEGIN - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring CONCURRENTLY; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen1 CONCURRENTLY; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen2 CONCURRENTLY; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen3 CONCURRENTLY; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen1; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen2; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen3; RETURN null; END; $BODY$ -language plpgsql +language plpgsql; CREATE TRIGGER trigger_refresh_osm_important_waterway_linestring AFTER INSERT OR UPDATE OR DELETE ON osm_waterway_linestring FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_important_waterway_linestring; + EXECUTE PROCEDURE refresh_osm_important_waterway_linestring(); From 2ef6a19149c492fe29904c41899cc668b625bb73 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 01:03:00 -0500 Subject: [PATCH 04/54] when create materialized do so without data, better logging --- layers/transportation_name/merge_highways.sql | 9 +++++---- layers/water_name/water_lakeline.sql | 3 ++- layers/water_name/water_point.sql | 3 ++- layers/waterway/merge_waterway.sql | 9 +++++---- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 76debce..59b75f8 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -29,7 +29,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring AS ( WHERE (name <> '' OR ref <> '') AND NULLIF(highway, '') IS NOT NULL GROUP BY name, highway, ref ) AS highway_union -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_geometry_idx ON osm_transportation_name_linestring USING gist(geometry); -- etldoc: osm_transportation_name_linestring -> osm_transportation_name_linestring_gen1 @@ -37,7 +37,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 50) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring WHERE highway IN ('motorway','trunk') AND ST_Length(geometry) > 8000 -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_geometry_idx ON osm_transportation_name_linestring_gen1 USING gist(geometry); -- etldoc: osm_transportation_name_linestring_gen1 -> osm_transportation_name_linestring_gen2 @@ -45,7 +45,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring_gen1 WHERE highway IN ('motorway','trunk') AND ST_Length(geometry) > 14000 -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_geometry_idx ON osm_transportation_name_linestring_gen2 USING gist(geometry); -- etldoc: osm_transportation_name_linestring_gen2 -> osm_transportation_name_linestring_gen3 @@ -53,7 +53,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring_gen2 WHERE highway = 'motorway' AND ST_Length(geometry) > 20000 -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx ON osm_transportation_name_linestring_gen3 USING gist(geometry); --- Triggers @@ -61,6 +61,7 @@ CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx CREATE OR REPLACE FUNCTION refresh_osm_transportation_name_linestring() RETURNS trigger AS $BODY$ BEGIN + RAISE LOG 'Refresh osm_highway_linestring based tables'; REFRESH MATERIALIZED VIEW osm_transportation_name_linestring; REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen1; REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen2; diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index dcc96e9..bf45b89 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -8,7 +8,7 @@ CREATE MATERIALIZED VIEW osm_water_lakeline AS ( FROM osm_water_polygon AS wp INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id WHERE wp.name <> '' -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline USING gist(geometry); -- Triggers @@ -16,6 +16,7 @@ CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline CREATE OR REPLACE FUNCTION refresh_osm_water_lakeline() RETURNS trigger AS $BODY$ BEGIN + RAISE LOG 'Refresh osm_water_lakeline based tables'; REFRESH MATERIALIZED VIEW osm_water_lakeline; RETURN null; END; diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index f736dc9..4d84ae9 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -8,7 +8,7 @@ CREATE MATERIALIZED VIEW osm_water_point AS ( FROM osm_water_polygon AS wp LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id WHERE ll.osm_id IS NULL AND wp.name <> '' -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry); -- Triggers @@ -16,6 +16,7 @@ CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING CREATE OR REPLACE FUNCTION refresh_osm_water_point() RETURNS trigger AS $BODY$ BEGIN + RAISE LOG 'Refresh osm_water polygon based tables'; REFRESH MATERIALIZED VIEW osm_water_point; RETURN null; END; diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index 7b3c5db..efb2d71 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -16,7 +16,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring AS ( WHERE name <> '' AND waterway = 'river' GROUP BY name ) AS waterway_union -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_geometry_idx ON osm_important_waterway_linestring USING gist(geometry); -- etldoc: osm_important_waterway_linestring -> osm_important_waterway_linestring_gen1 @@ -24,7 +24,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 60) AS geometry, name FROM osm_important_waterway_linestring WHERE ST_Length(geometry) > 1000 -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen1_geometry_idx ON osm_important_waterway_linestring_gen1 USING gist(geometry); -- etldoc: osm_important_waterway_linestring_gen1 -> osm_important_waterway_linestring_gen2 @@ -32,7 +32,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 100) AS geometry, name FROM osm_important_waterway_linestring_gen1 WHERE ST_Length(geometry) > 4000 -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen2_geometry_idx ON osm_important_waterway_linestring_gen2 USING gist(geometry); -- etldoc: osm_important_waterway_linestring_gen2 -> osm_important_waterway_linestring_gen3 @@ -40,7 +40,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 200) AS geometry, name FROM osm_important_waterway_linestring_gen2 WHERE ST_Length(geometry) > 8000 -); +) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen3_geometry_idx ON osm_important_waterway_linestring_gen3 USING gist(geometry); --- Triggers @@ -48,6 +48,7 @@ CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen3_geometry_idx O CREATE OR REPLACE FUNCTION refresh_osm_important_waterway_linestring() RETURNS trigger AS $BODY$ BEGIN + RAISE LOG 'Refresh osm_waterway_linestring based tables'; REFRESH MATERIALIZED VIEW osm_important_waterway_linestring; REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen1; REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen2; From 3c4205055bd14a04fecde07825cb241963d19ed0 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 11:49:14 -0500 Subject: [PATCH 05/54] Refresh concurrently --- layers/transportation_name/merge_highways.sql | 8 ++++---- layers/water_name/water_lakeline.sql | 2 +- layers/water_name/water_point.sql | 2 +- layers/waterway/merge_waterway.sql | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 59b75f8..d2310d1 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -62,10 +62,10 @@ CREATE OR REPLACE FUNCTION refresh_osm_transportation_name_linestring() RETURNS $BODY$ BEGIN RAISE LOG 'Refresh osm_highway_linestring based tables'; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen1; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen2; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen3; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen1; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen2; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen3; RETURN null; END; $BODY$ diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index bf45b89..0d66c69 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -17,7 +17,7 @@ CREATE OR REPLACE FUNCTION refresh_osm_water_lakeline() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh osm_water_lakeline based tables'; - REFRESH MATERIALIZED VIEW osm_water_lakeline; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_water_lakeline; RETURN null; END; $BODY$ diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 4d84ae9..7c70f39 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -17,7 +17,7 @@ CREATE OR REPLACE FUNCTION refresh_osm_water_point() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh osm_water polygon based tables'; - REFRESH MATERIALIZED VIEW osm_water_point; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_water_point; RETURN null; END; $BODY$ diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index efb2d71..be2a602 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -49,10 +49,10 @@ CREATE OR REPLACE FUNCTION refresh_osm_important_waterway_linestring() RETURNS t $BODY$ BEGIN RAISE LOG 'Refresh osm_waterway_linestring based tables'; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen1; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen2; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen3; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen1; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen2; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen3; RETURN null; END; $BODY$ From 35a5ba1ade61ec53aed66771065dd80c774b9b9b Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 15:53:58 -0500 Subject: [PATCH 06/54] Lakelines refresh once at commit time. --- layers/water_name/water_lakeline.sql | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index 0d66c69..d122371 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -11,14 +11,23 @@ CREATE MATERIALIZED VIEW osm_water_lakeline AS ( ) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline USING gist(geometry); --- Triggers +-- Handle updates + +CREATE TABLE IF NOT EXISTS updates_osm_water_polygon(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION flag_update_osm_water_polygon() RETURNS trigger AS $$ +BEGIN + INSERT INTO updates_osm_water_polygon(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; CREATE OR REPLACE FUNCTION refresh_osm_water_lakeline() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh osm_water_lakeline based tables'; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_water_lakeline; - RETURN null; + REFRESH MATERIALIZED VIEW osm_water_lakeline; + DELETE FROM updates_osm_water_polygon; + RETURN null; END; $BODY$ language plpgsql; @@ -26,4 +35,10 @@ language plpgsql; CREATE TRIGGER trigger_refresh_osm_water_lakeline AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT + EXECUTE PROCEDURE flag_update_osm_water_polygon(); + +CREATE CONSTRAINT TRIGGER commit_osm_water_polygon + AFTER INSERT ON updates_osm_water_polygon + INITIALLY DEFERRED + FOR EACH ROW EXECUTE PROCEDURE refresh_osm_water_lakeline(); From 2fd3be14037eeb17eded3f26c534a2785f890618 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 16:59:42 -0500 Subject: [PATCH 07/54] Use schemas to update on commit, untested --- layers/transportation_name/merge_highways.sql | 38 ++++++++++++++----- layers/water_name/water_lakeline.sql | 24 ++++++------ layers/water_name/water_point.sql | 31 +++++++++++---- layers/waterway/merge_waterway.sql | 38 +++++++++++++------ 4 files changed, 92 insertions(+), 39 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index d2310d1..2664db4 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -56,25 +56,43 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen3 AS ( ) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx ON osm_transportation_name_linestring_gen3 USING gist(geometry); ---- Triggers +-- Handle updates -CREATE OR REPLACE FUNCTION refresh_osm_transportation_name_linestring() RETURNS trigger AS +CREATE SCHEMA transportation_name; + +CREATE TABLE IF NOT EXISTS transportation_name.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION transportation_name.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO transportation_name.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION transportation_name.refresh() RETURNS trigger AS $BODY$ BEGIN - RAISE LOG 'Refresh osm_highway_linestring based tables'; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen1; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen2; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen3; - RETURN null; + RAISE LOG 'Refresh transportation_name'; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen1; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen2; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen3; + DELETE FROM transportation_name.updates; + RETURN null; END; $BODY$ language plpgsql; -CREATE TRIGGER trigger_refresh_osm_transportation_name_linestring +CREATE TRIGGER transportation_name.trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_highway_linestring FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_transportation_name_linestring(); + EXECUTE PROCEDURE transportation_name.flag(); + +CREATE CONSTRAINT TRIGGER transportation_name.trigger_refresh + AFTER INSERT ON transportation_name.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE transportation_name.refresh(); + diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index d122371..b52355f 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -13,32 +13,34 @@ CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline -- Handle updates -CREATE TABLE IF NOT EXISTS updates_osm_water_polygon(id serial primary key, t text, unique (t)); -CREATE OR REPLACE FUNCTION flag_update_osm_water_polygon() RETURNS trigger AS $$ +CREATE SCHEMA water_lakeline; + +CREATE TABLE IF NOT EXISTS water_lakeline.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION water_lakeline.flag() RETURNS trigger AS $$ BEGIN - INSERT INTO updates_osm_water_polygon(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + INSERT INTO water_lakeline.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; RETURN null; END; $$ language plpgsql; -CREATE OR REPLACE FUNCTION refresh_osm_water_lakeline() RETURNS trigger AS +CREATE OR REPLACE FUNCTION water_lakeline.refresh() RETURNS trigger AS $BODY$ BEGIN - RAISE LOG 'Refresh osm_water_lakeline based tables'; + RAISE LOG 'Refresh water_lakeline'; REFRESH MATERIALIZED VIEW osm_water_lakeline; - DELETE FROM updates_osm_water_polygon; + DELETE FROM water_lakeline.updates; RETURN null; END; $BODY$ language plpgsql; -CREATE TRIGGER trigger_refresh_osm_water_lakeline +CREATE TRIGGER water_lakeline.trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT - EXECUTE PROCEDURE flag_update_osm_water_polygon(); + EXECUTE PROCEDURE water_lakeline.flag(); -CREATE CONSTRAINT TRIGGER commit_osm_water_polygon - AFTER INSERT ON updates_osm_water_polygon +CREATE CONSTRAINT TRIGGER water_lakeline.trigger_refresh + AFTER INSERT ON water_lakeline.updates INITIALLY DEFERRED FOR EACH ROW - EXECUTE PROCEDURE refresh_osm_water_lakeline(); + EXECUTE PROCEDURE water_lakeline.refresh(); diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 7c70f39..c61d5a7 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -11,19 +11,36 @@ CREATE MATERIALIZED VIEW osm_water_point AS ( ) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry); --- Triggers +-- Handle updates -CREATE OR REPLACE FUNCTION refresh_osm_water_point() RETURNS trigger AS +CREATE SCHEMA water_name; + +CREATE TABLE IF NOT EXISTS water_name.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION water_name.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO water_name.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS $BODY$ BEGIN - RAISE LOG 'Refresh osm_water polygon based tables'; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_water_point; - RETURN null; + RAISE LOG 'Refresh water_name'; + REFRESH MATERIALIZED VIEW osm_water_point; + DELETE FROM water_name.updates; + RETURN null; END; $BODY$ language plpgsql; -CREATE TRIGGER trigger_refresh_osm_water_point +CREATE TRIGGER water_name.trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_water_point(); + EXECUTE PROCEDURE water_name.flag(); + +CREATE CONSTRAINT TRIGGER water_name.trigger_refresh + AFTER INSERT ON water_name.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE water_name.refresh(); diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index be2a602..321fbf6 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -43,26 +43,42 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen3 AS ( ) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen3_geometry_idx ON osm_important_waterway_linestring_gen3 USING gist(geometry); ---- Triggers +-- Handle updates -CREATE OR REPLACE FUNCTION refresh_osm_important_waterway_linestring() RETURNS trigger AS +CREATE SCHEMA waterway; + +CREATE TABLE IF NOT EXISTS waterway.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION waterway.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO waterway.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION waterway.refresh() RETURNS trigger AS $BODY$ BEGIN - RAISE LOG 'Refresh osm_waterway_linestring based tables'; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen1; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen2; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen3; - RETURN null; + RAISE LOG 'Refresh transportation_name'; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen1; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen2; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen3; + DELETE FROM waterway.updates; + RETURN null; END; $BODY$ language plpgsql; -CREATE TRIGGER trigger_refresh_osm_important_waterway_linestring +CREATE TRIGGER waterway.trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_waterway_linestring FOR EACH STATEMENT - EXECUTE PROCEDURE refresh_osm_important_waterway_linestring(); - + EXECUTE PROCEDURE waterway.flag(); + +CREATE CONSTRAINT TRIGGER waterway.trigger_refresh + AFTER INSERT ON waterway.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE waterway.refresh(); From b97e5ad9fbc63eacc8c1b3798a6e4980ca55dee1 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 17:40:52 -0500 Subject: [PATCH 08/54] Convert housenumber point conversion to function --- layers/housenumber/housenumber_centroid.sql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/layers/housenumber/housenumber_centroid.sql b/layers/housenumber/housenumber_centroid.sql index 8a81d6a..101f701 100644 --- a/layers/housenumber/housenumber_centroid.sql +++ b/layers/housenumber/housenumber_centroid.sql @@ -1,4 +1,9 @@ -- etldoc: osm_housenumber_point -> osm_housenumber_point -UPDATE osm_housenumber_point SET geometry=topoint(geometry) -WHERE ST_GeometryType(geometry) <> 'ST_Point'; +CREATE FUNCTION convert_housenumber_point() RETURNS VOID AS $$ +BEGIN + UPDATE osm_housenumber_point SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; +END; +$$ LANGUAGE plpgsql; + +SELECT convert_housenumber_point(); From f1ee63e4bb90cee9ea5eadd3ec79b53fdf27c926 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 17:42:05 -0500 Subject: [PATCH 09/54] poi topoint function --- layers/poi/poi_polygon_update.sql | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/layers/poi/poi_polygon_update.sql b/layers/poi/poi_polygon_update.sql index 1e5073d..2840f29 100644 --- a/layers/poi/poi_polygon_update.sql +++ b/layers/poi/poi_polygon_update.sql @@ -1,5 +1,11 @@ -- etldoc: osm_poi_polygon -> osm_poi_polygon -UPDATE osm_poi_polygon SET geometry=topoint(geometry) -WHERE ST_GeometryType(geometry) <> 'ST_Point'; -ANALYZE osm_poi_polygon; +CREATE FUNCTION convert_poi_point() RETURNS VOID AS $$ +BEGIN + UPDATE osm_poi_polygon SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; + ANALYZE osm_poi_polygon; +END; +$$ LANGUAGE plpgsql; + +SELECT convert_poi_point(); + From 8b4777fd79d36eaba2844ae72ab166c23062f731 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 17:54:15 -0500 Subject: [PATCH 10/54] poi and housenumber triggers --- layers/housenumber/housenumber_centroid.sql | 34 +++++++++++++++++++++ layers/poi/poi_polygon_update.sql | 33 ++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/layers/housenumber/housenumber_centroid.sql b/layers/housenumber/housenumber_centroid.sql index 101f701..1d1cf83 100644 --- a/layers/housenumber/housenumber_centroid.sql +++ b/layers/housenumber/housenumber_centroid.sql @@ -7,3 +7,37 @@ END; $$ LANGUAGE plpgsql; SELECT convert_housenumber_point(); + +-- Handle updates + +CREATE SCHEMA housenumber; + +CREATE TABLE IF NOT EXISTS housenumber.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION housenumber.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO housenumber.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION housenumber.refresh() RETURNS trigger AS + $BODY$ + BEGIN + RAISE LOG 'Refresh housenumber'; + SELECT convert_housenumber_point(); + DELETE FROM housenumber.updates; + RETURN null; + END; + $BODY$ +language plpgsql; + +CREATE TRIGGER trigger_flag + AFTER INSERT OR UPDATE OR DELETE ON osm_housenumber_point + FOR EACH STATEMENT + EXECUTE PROCEDURE housenumber.flag(); + +CREATE CONSTRAINT TRIGGER trigger_refresh + AFTER INSERT ON housenumber.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE housenumber.refresh(); diff --git a/layers/poi/poi_polygon_update.sql b/layers/poi/poi_polygon_update.sql index 2840f29..51259e5 100644 --- a/layers/poi/poi_polygon_update.sql +++ b/layers/poi/poi_polygon_update.sql @@ -9,3 +9,36 @@ $$ LANGUAGE plpgsql; SELECT convert_poi_point(); +-- Handle updates + +CREATE SCHEMA poi; + +CREATE TABLE IF NOT EXISTS poi.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION poi.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO poi.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION poi.refresh() RETURNS trigger AS + $BODY$ + BEGIN + RAISE LOG 'Refresh poi'; + SELECT convert_poi_point(); + DELETE FROM poi.updates; + RETURN null; + END; + $BODY$ +language plpgsql; + +CREATE TRIGGER trigger_flag + AFTER INSERT OR UPDATE OR DELETE ON osm_poi_polygon + FOR EACH STATEMENT + EXECUTE PROCEDURE poi.flag(); + +CREATE CONSTRAINT TRIGGER trigger_refresh + AFTER INSERT ON poi.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE poi.refresh(); From c5d4c1ea805b003d9ae2bf0a8f8e5f96d1fbfedc Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 17:56:51 -0500 Subject: [PATCH 11/54] Fix the name of triggers, cannot be schema-qualified --- layers/transportation_name/merge_highways.sql | 13 +++---------- layers/water_name/water_lakeline.sql | 4 ++-- layers/water_name/water_point.sql | 4 ++-- layers/waterway/merge_waterway.sql | 4 ++-- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 2664db4..7a81a87 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -82,20 +82,13 @@ CREATE OR REPLACE FUNCTION transportation_name.refresh() RETURNS trigger AS $BODY$ language plpgsql; -CREATE TRIGGER transportation_name.trigger_flag +CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_highway_linestring FOR EACH STATEMENT EXECUTE PROCEDURE transportation_name.flag(); -CREATE CONSTRAINT TRIGGER transportation_name.trigger_refresh +CREATE CONSTRAINT TRIGGER trigger_refresh AFTER INSERT ON transportation_name.updates INITIALLY DEFERRED FOR EACH ROW - EXECUTE PROCEDURE transportation_name.refresh(); - - - - - - - + EXECUTE PROCEDURE transportation_name.refresh(); \ No newline at end of file diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index b52355f..fc776db 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -34,12 +34,12 @@ CREATE OR REPLACE FUNCTION water_lakeline.refresh() RETURNS trigger AS $BODY$ language plpgsql; -CREATE TRIGGER water_lakeline.trigger_flag +CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT EXECUTE PROCEDURE water_lakeline.flag(); -CREATE CONSTRAINT TRIGGER water_lakeline.trigger_refresh +CREATE CONSTRAINT TRIGGER trigger_refresh AFTER INSERT ON water_lakeline.updates INITIALLY DEFERRED FOR EACH ROW diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index c61d5a7..f938b76 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -34,12 +34,12 @@ CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS $BODY$ language plpgsql; -CREATE TRIGGER water_name.trigger_flag +CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT EXECUTE PROCEDURE water_name.flag(); -CREATE CONSTRAINT TRIGGER water_name.trigger_refresh +CREATE CONSTRAINT TRIGGER trigger_refresh AFTER INSERT ON water_name.updates INITIALLY DEFERRED FOR EACH ROW diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index 321fbf6..ec2628f 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -69,12 +69,12 @@ CREATE OR REPLACE FUNCTION waterway.refresh() RETURNS trigger AS $BODY$ language plpgsql; -CREATE TRIGGER waterway.trigger_flag +CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_waterway_linestring FOR EACH STATEMENT EXECUTE PROCEDURE waterway.flag(); -CREATE CONSTRAINT TRIGGER waterway.trigger_refresh +CREATE CONSTRAINT TRIGGER trigger_refresh AFTER INSERT ON waterway.updates INITIALLY DEFERRED FOR EACH ROW From 9573aca4f2f509c5ff136398f962c98592a6ff67 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 4 Jan 2017 19:53:16 -0500 Subject: [PATCH 12/54] place island topoint function --- layers/place/island_polygon_update.sql | 44 ++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/layers/place/island_polygon_update.sql b/layers/place/island_polygon_update.sql index 7b5c44c..57cca41 100644 --- a/layers/place/island_polygon_update.sql +++ b/layers/place/island_polygon_update.sql @@ -1,5 +1,43 @@ -- etldoc: osm_island_polygon -> osm_island_polygon -UPDATE osm_island_polygon SET geometry=topoint(geometry) -WHERE ST_GeometryType(geometry) <> 'ST_Point'; +CREATE FUNCTION convert_island_polygon_point() RETURNS VOID AS $$ +BEGIN + UPDATE osm_island_polygon SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; + ANALYZE osm_island_polygon; +END; +$$ LANGUAGE plpgsql; -ANALYZE osm_island_polygon; +SELECT convert_island_polygon_point(); + +-- Handle updates + +CREATE SCHEMA place_island; + +CREATE TABLE IF NOT EXISTS place_island.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION place_island.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO place_island.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION place_island.refresh() RETURNS trigger AS + $BODY$ + BEGIN + RAISE LOG 'Refresh place_island'; + SELECT convert_island_polygon_point(); + DELETE FROM place_island.updates; + RETURN null; + END; + $BODY$ +language plpgsql; + +CREATE TRIGGER trigger_flag + AFTER INSERT OR UPDATE OR DELETE ON osm_island_polygon + FOR EACH STATEMENT + EXECUTE PROCEDURE place_island.flag(); + +CREATE CONSTRAINT TRIGGER trigger_refresh + AFTER INSERT ON place_island.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE place_island.refresh(); From c5f05e7c3a156d2754d105308bcb7f974492ef02 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Thu, 5 Jan 2017 08:59:15 -0500 Subject: [PATCH 13/54] Functions and triggers for state, country and city ranks --- layers/place/merge_city_rank.sql | 105 ++++++++++++++++++++-------- layers/place/merge_country_rank.sql | 92 +++++++++++++++++------- layers/place/merge_state_rank.sql | 84 ++++++++++++++++------ 3 files changed, 204 insertions(+), 77 deletions(-) diff --git a/layers/place/merge_city_rank.sql b/layers/place/merge_city_rank.sql index 9c68a7e..78587ab 100644 --- a/layers/place/merge_city_rank.sql +++ b/layers/place/merge_city_rank.sql @@ -1,38 +1,81 @@ CREATE EXTENSION IF NOT EXISTS unaccent; --- Clear OSM key:rank ( https://github.com/openmaptiles/openmaptiles/issues/108 ) --- etldoc: osm_city_point -> osm_city_point -UPDATE osm_city_point AS osm SET "rank" = NULL WHERE "rank" IS NOT NULL; --- etldoc: ne_10m_populated_places -> osm_city_point --- etldoc: osm_city_point -> osm_city_point +CREATE FUNCTION update_osm_city_point() RETURNS VOID AS $$ +BEGIN -WITH important_city_point AS ( - SELECT osm.geometry, osm.osm_id, osm.name, osm.name_en, ne.scalerank, ne.labelrank - FROM ne_10m_populated_places AS ne, osm_city_point AS osm - WHERE - ( - ne.name ILIKE osm.name OR - ne.name ILIKE osm.name_en OR - ne.namealt ILIKE osm.name OR - ne.namealt ILIKE osm.name_en OR - ne.meganame ILIKE osm.name OR - ne.meganame ILIKE osm.name_en OR - ne.gn_ascii ILIKE osm.name OR - ne.gn_ascii ILIKE osm.name_en OR - ne.nameascii ILIKE osm.name OR - ne.nameascii ILIKE osm.name_en OR - ne.name = unaccent(osm.name) - ) - AND osm.place IN ('city', 'town', 'village') - AND ST_DWithin(ne.geometry, osm.geometry, 50000) -) -UPDATE osm_city_point AS osm --- Move scalerank to range 1 to 10 and merge scalerank 5 with 6 since not enough cities --- are in the scalerank 5 bucket -SET "rank" = CASE WHEN scalerank <= 5 THEN scalerank + 1 ELSE scalerank END -FROM important_city_point AS ne -WHERE osm.osm_id = ne.osm_id; + -- Clear OSM key:rank ( https://github.com/openmaptiles/openmaptiles/issues/108 ) + -- etldoc: osm_city_point -> osm_city_point + UPDATE osm_city_point AS osm SET "rank" = NULL WHERE "rank" IS NOT NULL; + + -- etldoc: ne_10m_populated_places -> osm_city_point + -- etldoc: osm_city_point -> osm_city_point + + WITH important_city_point AS ( + SELECT osm.geometry, osm.osm_id, osm.name, osm.name_en, ne.scalerank, ne.labelrank + FROM ne_10m_populated_places AS ne, osm_city_point AS osm + WHERE + ( + ne.name ILIKE osm.name OR + ne.name ILIKE osm.name_en OR + ne.namealt ILIKE osm.name OR + ne.namealt ILIKE osm.name_en OR + ne.meganame ILIKE osm.name OR + ne.meganame ILIKE osm.name_en OR + ne.gn_ascii ILIKE osm.name OR + ne.gn_ascii ILIKE osm.name_en OR + ne.nameascii ILIKE osm.name OR + ne.nameascii ILIKE osm.name_en OR + ne.name = unaccent(osm.name) + ) + AND osm.place IN ('city', 'town', 'village') + AND ST_DWithin(ne.geometry, osm.geometry, 50000) + ) + UPDATE osm_city_point AS osm + -- Move scalerank to range 1 to 10 and merge scalerank 5 with 6 since not enough cities + -- are in the scalerank 5 bucket + SET "rank" = CASE WHEN scalerank <= 5 THEN scalerank + 1 ELSE scalerank END + FROM important_city_point AS ne + WHERE osm.osm_id = ne.osm_id; + +END; +$$ LANGUAGE plpgsql; + +SELECT update_osm_city_point(); CREATE INDEX IF NOT EXISTS osm_city_point_rank_idx ON osm_city_point("rank"); + +-- Handle updates + +CREATE SCHEMA place_city; + +CREATE TABLE IF NOT EXISTS place_city.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION place_city.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO place_city.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION place_city.refresh() RETURNS trigger AS + $BODY$ + BEGIN + RAISE LOG 'Refresh place_city rank'; + SELECT update_osm_city_point(); + DELETE FROM place_city.updates; + RETURN null; + END; + $BODY$ +language plpgsql; + +CREATE TRIGGER trigger_flag + AFTER INSERT OR UPDATE OR DELETE ON osm_city_point + FOR EACH STATEMENT + EXECUTE PROCEDURE place_city.flag(); + +CREATE CONSTRAINT TRIGGER trigger_refresh + AFTER INSERT ON place_city.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE place_city.refresh(); diff --git a/layers/place/merge_country_rank.sql b/layers/place/merge_country_rank.sql index f1489e3..95a164a 100644 --- a/layers/place/merge_country_rank.sql +++ b/layers/place/merge_country_rank.sql @@ -3,32 +3,74 @@ ALTER TABLE osm_country_point DROP CONSTRAINT IF EXISTS osm_country_point_rank_c -- etldoc: ne_10m_admin_0_countries -> osm_country_point -- etldoc: osm_country_point -> osm_country_point -WITH important_country_point AS ( - SELECT osm.geometry, osm.osm_id, osm.name, COALESCE(NULLIF(osm.name_en, ''), ne.name) AS name_en, ne.scalerank, ne.labelrank - FROM ne_10m_admin_0_countries AS ne, osm_country_point AS osm - WHERE - -- We only match whether the point is within the Natural Earth polygon - -- because name matching is to difficult since OSM does not contain good - -- enough coverage of ISO codesy - ST_Within(osm.geometry, ne.geometry) - -- We leave out tiny countries - AND ne.scalerank <= 1 -) -UPDATE osm_country_point AS osm --- Normalize both scalerank and labelrank into a ranking system from 1 to 6 --- where the ranks are still distributed uniform enough across all countries -SET "rank" = LEAST(6, CEILING((scalerank + labelrank)/2.0)) -FROM important_country_point AS ne -WHERE osm.osm_id = ne.osm_id; +CREATE FUNCTION update_osm_country_point() RETURNS VOID AS $$ +BEGIN -UPDATE osm_country_point AS osm -SET "rank" = 6 -WHERE "rank" IS NULL; + WITH important_country_point AS ( + SELECT osm.geometry, osm.osm_id, osm.name, COALESCE(NULLIF(osm.name_en, ''), ne.name) AS name_en, ne.scalerank, ne.labelrank + FROM ne_10m_admin_0_countries AS ne, osm_country_point AS osm + WHERE + -- We only match whether the point is within the Natural Earth polygon + -- because name matching is to difficult since OSM does not contain good + -- enough coverage of ISO codesy + ST_Within(osm.geometry, ne.geometry) + -- We leave out tiny countries + AND ne.scalerank <= 1 + ) + UPDATE osm_country_point AS osm + -- Normalize both scalerank and labelrank into a ranking system from 1 to 6 + -- where the ranks are still distributed uniform enough across all countries + SET "rank" = LEAST(6, CEILING((scalerank + labelrank)/2.0)) + FROM important_country_point AS ne + WHERE osm.osm_id = ne.osm_id; --- TODO: This shouldn't be necessary? The rank function makes something wrong... -UPDATE osm_country_point AS osm -SET "rank" = 1 -WHERE "rank" = 0; + UPDATE osm_country_point AS osm + SET "rank" = 6 + WHERE "rank" IS NULL; -ALTER TABLE osm_country_point ADD CONSTRAINT osm_country_point_rank_constraint CHECK("rank" BETWEEN 1 AND 6); + -- TODO: This shouldn't be necessary? The rank function makes something wrong... + UPDATE osm_country_point AS osm + SET "rank" = 1 + WHERE "rank" = 0; + +END; +$$ LANGUAGE plpgsql; + +SELECT update_osm_country_point(); + +-- ALTER TABLE osm_country_point ADD CONSTRAINT osm_country_point_rank_constraint CHECK("rank" BETWEEN 1 AND 6); CREATE INDEX IF NOT EXISTS osm_country_point_rank_idx ON osm_country_point("rank"); + +-- Handle updates + +CREATE SCHEMA place_country; + +CREATE TABLE IF NOT EXISTS place_country.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION place_country.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO place_country.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION place_country.refresh() RETURNS trigger AS + $BODY$ + BEGIN + RAISE LOG 'Refresh place_country rank'; + SELECT update_osm_country_point(); + DELETE FROM place_country.updates; + RETURN null; + END; + $BODY$ +language plpgsql; + +CREATE TRIGGER trigger_flag + AFTER INSERT OR UPDATE OR DELETE ON osm_country_point + FOR EACH STATEMENT + EXECUTE PROCEDURE place_country.flag(); + +CREATE CONSTRAINT TRIGGER trigger_refresh + AFTER INSERT ON place_country.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE place_country.refresh(); diff --git a/layers/place/merge_state_rank.sql b/layers/place/merge_state_rank.sql index 54e6803..d204a21 100644 --- a/layers/place/merge_state_rank.sql +++ b/layers/place/merge_state_rank.sql @@ -3,28 +3,70 @@ ALTER TABLE osm_state_point DROP CONSTRAINT IF EXISTS osm_state_point_rank_const -- etldoc: ne_10m_admin_1_states_provinces_shp -> osm_state_point -- etldoc: osm_state_point -> osm_state_point -WITH important_state_point AS ( - SELECT osm.geometry, osm.osm_id, osm.name, COALESCE(NULLIF(osm.name_en, ''), ne.name) AS name_en, ne.scalerank, ne.labelrank, ne.datarank - FROM ne_10m_admin_1_states_provinces_shp AS ne, osm_state_point AS osm - WHERE - -- We only match whether the point is within the Natural Earth polygon - -- because name matching is difficult - ST_Within(osm.geometry, ne.geometry) - -- We leave out leess important states - AND ne.scalerank <= 3 AND ne.labelrank <= 2 -) -UPDATE osm_state_point AS osm --- Normalize both scalerank and labelrank into a ranking system from 1 to 6. -SET "rank" = LEAST(6, CEILING((scalerank + labelrank + datarank)/3.0)) -FROM important_state_point AS ne -WHERE osm.osm_id = ne.osm_id; +CREATE FUNCTION update_osm_state_point() RETURNS VOID AS $$ +BEGIN --- TODO: This shouldn't be necessary? The rank function makes something wrong... -UPDATE osm_state_point AS osm -SET "rank" = 1 -WHERE "rank" = 0; + WITH important_state_point AS ( + SELECT osm.geometry, osm.osm_id, osm.name, COALESCE(NULLIF(osm.name_en, ''), ne.name) AS name_en, ne.scalerank, ne.labelrank, ne.datarank + FROM ne_10m_admin_1_states_provinces_shp AS ne, osm_state_point AS osm + WHERE + -- We only match whether the point is within the Natural Earth polygon + -- because name matching is difficult + ST_Within(osm.geometry, ne.geometry) + -- We leave out leess important states + AND ne.scalerank <= 3 AND ne.labelrank <= 2 + ) + UPDATE osm_state_point AS osm + -- Normalize both scalerank and labelrank into a ranking system from 1 to 6. + SET "rank" = LEAST(6, CEILING((scalerank + labelrank + datarank)/3.0)) + FROM important_state_point AS ne + WHERE osm.osm_id = ne.osm_id; -DELETE FROM osm_state_point WHERE "rank" IS NULL; + -- TODO: This shouldn't be necessary? The rank function makes something wrong... + UPDATE osm_state_point AS osm + SET "rank" = 1 + WHERE "rank" = 0; -ALTER TABLE osm_state_point ADD CONSTRAINT osm_state_point_rank_constraint CHECK("rank" BETWEEN 1 AND 6); + DELETE FROM osm_state_point WHERE "rank" IS NULL; + +END; +$$ LANGUAGE plpgsql; + +SELECT update_osm_state_point(); + +-- ALTER TABLE osm_state_point ADD CONSTRAINT osm_state_point_rank_constraint CHECK("rank" BETWEEN 1 AND 6); CREATE INDEX IF NOT EXISTS osm_state_point_rank_idx ON osm_state_point("rank"); + +-- Handle updates + +CREATE SCHEMA place_state; + +CREATE TABLE IF NOT EXISTS place_state.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION place_state.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO place_state.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION place_state.refresh() RETURNS trigger AS + $BODY$ + BEGIN + RAISE LOG 'Refresh place_state rank'; + SELECT update_osm_state_point(); + DELETE FROM place_state.updates; + RETURN null; + END; + $BODY$ +language plpgsql; + +CREATE TRIGGER trigger_flag + AFTER INSERT OR UPDATE OR DELETE ON osm_state_point + FOR EACH STATEMENT + EXECUTE PROCEDURE place_state.flag(); + +CREATE CONSTRAINT TRIGGER trigger_refresh + AFTER INSERT ON place_state.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE place_state.refresh(); From 9437a1b8d6635c7ac9d53452e74cd969977291e7 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Thu, 5 Jan 2017 12:47:10 -0500 Subject: [PATCH 14/54] Make the sql rerunnable without error --- layers/place/island_polygon_update.sql | 7 +++++-- layers/place/merge_city_rank.sql | 7 +++++-- layers/place/merge_country_rank.sql | 7 +++++-- layers/place/merge_state_rank.sql | 7 +++++-- layers/poi/poi_polygon_update.sql | 7 +++++-- layers/transportation_name/merge_highways.sql | 12 ++++++++++-- layers/water_name/water_lakeline.sql | 7 ++++++- layers/water_name/water_point.sql | 9 +++++++-- layers/waterway/merge_waterway.sql | 10 +++++++++- 9 files changed, 57 insertions(+), 16 deletions(-) diff --git a/layers/place/island_polygon_update.sql b/layers/place/island_polygon_update.sql index 57cca41..04c5432 100644 --- a/layers/place/island_polygon_update.sql +++ b/layers/place/island_polygon_update.sql @@ -1,5 +1,5 @@ -- etldoc: osm_island_polygon -> osm_island_polygon -CREATE FUNCTION convert_island_polygon_point() RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION convert_island_polygon_point() RETURNS VOID AS $$ BEGIN UPDATE osm_island_polygon SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; ANALYZE osm_island_polygon; @@ -10,7 +10,7 @@ SELECT convert_island_polygon_point(); -- Handle updates -CREATE SCHEMA place_island; +CREATE SCHEMA IF NOT EXISTS place_island; CREATE TABLE IF NOT EXISTS place_island.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION place_island.flag() RETURNS trigger AS $$ @@ -31,6 +31,9 @@ CREATE OR REPLACE FUNCTION place_island.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_island_polygon; +DROP TRIGGER IF EXISTS trigger_refresh ON place_island.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_island_polygon FOR EACH STATEMENT diff --git a/layers/place/merge_city_rank.sql b/layers/place/merge_city_rank.sql index 78587ab..e7d9276 100644 --- a/layers/place/merge_city_rank.sql +++ b/layers/place/merge_city_rank.sql @@ -2,7 +2,7 @@ CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE FUNCTION update_osm_city_point() RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION update_osm_city_point() RETURNS VOID AS $$ BEGIN -- Clear OSM key:rank ( https://github.com/openmaptiles/openmaptiles/issues/108 ) @@ -48,7 +48,7 @@ CREATE INDEX IF NOT EXISTS osm_city_point_rank_idx ON osm_city_point("rank"); -- Handle updates -CREATE SCHEMA place_city; +CREATE SCHEMA IF NOT EXISTS place_city; CREATE TABLE IF NOT EXISTS place_city.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION place_city.flag() RETURNS trigger AS $$ @@ -69,6 +69,9 @@ CREATE OR REPLACE FUNCTION place_city.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_city_point; +DROP TRIGGER IF EXISTS trigger_refresh ON place_city.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_city_point FOR EACH STATEMENT diff --git a/layers/place/merge_country_rank.sql b/layers/place/merge_country_rank.sql index 95a164a..fbf1d95 100644 --- a/layers/place/merge_country_rank.sql +++ b/layers/place/merge_country_rank.sql @@ -3,7 +3,7 @@ ALTER TABLE osm_country_point DROP CONSTRAINT IF EXISTS osm_country_point_rank_c -- etldoc: ne_10m_admin_0_countries -> osm_country_point -- etldoc: osm_country_point -> osm_country_point -CREATE FUNCTION update_osm_country_point() RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION update_osm_country_point() RETURNS VOID AS $$ BEGIN WITH important_country_point AS ( @@ -43,7 +43,7 @@ CREATE INDEX IF NOT EXISTS osm_country_point_rank_idx ON osm_country_point("rank -- Handle updates -CREATE SCHEMA place_country; +CREATE SCHEMA IF NOT EXISTS place_country; CREATE TABLE IF NOT EXISTS place_country.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION place_country.flag() RETURNS trigger AS $$ @@ -64,6 +64,9 @@ CREATE OR REPLACE FUNCTION place_country.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_country_point; +DROP TRIGGER IF EXISTS trigger_refresh ON place_country.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_country_point FOR EACH STATEMENT diff --git a/layers/place/merge_state_rank.sql b/layers/place/merge_state_rank.sql index d204a21..90c9616 100644 --- a/layers/place/merge_state_rank.sql +++ b/layers/place/merge_state_rank.sql @@ -3,7 +3,7 @@ ALTER TABLE osm_state_point DROP CONSTRAINT IF EXISTS osm_state_point_rank_const -- etldoc: ne_10m_admin_1_states_provinces_shp -> osm_state_point -- etldoc: osm_state_point -> osm_state_point -CREATE FUNCTION update_osm_state_point() RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION update_osm_state_point() RETURNS VOID AS $$ BEGIN WITH important_state_point AS ( @@ -39,7 +39,7 @@ CREATE INDEX IF NOT EXISTS osm_state_point_rank_idx ON osm_state_point("rank"); -- Handle updates -CREATE SCHEMA place_state; +CREATE SCHEMA IF NOT EXISTS place_state; CREATE TABLE IF NOT EXISTS place_state.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION place_state.flag() RETURNS trigger AS $$ @@ -60,6 +60,9 @@ CREATE OR REPLACE FUNCTION place_state.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_state_point; +DROP TRIGGER IF EXISTS trigger_refresh ON place_state.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_state_point FOR EACH STATEMENT diff --git a/layers/poi/poi_polygon_update.sql b/layers/poi/poi_polygon_update.sql index 51259e5..dcd4b3b 100644 --- a/layers/poi/poi_polygon_update.sql +++ b/layers/poi/poi_polygon_update.sql @@ -1,6 +1,6 @@ -- etldoc: osm_poi_polygon -> osm_poi_polygon -CREATE FUNCTION convert_poi_point() RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION convert_poi_point() RETURNS VOID AS $$ BEGIN UPDATE osm_poi_polygon SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; ANALYZE osm_poi_polygon; @@ -11,7 +11,7 @@ SELECT convert_poi_point(); -- Handle updates -CREATE SCHEMA poi; +CREATE SCHEMA IF NOT EXISTS poi; CREATE TABLE IF NOT EXISTS poi.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION poi.flag() RETURNS trigger AS $$ @@ -32,6 +32,9 @@ CREATE OR REPLACE FUNCTION poi.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_poi_polygon; +DROP TRIGGER IF EXISTS trigger_refresh ON poi.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_poi_polygon FOR EACH STATEMENT diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 7a81a87..0648acb 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -4,6 +4,11 @@ -- to allow for nice label rendering -- Because this works well for roads that do not have relations as well +DROP MATERIALIZED VIEW IF EXISTS osm_transportation_name_linestring CASCADE; +DROP MATERIALIZED VIEW IF EXISTS osm_transportation_name_linestring_gen1 CASCADE; +DROP MATERIALIZED VIEW IF EXISTS osm_transportation_name_linestring_gen2 CASCADE; +DROP MATERIALIZED VIEW IF EXISTS osm_transportation_name_linestring_gen3 CASCADE; + -- etldoc: osm_highway_linestring -> osm_transportation_name_linestring CREATE MATERIALIZED VIEW osm_transportation_name_linestring AS ( SELECT @@ -58,7 +63,7 @@ CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx -- Handle updates -CREATE SCHEMA transportation_name; +CREATE SCHEMA IF NOT EXISTS transportation_name; CREATE TABLE IF NOT EXISTS transportation_name.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION transportation_name.flag() RETURNS trigger AS $$ @@ -82,6 +87,9 @@ CREATE OR REPLACE FUNCTION transportation_name.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_highway_linestring; +DROP TRIGGER IF EXISTS trigger_refresh ON transportation_name.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_highway_linestring FOR EACH STATEMENT @@ -91,4 +99,4 @@ CREATE CONSTRAINT TRIGGER trigger_refresh AFTER INSERT ON transportation_name.updates INITIALLY DEFERRED FOR EACH ROW - EXECUTE PROCEDURE transportation_name.refresh(); \ No newline at end of file + EXECUTE PROCEDURE transportation_name.refresh(); diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index fc776db..b918f4b 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -1,6 +1,8 @@ -- etldoc: osm_water_polygon -> osm_water_lakeline -- etldoc: lake_centerline -> osm_water_lakeline +DROP MATERIALIZED VIEW IF EXISTS osm_water_lakeline CASCADE; + CREATE MATERIALIZED VIEW osm_water_lakeline AS ( SELECT wp.osm_id, ll.wkb_geometry AS geometry, @@ -13,7 +15,7 @@ CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline -- Handle updates -CREATE SCHEMA water_lakeline; +CREATE SCHEMA IF NOT EXISTS water_lakeline; CREATE TABLE IF NOT EXISTS water_lakeline.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION water_lakeline.flag() RETURNS trigger AS $$ @@ -34,6 +36,9 @@ CREATE OR REPLACE FUNCTION water_lakeline.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; +DROP TRIGGER IF EXISTS trigger_refresh ON water_lakeline.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index f938b76..2bf0de8 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -1,6 +1,8 @@ -- etldoc: osm_water_polygon -> osm_water_point -- etldoc: lake_centerline -> osm_water_point +DROP MATERIALIZED VIEW IF EXISTS osm_water_point CASCADE; + CREATE MATERIALIZED VIEW osm_water_point AS ( SELECT wp.osm_id, topoint(wp.geometry) AS geometry, @@ -10,10 +12,10 @@ CREATE MATERIALIZED VIEW osm_water_point AS ( WHERE ll.osm_id IS NULL AND wp.name <> '' ) WITH NO DATA; CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry); - + -- Handle updates -CREATE SCHEMA water_name; +CREATE SCHEMA IF NOT EXISTS water_name; CREATE TABLE IF NOT EXISTS water_name.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION water_name.flag() RETURNS trigger AS $$ @@ -34,6 +36,9 @@ CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; +DROP TRIGGER IF EXISTS trigger_refresh ON water_name.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index ec2628f..614cfae 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -4,6 +4,11 @@ -- and also makes it possible to filter out too short rivers -- etldoc: osm_waterway_linestring -> osm_important_waterway_linestring +DROP MATERIALIZED VIEW IF EXISTS osm_important_waterway_linestring CASCADE; +DROP MATERIALIZED VIEW IF EXISTS osm_important_waterway_linestring_gen1 CASCADE; +DROP MATERIALIZED VIEW IF EXISTS osm_important_waterway_linestring_gen2 CASCADE; +DROP MATERIALIZED VIEW IF EXISTS osm_important_waterway_linestring_gen3 CASCADE; + CREATE MATERIALIZED VIEW osm_important_waterway_linestring AS ( SELECT (ST_Dump(geometry)).geom AS geometry, @@ -45,7 +50,7 @@ CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen3_geometry_idx O -- Handle updates -CREATE SCHEMA waterway; +CREATE SCHEMA IF NOT EXISTS waterway; CREATE TABLE IF NOT EXISTS waterway.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION waterway.flag() RETURNS trigger AS $$ @@ -69,6 +74,9 @@ CREATE OR REPLACE FUNCTION waterway.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_waterway_linestring; +DROP TRIGGER IF EXISTS trigger_refresh ON waterway.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_waterway_linestring FOR EACH STATEMENT From b58d80ce4473e799d9d5b997ec685c7f87ed04e0 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Thu, 5 Jan 2017 14:39:04 -0500 Subject: [PATCH 15/54] Make housenumber rerunnable --- layers/housenumber/housenumber_centroid.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/layers/housenumber/housenumber_centroid.sql b/layers/housenumber/housenumber_centroid.sql index 1d1cf83..24c05ae 100644 --- a/layers/housenumber/housenumber_centroid.sql +++ b/layers/housenumber/housenumber_centroid.sql @@ -1,6 +1,6 @@ -- etldoc: osm_housenumber_point -> osm_housenumber_point -CREATE FUNCTION convert_housenumber_point() RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION convert_housenumber_point() RETURNS VOID AS $$ BEGIN UPDATE osm_housenumber_point SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; END; @@ -10,7 +10,7 @@ SELECT convert_housenumber_point(); -- Handle updates -CREATE SCHEMA housenumber; +CREATE SCHEMA IF NOT EXISTS housenumber; CREATE TABLE IF NOT EXISTS housenumber.updates(id serial primary key, t text, unique (t)); CREATE OR REPLACE FUNCTION housenumber.flag() RETURNS trigger AS $$ @@ -31,6 +31,9 @@ CREATE OR REPLACE FUNCTION housenumber.refresh() RETURNS trigger AS $BODY$ language plpgsql; +DROP TRIGGER IF EXISTS trigger_flag ON osm_housenumber_point; +DROP TRIGGER IF EXISTS trigger_refresh ON housenumber.updates; + CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_housenumber_point FOR EACH STATEMENT From cda3af921596fd337da8b258d5e151f8f1f40923 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Fri, 6 Jan 2017 20:33:43 -0500 Subject: [PATCH 16/54] Change SELECT to PERFORM in function and drop triggers first --- layers/housenumber/housenumber_centroid.sql | 7 +++---- layers/place/island_polygon_update.sql | 8 ++++---- layers/place/merge_city_rank.sql | 8 +++----- layers/place/merge_country_rank.sql | 8 ++++---- layers/place/merge_state_rank.sql | 8 ++++---- layers/poi/poi_polygon_update.sql | 8 ++++---- layers/transportation_name/merge_highways.sql | 5 ++--- layers/water_name/water_lakeline.sql | 5 ++--- layers/water_name/water_point.sql | 5 ++--- layers/waterway/merge_waterway.sql | 5 ++--- 10 files changed, 30 insertions(+), 37 deletions(-) diff --git a/layers/housenumber/housenumber_centroid.sql b/layers/housenumber/housenumber_centroid.sql index 24c05ae..29c687e 100644 --- a/layers/housenumber/housenumber_centroid.sql +++ b/layers/housenumber/housenumber_centroid.sql @@ -1,3 +1,5 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_housenumber_point; +DROP TRIGGER IF EXISTS trigger_refresh ON housenumber.updates; -- etldoc: osm_housenumber_point -> osm_housenumber_point CREATE OR REPLACE FUNCTION convert_housenumber_point() RETURNS VOID AS $$ @@ -24,16 +26,13 @@ CREATE OR REPLACE FUNCTION housenumber.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh housenumber'; - SELECT convert_housenumber_point(); + PERFORM convert_housenumber_point(); DELETE FROM housenumber.updates; RETURN null; END; $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_housenumber_point; -DROP TRIGGER IF EXISTS trigger_refresh ON housenumber.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_housenumber_point FOR EACH STATEMENT diff --git a/layers/place/island_polygon_update.sql b/layers/place/island_polygon_update.sql index 04c5432..ef937a5 100644 --- a/layers/place/island_polygon_update.sql +++ b/layers/place/island_polygon_update.sql @@ -1,3 +1,6 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_island_polygon; +DROP TRIGGER IF EXISTS trigger_refresh ON place_island.updates; + -- etldoc: osm_island_polygon -> osm_island_polygon CREATE OR REPLACE FUNCTION convert_island_polygon_point() RETURNS VOID AS $$ BEGIN @@ -24,16 +27,13 @@ CREATE OR REPLACE FUNCTION place_island.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh place_island'; - SELECT convert_island_polygon_point(); + PERFORM convert_island_polygon_point(); DELETE FROM place_island.updates; RETURN null; END; $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_island_polygon; -DROP TRIGGER IF EXISTS trigger_refresh ON place_island.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_island_polygon FOR EACH STATEMENT diff --git a/layers/place/merge_city_rank.sql b/layers/place/merge_city_rank.sql index e7d9276..fa91994 100644 --- a/layers/place/merge_city_rank.sql +++ b/layers/place/merge_city_rank.sql @@ -1,7 +1,8 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_city_point; +DROP TRIGGER IF EXISTS trigger_refresh ON place_city.updates; CREATE EXTENSION IF NOT EXISTS unaccent; - CREATE OR REPLACE FUNCTION update_osm_city_point() RETURNS VOID AS $$ BEGIN @@ -62,16 +63,13 @@ CREATE OR REPLACE FUNCTION place_city.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh place_city rank'; - SELECT update_osm_city_point(); + PERFORM update_osm_city_point(); DELETE FROM place_city.updates; RETURN null; END; $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_city_point; -DROP TRIGGER IF EXISTS trigger_refresh ON place_city.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_city_point FOR EACH STATEMENT diff --git a/layers/place/merge_country_rank.sql b/layers/place/merge_country_rank.sql index fbf1d95..afc14ea 100644 --- a/layers/place/merge_country_rank.sql +++ b/layers/place/merge_country_rank.sql @@ -1,3 +1,6 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_country_point; +DROP TRIGGER IF EXISTS trigger_refresh ON place_country.updates; + ALTER TABLE osm_country_point DROP CONSTRAINT IF EXISTS osm_country_point_rank_constraint; -- etldoc: ne_10m_admin_0_countries -> osm_country_point @@ -57,16 +60,13 @@ CREATE OR REPLACE FUNCTION place_country.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh place_country rank'; - SELECT update_osm_country_point(); + PERFORM update_osm_country_point(); DELETE FROM place_country.updates; RETURN null; END; $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_country_point; -DROP TRIGGER IF EXISTS trigger_refresh ON place_country.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_country_point FOR EACH STATEMENT diff --git a/layers/place/merge_state_rank.sql b/layers/place/merge_state_rank.sql index 90c9616..fcf3279 100644 --- a/layers/place/merge_state_rank.sql +++ b/layers/place/merge_state_rank.sql @@ -1,3 +1,6 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_state_point; +DROP TRIGGER IF EXISTS trigger_refresh ON place_state.updates; + ALTER TABLE osm_state_point DROP CONSTRAINT IF EXISTS osm_state_point_rank_constraint; -- etldoc: ne_10m_admin_1_states_provinces_shp -> osm_state_point @@ -53,16 +56,13 @@ CREATE OR REPLACE FUNCTION place_state.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh place_state rank'; - SELECT update_osm_state_point(); + PERFORM update_osm_state_point(); DELETE FROM place_state.updates; RETURN null; END; $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_state_point; -DROP TRIGGER IF EXISTS trigger_refresh ON place_state.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_state_point FOR EACH STATEMENT diff --git a/layers/poi/poi_polygon_update.sql b/layers/poi/poi_polygon_update.sql index dcd4b3b..22484ac 100644 --- a/layers/poi/poi_polygon_update.sql +++ b/layers/poi/poi_polygon_update.sql @@ -1,3 +1,6 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_poi_polygon; +DROP TRIGGER IF EXISTS trigger_refresh ON poi.updates; + -- etldoc: osm_poi_polygon -> osm_poi_polygon CREATE OR REPLACE FUNCTION convert_poi_point() RETURNS VOID AS $$ @@ -25,16 +28,13 @@ CREATE OR REPLACE FUNCTION poi.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh poi'; - SELECT convert_poi_point(); + PERFORM convert_poi_point(); DELETE FROM poi.updates; RETURN null; END; $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_poi_polygon; -DROP TRIGGER IF EXISTS trigger_refresh ON poi.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_poi_polygon FOR EACH STATEMENT diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 0648acb..025b274 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -1,3 +1,5 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_highway_linestring; +DROP TRIGGER IF EXISTS trigger_refresh ON transportation_name.updates; -- Instead of using relations to find out the road names we -- stitch together the touching ways with the same name @@ -87,9 +89,6 @@ CREATE OR REPLACE FUNCTION transportation_name.refresh() RETURNS trigger AS $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_highway_linestring; -DROP TRIGGER IF EXISTS trigger_refresh ON transportation_name.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_highway_linestring FOR EACH STATEMENT diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index b918f4b..3570cc2 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -1,3 +1,5 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; +DROP TRIGGER IF EXISTS trigger_refresh ON water_lakeline.updates; -- etldoc: osm_water_polygon -> osm_water_lakeline -- etldoc: lake_centerline -> osm_water_lakeline @@ -36,9 +38,6 @@ CREATE OR REPLACE FUNCTION water_lakeline.refresh() RETURNS trigger AS $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; -DROP TRIGGER IF EXISTS trigger_refresh ON water_lakeline.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 2bf0de8..7c49547 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -1,3 +1,5 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; +DROP TRIGGER IF EXISTS trigger_refresh ON water_name.updates; -- etldoc: osm_water_polygon -> osm_water_point -- etldoc: lake_centerline -> osm_water_point @@ -36,9 +38,6 @@ CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; -DROP TRIGGER IF EXISTS trigger_refresh ON water_name.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index 614cfae..e3bcd89 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -1,3 +1,5 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_waterway_linestring; +DROP TRIGGER IF EXISTS trigger_refresh ON waterway.updates; -- We merge the waterways by name like the highways -- This helps to drop not important rivers (since they do not have a name) @@ -74,9 +76,6 @@ CREATE OR REPLACE FUNCTION waterway.refresh() RETURNS trigger AS $BODY$ language plpgsql; -DROP TRIGGER IF EXISTS trigger_flag ON osm_waterway_linestring; -DROP TRIGGER IF EXISTS trigger_refresh ON waterway.updates; - CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_waterway_linestring FOR EACH STATEMENT From 88183e614302f45a9a63a90286329d7f46071e4c Mon Sep 17 00:00:00 2001 From: OpenMapTiles Date: Sat, 7 Jan 2017 15:10:07 +0100 Subject: [PATCH 17/54] Create LICENSE.md --- LICENSE.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..245afac --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,60 @@ +Copyright (c) 2016, KlokanTech.com & OpenMapTiles contributors. +All rights reserved. + +The vector tile schema has been developed by Klokan Technologies GmbH and +was initially modelled after the cartography of the CARTO's Positron basemap +with permission from CartoDB Inc. +The vector tile schema has been refined and improved in cooperation with +the Wikimedia Foundation and is heavily influenced by years of +Paul Norman's experience of creating maps from OpenStreetMap data. + +# Code license: BSD 3-Clause License + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Design license: CC-BY 4.0 + +The cartography and visual design features of the map tile schema (also known as +the "look and feel" of the map) are licensed under the Creative Commons +Attribution 4.0 license. +To view a copy of the license, visit http://creativecommons.org/licenses/by/4.0/. + +Products or services using maps derived from OpenMapTiles schema need to visibly +credit "OpenMapTiles.org" or reference "OpenMapTiles" with a link to +http://openmaptiles.org/. + +For a browsable electronic map based on OpenMapTiles and OpenStreetMap data, the +credit should appear in the corner of the map. For example: + +[© OpenMapTiles](http://openmaptiles.org/) [© OpenStreetMap contributors](http://www.openstreetmap.org/copyright) + +For printed and static maps a similar attribution should be made in a textual +description near the image, in the same fashion as if you cite a photograph. + +Exceptions to OpenMapTiles attribution requirement can be in a written form granted +by Klokan Technologies GmbH (info@klokantech.com). +The project contributors grant Klokan Technologies GmbH the license to give such +exceptions on a commercial basis. From f8d1c1fb88fc3b0cfe55c430e9bdad06ee4fbffa Mon Sep 17 00:00:00 2001 From: OpenMapTiles Date: Sat, 7 Jan 2017 15:18:56 +0100 Subject: [PATCH 18/54] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7958aee..e459eb8 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Each time you modify layer SQL code run `make` and `docker-compose run import-sq make clean && make && docker-compose run import-sql ``` -Now you are ready to **generate the vector tiles** using a single process (for a full blown distributed workflow of rendering tiles check out [openmaptiles/distributed](https://github.com/openmaptiles/distributed)). Using environment variables +Now you are ready to **generate the vector tiles**. Using environment variables you can limit the bounding box and zoom levels of what you want to generate (`docker-compose.yml`). ``` @@ -125,6 +125,6 @@ docker-compose run generate-vectortiles ## License -*LICENSE HAS NOT BEEN YET DECIDED* +All code in this repository is under the [BSD license](./LICENSE.md) and the cartography decisions encoded in the schema and SQL are licensed under [CC-BY](./LICENSE.md). -All code in this repository is under the [MIT license](./LICENSE) and the cartography decisions encoded in the schema and SQL is licensed under [CC0](https://creativecommons.org/publicdomain/zero/1.0/). +Products or services using maps derived from OpenMapTiles schema need to visibly credit "OpenMapTiles.org" or reference "OpenMapTiles" with a link to http://openmaptiles.org/. Exceptions to attribution requirement can be granted on request. From a6353e2360193d16cbdb563fd9a6e3e28e77382a Mon Sep 17 00:00:00 2001 From: OpenMapTiles Date: Sat, 7 Jan 2017 17:31:33 +0100 Subject: [PATCH 19/54] Update README.md --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e459eb8..f484f39 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,14 @@ docker-compose run generate-vectortiles ## License -All code in this repository is under the [BSD license](./LICENSE.md) and the cartography decisions encoded in the schema and SQL are licensed under [CC-BY](./LICENSE.md). +All code in this repository is under the [BSD license](./LICENSE.md) and the cartography decisions encoded in the schema and SQL are licensed under [CC-BY](./LICENSE.md). Products or services using maps derived from OpenMapTiles schema need to visibly credit "OpenMapTiles.org" or reference "OpenMapTiles" with a link to http://openmaptiles.org/. Exceptions to attribution requirement can be granted on request. + +For a browsable electronic map based on OpenMapTiles and OpenStreetMap data, the +credit should appear in the corner of the map. For example: + +[© OpenMapTiles](http://openmaptiles.org/) [© OpenStreetMap contributors](http://www.openstreetmap.org/copyright) + +For printed and static maps a similar attribution should be made in a textual +description near the image, in the same fashion as if you cite a photograph. From 4c1ea3e4b1dd8edf4a558e60e38c2018425e06d7 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Sat, 7 Jan 2017 22:26:09 -0500 Subject: [PATCH 20/54] Refresh concurrently and create materialized views with data. --- layers/transportation_name/merge_highways.sql | 16 ++++++++-------- layers/water_name/water_lakeline.sql | 4 ++-- layers/water_name/water_point.sql | 4 ++-- layers/waterway/merge_waterway.sql | 16 ++++++++-------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 025b274..d3ebedf 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -36,7 +36,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring AS ( WHERE (name <> '' OR ref <> '') AND NULLIF(highway, '') IS NOT NULL GROUP BY name, highway, ref ) AS highway_union -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_geometry_idx ON osm_transportation_name_linestring USING gist(geometry); -- etldoc: osm_transportation_name_linestring -> osm_transportation_name_linestring_gen1 @@ -44,7 +44,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 50) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring WHERE highway IN ('motorway','trunk') AND ST_Length(geometry) > 8000 -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen1_geometry_idx ON osm_transportation_name_linestring_gen1 USING gist(geometry); -- etldoc: osm_transportation_name_linestring_gen1 -> osm_transportation_name_linestring_gen2 @@ -52,7 +52,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring_gen1 WHERE highway IN ('motorway','trunk') AND ST_Length(geometry) > 14000 -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen2_geometry_idx ON osm_transportation_name_linestring_gen2 USING gist(geometry); -- etldoc: osm_transportation_name_linestring_gen2 -> osm_transportation_name_linestring_gen3 @@ -60,7 +60,7 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 120) AS geometry, osm_id, member_osm_ids, name, ref, highway, z_order FROM osm_transportation_name_linestring_gen2 WHERE highway = 'motorway' AND ST_Length(geometry) > 20000 -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_transportation_name_linestring_gen3_geometry_idx ON osm_transportation_name_linestring_gen3 USING gist(geometry); -- Handle updates @@ -79,10 +79,10 @@ CREATE OR REPLACE FUNCTION transportation_name.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh transportation_name'; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen1; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen2; - REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen3; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen1; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen2; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen3; DELETE FROM transportation_name.updates; RETURN null; END; diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index 3570cc2..dea4251 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -12,7 +12,7 @@ CREATE MATERIALIZED VIEW osm_water_lakeline AS ( FROM osm_water_polygon AS wp INNER JOIN lake_centerline ll ON wp.osm_id = ll.osm_id WHERE wp.name <> '' -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_water_lakeline_geometry_idx ON osm_water_lakeline USING gist(geometry); -- Handle updates @@ -31,7 +31,7 @@ CREATE OR REPLACE FUNCTION water_lakeline.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh water_lakeline'; - REFRESH MATERIALIZED VIEW osm_water_lakeline; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_water_lakeline; DELETE FROM water_lakeline.updates; RETURN null; END; diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 7c49547..68a7657 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -12,7 +12,7 @@ CREATE MATERIALIZED VIEW osm_water_point AS ( FROM osm_water_polygon AS wp LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id WHERE ll.osm_id IS NULL AND wp.name <> '' -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING gist (geometry); -- Handle updates @@ -31,7 +31,7 @@ CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh water_name'; - REFRESH MATERIALIZED VIEW osm_water_point; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_water_point; DELETE FROM water_name.updates; RETURN null; END; diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index e3bcd89..03f248f 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -23,7 +23,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring AS ( WHERE name <> '' AND waterway = 'river' GROUP BY name ) AS waterway_union -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_geometry_idx ON osm_important_waterway_linestring USING gist(geometry); -- etldoc: osm_important_waterway_linestring -> osm_important_waterway_linestring_gen1 @@ -31,7 +31,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen1 AS ( SELECT ST_Simplify(geometry, 60) AS geometry, name FROM osm_important_waterway_linestring WHERE ST_Length(geometry) > 1000 -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen1_geometry_idx ON osm_important_waterway_linestring_gen1 USING gist(geometry); -- etldoc: osm_important_waterway_linestring_gen1 -> osm_important_waterway_linestring_gen2 @@ -39,7 +39,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen2 AS ( SELECT ST_Simplify(geometry, 100) AS geometry, name FROM osm_important_waterway_linestring_gen1 WHERE ST_Length(geometry) > 4000 -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen2_geometry_idx ON osm_important_waterway_linestring_gen2 USING gist(geometry); -- etldoc: osm_important_waterway_linestring_gen2 -> osm_important_waterway_linestring_gen3 @@ -47,7 +47,7 @@ CREATE MATERIALIZED VIEW osm_important_waterway_linestring_gen3 AS ( SELECT ST_Simplify(geometry, 200) AS geometry, name FROM osm_important_waterway_linestring_gen2 WHERE ST_Length(geometry) > 8000 -) WITH NO DATA; +); CREATE INDEX IF NOT EXISTS osm_important_waterway_linestring_gen3_geometry_idx ON osm_important_waterway_linestring_gen3 USING gist(geometry); -- Handle updates @@ -66,10 +66,10 @@ CREATE OR REPLACE FUNCTION waterway.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh transportation_name'; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen1; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen2; - REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen3; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen1; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen2; + REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen3; DELETE FROM waterway.updates; RETURN null; END; From 5fa388d52aef2d1cf68d418b1ee12abe5cbbb097 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 8 Jan 2017 13:22:19 +0100 Subject: [PATCH 21/54] add metadata to ./data/tiles.mbtiles --- quickstart.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/quickstart.sh b/quickstart.sh index 1330d02..6b445d6 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -241,6 +241,12 @@ echo " : like : Mapnik LOG> ... is deprecated and will be removed in M docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml run --rm generate-vectortiles +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Add special metadata to mbtiles! " +docker-compose run --rm openmaptiles-tools generate-metadata ./data/tiles.mbtiles +docker-compose run --rm openmaptiles-tools chmod 666 ./data/tiles.mbtiles + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Stop PostgreSQL service ( but we keep PostgreSQL data volume for debugging )" From 524ff207b8535704c82a775953f70505966b53f2 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 8 Jan 2017 13:31:24 +0100 Subject: [PATCH 22/54] add: make start-tileserver --- Makefile | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 937060b..9ca89c7 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ help: @echo " " @echo "Hints for designers:" @echo " ....TODO.... # start Maputnik " - @echo " ....TODO.... # start Tileserver-gl-light" + @echo " make start-tileserver # start klokantech/tileserver-gl [ see localhost:8080 ] " @echo " make start-mapbox-studio # start Mapbox Studio" @echo " " @echo "Hints for developers:" @@ -64,6 +64,7 @@ remove-docker-images: @docker images | grep "" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi @docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + @docker images | grep "klokantech/tileserver-gl" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi docker-unnecessary-clean: @echo "Deleting unnecessary container(s)..." @@ -117,6 +118,26 @@ list: download-geofabrik-list: docker-compose run --rm import-osm ./download-geofabrik-list.sh +start-tileserver: + @echo " " + @echo "***********************************************************" + @echo "* " + @echo "* Download latest klokantech/tileserver-gl docker image " + @echo "* see documentation: https://github.com/klokantech/tileserver-gl" + @echo "* " + @echo "***********************************************************" + @echo " " + docker pull klokantech/tileserver-gl + @echo " " + @echo "***********************************************************" + @echo "* " + @echo "* Start klokantech/tileserver-gl " + @echo "* ----------------------------> check localhost:8080 " + @echo "* " + @echo "***********************************************************" + @echo " " + docker run -it -v $$(pwd)/data:/data -p 8080:80 klokantech/tileserver-gl + start-mapbox-studio: docker-compose up mapbox-studio From aa2b5d22c24002e3be3649075d6f2a8a08fe25c2 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 8 Jan 2017 13:32:38 +0100 Subject: [PATCH 23/54] image version -> "openmaptiles/openmaptiles-tools:0.3" --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1e26f73..ba03389 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: volumes: - ./build:/sql openmaptiles-tools: - image: "openmaptiles/openmaptiles-tools:latest" + image: "openmaptiles/openmaptiles-tools:0.3" env_file: .env links: - postgres From af09175141a9d9d4130bf9840897db720033b745 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 8 Jan 2017 14:08:08 +0100 Subject: [PATCH 24/54] add `make start-tileserver` --- QUICKSTART.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/QUICKSTART.md b/QUICKSTART.md index 36abe18..a0f646a 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -272,14 +272,18 @@ This is generating `.mbtiles` for your area : [ MIN_ZOOM: "0" - MAX_ZOOM: "7" ./quickstart.sh yukon ``` -### Check other commands +### Check tileserver -`make help` +start: +* ` make start-tileserver` +and the generated maps are going to be available in webbrowser on [localhost:8080](http://localhost:8080/). + +This is only a quick preview, because your mbtiles only generated to zoom level 7 ! ### Change MIN_ZOOM and MAX_ZOOM -modify the settings in the `.env` file +modify the settings in the `.env` file, the defaults : * QUICKSTART_MIN_ZOOM=0 * QUICKSTART_MAX_ZOOM=7 @@ -292,3 +296,6 @@ Hints: * Small increments! Never starts with the MAX_ZOOM = 14 * The suggested MAX_ZOOM = 14 - use only with small extracts +### Check other commands + +`make help` From e10e2b9b941acbe2733b9c4c549ab0dfaa563e36 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Sun, 8 Jan 2017 22:41:46 +0100 Subject: [PATCH 25/54] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f484f39..7bf58c6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## OpenMapTiles +## OpenMapTiles [![Build Status](https://travis-ci.org/openmaptiles/openmaptiles.svg?branch=master)](https://travis-ci.org/openmaptiles/openmaptiles) OpenMapTiles is an extensible and open vector tile schema for a OpenStreetMap basemap. It is used to generate vector tiles for [openmaptiles.org](http://openmaptiles.org/) and [openmaptiles.com](http://openmaptiles.com/). From 3279c252832961fb33e936146fb4584096210241 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 9 Jan 2017 13:44:54 +0100 Subject: [PATCH 26/54] fix `make remove-docker-images` --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9ca89c7..997cace 100644 --- a/Makefile +++ b/Makefile @@ -61,10 +61,9 @@ refresh-docker-images: remove-docker-images: @echo "Deleting all openmaptiles related docker image(s)..." @docker-compose down - @docker images | grep "" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi - @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi - @docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi - @docker images | grep "klokantech/tileserver-gl" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f + @docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f + @docker images | grep "klokantech/tileserver-gl" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f docker-unnecessary-clean: @echo "Deleting unnecessary container(s)..." From e52fa9dd609f297c98f91260e1b878483445e059 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Mon, 9 Jan 2017 13:46:47 +0100 Subject: [PATCH 27/54] change version back to "openmaptiles/openmaptiles-tools:latest" --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ba03389..1e26f73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: volumes: - ./build:/sql openmaptiles-tools: - image: "openmaptiles/openmaptiles-tools:0.3" + image: "openmaptiles/openmaptiles-tools:latest" env_file: .env links: - postgres From a9a17cb263ae15a1d6027adbbf70c4db458dbe0e Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Tue, 10 Jan 2017 19:44:31 +0100 Subject: [PATCH 28/54] =?UTF-8?q?add:=20Clean=20up=20(=E2=80=93rm)=20to=20?= =?UTF-8?q?klokantech/tileserver-gl=20docker=20start?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 997cace..97797bc 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ start-tileserver: @echo "* " @echo "***********************************************************" @echo " " - docker run -it -v $$(pwd)/data:/data -p 8080:80 klokantech/tileserver-gl + docker run -it --rm -v $$(pwd)/data:/data -p 8080:80 klokantech/tileserver-gl start-mapbox-studio: docker-compose up mapbox-studio From f6c3b932c7369f8d3ae8defd05b19e91885b0629 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Tue, 10 Jan 2017 14:50:35 -0500 Subject: [PATCH 29/54] Fix raise log for waterway --- layers/waterway/merge_waterway.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index 03f248f..93c8200 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -65,7 +65,7 @@ $$ language plpgsql; CREATE OR REPLACE FUNCTION waterway.refresh() RETURNS trigger AS $BODY$ BEGIN - RAISE LOG 'Refresh transportation_name'; + RAISE LOG 'Refresh waterway'; REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring; REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen1; REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen2; From 1dfb6d76a5e69d0bbaf343bf51c76cac034a4dd4 Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Wed, 11 Jan 2017 09:46:19 +0100 Subject: [PATCH 30/54] start-tileserver: text improvement --- Makefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 97797bc..05a662f 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ refresh-docker-images: remove-docker-images: @echo "Deleting all openmaptiles related docker image(s)..." @docker-compose down - @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f + @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f @docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f @docker images | grep "klokantech/tileserver-gl" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f @@ -118,21 +118,21 @@ download-geofabrik-list: docker-compose run --rm import-osm ./download-geofabrik-list.sh start-tileserver: - @echo " " + @echo " " @echo "***********************************************************" - @echo "* " - @echo "* Download latest klokantech/tileserver-gl docker image " - @echo "* see documentation: https://github.com/klokantech/tileserver-gl" - @echo "* " + @echo "* " + @echo "* Download/refresh klokantech/tileserver-gl docker image" + @echo "* see documentation: https://github.com/klokantech/tileserver-gl" + @echo "* " @echo "***********************************************************" - @echo " " + @echo " " docker pull klokantech/tileserver-gl - @echo " " + @echo " " @echo "***********************************************************" - @echo "* " + @echo "* " @echo "* Start klokantech/tileserver-gl " @echo "* ----------------------------> check localhost:8080 " - @echo "* " + @echo "* " @echo "***********************************************************" @echo " " docker run -it --rm -v $$(pwd)/data:/data -p 8080:80 klokantech/tileserver-gl From 18110d8ca6a09e048761f6d6fe6a037f6b260063 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 11 Jan 2017 13:05:17 -0500 Subject: [PATCH 31/54] Do not refresh osm_water_point concurrently "Create a unique index with no WHERE clause on one or more columns of the materialized view." --- layers/water_name/water_point.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 68a7657..856fc9c 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -31,7 +31,7 @@ CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh water_name'; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_water_point; + REFRESH MATERIALIZED VIEW osm_water_point; DELETE FROM water_name.updates; RETURN null; END; From beda95186379a10eb07adefdd842584ad3e9f739 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 11 Jan 2017 16:12:11 -0500 Subject: [PATCH 32/54] Change log notice for water_point --- layers/water_name/water_point.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 856fc9c..7fdc379 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -30,7 +30,7 @@ $$ language plpgsql; CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS $BODY$ BEGIN - RAISE LOG 'Refresh water_name'; + RAISE LOG 'Refresh water_point'; REFRESH MATERIALIZED VIEW osm_water_point; DELETE FROM water_name.updates; RETURN null; From 31e0466b678b5d1a2b05ae5fb0b23d5a458c2ace Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 11 Jan 2017 16:31:47 -0500 Subject: [PATCH 33/54] Disable concurrent refresh as its problematic. --- layers/transportation_name/merge_highways.sql | 8 ++++---- layers/water_name/water_lakeline.sql | 2 +- layers/water_name/water_point.sql | 20 +++++++++---------- layers/waterway/merge_waterway.sql | 8 ++++---- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index d3ebedf..7ad1f16 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -79,10 +79,10 @@ CREATE OR REPLACE FUNCTION transportation_name.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh transportation_name'; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen1; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen2; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_transportation_name_linestring_gen3; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen1; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen2; + REFRESH MATERIALIZED VIEW osm_transportation_name_linestring_gen3; DELETE FROM transportation_name.updates; RETURN null; END; diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index dea4251..56aa383 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -31,7 +31,7 @@ CREATE OR REPLACE FUNCTION water_lakeline.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh water_lakeline'; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_water_lakeline; + REFRESH MATERIALIZED VIEW osm_water_lakeline; DELETE FROM water_lakeline.updates; RETURN null; END; diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 7fdc379..37640da 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -1,5 +1,5 @@ DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; -DROP TRIGGER IF EXISTS trigger_refresh ON water_name.updates; +DROP TRIGGER IF EXISTS trigger_refresh ON water_point.updates; -- etldoc: osm_water_polygon -> osm_water_point -- etldoc: lake_centerline -> osm_water_point @@ -17,22 +17,22 @@ CREATE INDEX IF NOT EXISTS osm_water_point_geometry_idx ON osm_water_point USING -- Handle updates -CREATE SCHEMA IF NOT EXISTS water_name; +CREATE SCHEMA IF NOT EXISTS water_point; -CREATE TABLE IF NOT EXISTS water_name.updates(id serial primary key, t text, unique (t)); -CREATE OR REPLACE FUNCTION water_name.flag() RETURNS trigger AS $$ +CREATE TABLE IF NOT EXISTS water_point.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION water_point.flag() RETURNS trigger AS $$ BEGIN - INSERT INTO water_name.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + INSERT INTO water_point.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; RETURN null; END; $$ language plpgsql; -CREATE OR REPLACE FUNCTION water_name.refresh() RETURNS trigger AS +CREATE OR REPLACE FUNCTION water_point.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh water_point'; REFRESH MATERIALIZED VIEW osm_water_point; - DELETE FROM water_name.updates; + DELETE FROM water_point.updates; RETURN null; END; $BODY$ @@ -41,10 +41,10 @@ language plpgsql; CREATE TRIGGER trigger_flag AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT - EXECUTE PROCEDURE water_name.flag(); + EXECUTE PROCEDURE water_point.flag(); CREATE CONSTRAINT TRIGGER trigger_refresh - AFTER INSERT ON water_name.updates + AFTER INSERT ON water_point.updates INITIALLY DEFERRED FOR EACH ROW - EXECUTE PROCEDURE water_name.refresh(); + EXECUTE PROCEDURE water_point.refresh(); diff --git a/layers/waterway/merge_waterway.sql b/layers/waterway/merge_waterway.sql index 93c8200..5cf6ab0 100644 --- a/layers/waterway/merge_waterway.sql +++ b/layers/waterway/merge_waterway.sql @@ -66,10 +66,10 @@ CREATE OR REPLACE FUNCTION waterway.refresh() RETURNS trigger AS $BODY$ BEGIN RAISE LOG 'Refresh waterway'; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen1; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen2; - REFRESH MATERIALIZED VIEW CONCURRENTLY osm_important_waterway_linestring_gen3; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen1; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen2; + REFRESH MATERIALIZED VIEW osm_important_waterway_linestring_gen3; DELETE FROM waterway.updates; RETURN null; END; From e71311a8d879e170b1c95c1099778c82dba5e964 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Wed, 11 Jan 2017 22:46:24 -0500 Subject: [PATCH 34/54] Have to change name on trigger so they don't overwrite each other... --- layers/water_name/water_lakeline.sql | 4 ++-- layers/water_name/water_point.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/layers/water_name/water_lakeline.sql b/layers/water_name/water_lakeline.sql index 56aa383..d9c58ea 100644 --- a/layers/water_name/water_lakeline.sql +++ b/layers/water_name/water_lakeline.sql @@ -1,4 +1,4 @@ -DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; +DROP TRIGGER IF EXISTS trigger_flag_line ON osm_water_polygon; DROP TRIGGER IF EXISTS trigger_refresh ON water_lakeline.updates; -- etldoc: osm_water_polygon -> osm_water_lakeline @@ -38,7 +38,7 @@ CREATE OR REPLACE FUNCTION water_lakeline.refresh() RETURNS trigger AS $BODY$ language plpgsql; -CREATE TRIGGER trigger_flag +CREATE TRIGGER trigger_flag_line AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT EXECUTE PROCEDURE water_lakeline.flag(); diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 37640da..4870879 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -1,4 +1,4 @@ -DROP TRIGGER IF EXISTS trigger_flag ON osm_water_polygon; +DROP TRIGGER IF EXISTS trigger_flag_point ON osm_water_polygon; DROP TRIGGER IF EXISTS trigger_refresh ON water_point.updates; -- etldoc: osm_water_polygon -> osm_water_point @@ -38,7 +38,7 @@ CREATE OR REPLACE FUNCTION water_point.refresh() RETURNS trigger AS $BODY$ language plpgsql; -CREATE TRIGGER trigger_flag +CREATE TRIGGER trigger_flag_point AFTER INSERT OR UPDATE OR DELETE ON osm_water_polygon FOR EACH STATEMENT EXECUTE PROCEDURE water_point.flag(); From ef233737819aabc5527520351de613b45c5d259d Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Fri, 6 Jan 2017 15:09:21 +0100 Subject: [PATCH 35/54] Add update command to compose --- docker-compose.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1e26f73..76e1199 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: links: - postgres import-osm: - image: "openmaptiles/import-osm:0.2" + image: "openmaptiles/import-osm:latest" env_file: .env environment: DIFF_MODE: ${DIFF_MODE} @@ -36,6 +36,18 @@ services: - ./data:/import - ./build:/mapping - cache:/cache + update-osm: + image: "openmaptiles/import-osm:latest" + env_file: .env + environment: + DIFF_MODE: ${DIFF_MODE} + command: ./import_update.sh + links: + - postgres + volumes: + - ./data:/import + - ./build:/mapping + - cache:/cache import-sql: image: "openmaptiles/import-sql:0.1" env_file: .env From 042c8f2d851df9d357ed2573343e3c2ead5dad95 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 09:57:23 +0100 Subject: [PATCH 36/54] Add command to generate vector tiles from tilelist.txt --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 76e1199..874e8d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,6 +71,15 @@ services: - postgres:db ports: - "3000:3000" + generate-changed-vectortiles: + image: "openmaptiles/generate-vectortiles:0.1" + command: ./export-list.sh + volumes: + - ./data:/export + - ./build/openmaptiles.tm2source:/tm2source + links: + - postgres:postgres + env_file: .env generate-vectortiles: image: "openmaptiles/generate-vectortiles:0.1" volumes: From 28c025f60a500c525a9a828ab4a97e6dd8a56439 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 10:01:36 +0100 Subject: [PATCH 37/54] Add import OSM diff command --- docker-compose.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 874e8d8..84c5e98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,18 @@ services: - ./data:/import - ./build:/mapping - cache:/cache + import-osm-diff: + image: "openmaptiles/import-osm:latest" + env_file: .env + command: ./import_diff.sh + environment: + DIFF_MODE: ${DIFF_MODE} + links: + - postgres + volumes: + - ./data:/import + - ./build:/mapping + - cache:/cache update-osm: image: "openmaptiles/import-osm:latest" env_file: .env From 260f2e579fb8c845acd11fa0c8a9a4ce93522f98 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 10:09:29 +0100 Subject: [PATCH 38/54] Add UPDATE docs --- UPDATE.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 UPDATE.md diff --git a/UPDATE.md b/UPDATE.md new file mode 100644 index 0000000..ed3a32c --- /dev/null +++ b/UPDATE.md @@ -0,0 +1,38 @@ +# Keep the vector tiles updated + +Once you have imported OpenMapTiles you can also keep it up to date by importing the latest OSM changes and +regenerating the tables. + +## Import + +You can either keep the database up to date based on the daily OSM change feed +or import specific change files. + +### Keep Database Updated + +You can use the new imposm3 feature to keep the database updated (thanks to the [work for @stirringhalo](https://github.com/openmaptiles/openmaptiles/pull/131)). This will automatically download +the OSM change feed and import it into the database. +After each run of you should also have a list of tiles that have updated. + +``` +docker-compose run update-osm +``` + +### Import Change File + +Given you have a file `changes.osc.gz` in your import folder. Once you ran the import command you should also have a list of tiles that have updated. + +``` +docker-compose run import-osm-diff +``` + +## Generate Changed Tiles + +After the import has finished **imposm3** will store a list of tiles in text format in the `diffdir`. +Copy the as `tiles.txt` to the import folder. + +Now run the command to read the tilelist and write the vector tiles for it to a new MBTiles. + +``` +docker-compose run generate-changed-vectortiles +``` From 7400b552c2f1cf0fd8e051d5e5b59b12f5615f07 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 16:00:45 +0100 Subject: [PATCH 39/54] Add seas and oceans to water_name --- layers/water_name/layer.sql | 8 ++++++++ layers/water_name/water_name.yaml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/layers/water_name/layer.sql b/layers/water_name/layer.sql index 0bfac8b..143a00d 100644 --- a/layers/water_name/layer.sql +++ b/layers/water_name/layer.sql @@ -19,5 +19,13 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class t WHERE geometry && bbox AND ( (zoom_level BETWEEN 9 AND 13 AND area > 70000*2^(20-zoom_level)) OR (zoom_level >= 14) + ) + -- etldoc: osm_marine_point -> layer_water_name:z14 + UNION ALL + SELECT osm_id, geometry, name, name_en, place::text AS class + FROM osm_marine_point + WHERE geometry && bbox AND ( + (zoom_level <= 4 AND place = 'ocean') + OR (zoom_level >= 5) ); $$ LANGUAGE SQL IMMUTABLE; diff --git a/layers/water_name/water_name.yaml b/layers/water_name/water_name.yaml index a1bd76b..33bff0b 100644 --- a/layers/water_name/water_name.yaml +++ b/layers/water_name/water_name.yaml @@ -12,7 +12,7 @@ layer: At the moment only `lake` since no ocean parts are labelled. *Reserved for future use*. values: - lake - buffer_size: 8 + buffer_size: 64 srs: +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over datasource: geometry_field: geometry From 1df806ba44f85df74aa1361f91dfac120d2f2d21 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 16:23:22 +0100 Subject: [PATCH 40/54] Merge NE marine rank --- layers/water_name/mapping.yaml | 3 ++ layers/water_name/merge_marine_rank.sql | 61 +++++++++++++++++++++++++ layers/water_name/water_name.yaml | 1 + 3 files changed, 65 insertions(+) create mode 100644 layers/water_name/merge_marine_rank.sql diff --git a/layers/water_name/mapping.yaml b/layers/water_name/mapping.yaml index c1d35cc..15476d8 100644 --- a/layers/water_name/mapping.yaml +++ b/layers/water_name/mapping.yaml @@ -16,6 +16,9 @@ tables: - name: place key: place type: string + - name: rank + key: rank + type: integer filters: exclude_tags: - [ "name", "__nil__" ] diff --git a/layers/water_name/merge_marine_rank.sql b/layers/water_name/merge_marine_rank.sql new file mode 100644 index 0000000..45566cf --- /dev/null +++ b/layers/water_name/merge_marine_rank.sql @@ -0,0 +1,61 @@ +DROP TRIGGER IF EXISTS trigger_flag ON osm_marine_point; +DROP TRIGGER IF EXISTS trigger_refresh ON water_name_marine.updates; + +CREATE EXTENSION IF NOT EXISTS unaccent; + +CREATE OR REPLACE FUNCTION update_osm_marine_point() RETURNS VOID AS $$ +BEGIN + -- etldoc: osm_marine_point -> osm_marine_point + UPDATE osm_marine_point AS osm SET "rank" = NULL WHERE "rank" IS NOT NULL; + + -- etldoc: ne_10m_geography_marine_polys -> osm_marine_point + -- etldoc: osm_marine_point -> osm_marine_point + + WITH important_marine_point AS ( + SELECT osm.geometry, osm.osm_id, osm.name, osm.name_en, ne.scalerank + FROM ne_10m_geography_marine_polys AS ne, osm_marine_point AS osm + WHERE ne.name ILIKE osm.name + ) + UPDATE osm_marine_point AS osm + SET "rank" = scalerank + FROM important_marine_point AS ne + WHERE osm.osm_id = ne.osm_id; +END; +$$ LANGUAGE plpgsql; + +SELECT update_osm_marine_point(); + +CREATE INDEX IF NOT EXISTS osm_marine_point_rank_idx ON osm_marine_point("rank"); + +-- Handle updates +CREATE SCHEMA IF NOT EXISTS water_name_marine; + +CREATE TABLE IF NOT EXISTS water_name_marine.updates(id serial primary key, t text, unique (t)); +CREATE OR REPLACE FUNCTION water_name_marine.flag() RETURNS trigger AS $$ +BEGIN + INSERT INTO water_name_marine.updates(t) VALUES ('y') ON CONFLICT(t) DO NOTHING; + RETURN null; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION water_name_marine.refresh() RETURNS trigger AS + $BODY$ + BEGIN + RAISE LOG 'Refresh water_name_marine rank'; + PERFORM update_osm_marine_point(); + DELETE FROM water_name_marine.updates; + RETURN null; + END; + $BODY$ +language plpgsql; + +CREATE TRIGGER trigger_flag + AFTER INSERT OR UPDATE OR DELETE ON osm_marine_point + FOR EACH STATEMENT + EXECUTE PROCEDURE water_name_marine.flag(); + +CREATE CONSTRAINT TRIGGER trigger_refresh + AFTER INSERT ON water_name_marine.updates + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE water_name_marine.refresh(); diff --git a/layers/water_name/water_name.yaml b/layers/water_name/water_name.yaml index 33bff0b..ab9da59 100644 --- a/layers/water_name/water_name.yaml +++ b/layers/water_name/water_name.yaml @@ -19,6 +19,7 @@ layer: srid: 900913 query: (SELECT geometry, name, name_en, class FROM layer_water_name(!bbox!, z(!scale_denominator!))) AS t schema: + - ./merge_marine_rank.sql - ./water_lakeline.sql - ./water_point.sql - ./layer.sql From 14dc0f0926b68c6b7dfc3bb0a1e3f1daea380b66 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 16:40:39 +0100 Subject: [PATCH 41/54] Limit sea/ocean labels by rank --- layers/water_name/layer.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/layers/water_name/layer.sql b/layers/water_name/layer.sql index 143a00d..b71b8cb 100644 --- a/layers/water_name/layer.sql +++ b/layers/water_name/layer.sql @@ -12,7 +12,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class t AND ((zoom_level BETWEEN 9 AND 13 AND LineLabel(zoom_level, NULLIF(name, ''), geometry)) OR (zoom_level >= 14)) -- etldoc: osm_water_point -> layer_water_name:z9_13 - -- etldoc: osm_water_point -> layer_water_name:z14_ + -- etldoc: osm_water_point -> layer_water_name:z14_ UNION ALL SELECT osm_id, geometry, name, name_en, 'lake'::text AS class FROM osm_water_point @@ -20,12 +20,13 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class t (zoom_level BETWEEN 9 AND 13 AND area > 70000*2^(20-zoom_level)) OR (zoom_level >= 14) ) - -- etldoc: osm_marine_point -> layer_water_name:z14 + -- etldoc: osm_marine_point -> layer_water_name:z0_14_ UNION ALL SELECT osm_id, geometry, name, name_en, place::text AS class FROM osm_marine_point WHERE geometry && bbox AND ( - (zoom_level <= 4 AND place = 'ocean') - OR (zoom_level >= 5) + place = 'ocean' + OR (zoom_level <= "rank" AND "rank" IS NOT NULL) + OR (zoom_level >= 8) ); $$ LANGUAGE SQL IMMUTABLE; From 84219067cff5528a34c31d909099104e63160a47 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 16:43:25 +0100 Subject: [PATCH 42/54] Show countries earlier #123 --- layers/place/layer.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layers/place/layer.sql b/layers/place/layer.sql index 0fbe902..5998a6e 100644 --- a/layers/place/layer.sql +++ b/layers/place/layer.sql @@ -21,7 +21,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class t osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, 'country' AS class, "rank", NULL::int AS capital FROM osm_country_point - WHERE geometry && bbox AND "rank" <= zoom_level AND name <> '' + WHERE geometry && bbox AND "rank" <= zoom_level - 1 AND name <> '' UNION ALL -- etldoc: osm_state_point -> layer_place:z0_3 From 8874140207b061c5ca81f30be2c7c14d22987ebe Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 20:11:35 +0100 Subject: [PATCH 43/54] Use boundaries from osmborder --- docker-compose.yml | 7 ++++ layers/boundary/boundary.sql | 60 +++++++++++++++++------------------ layers/boundary/boundary.yaml | 8 +++-- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 84c5e98..8e75af3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,13 @@ services: - ./data:/import - ./build:/mapping - cache:/cache + import-osmborder: + image: "openmaptiles/import-osmborder:0.1" + env_file: .env + links: + - postgres + volumes: + - ./data:/import import-osm-diff: image: "openmaptiles/import-osm:latest" env_file: .env diff --git a/layers/boundary/boundary.sql b/layers/boundary/boundary.sql index 31e951c..d37524c 100644 --- a/layers/boundary/boundary.sql +++ b/layers/boundary/boundary.sql @@ -3,7 +3,7 @@ -- etldoc: ne_110m_admin_0_boundary_lines_land -> boundary_z0 CREATE OR REPLACE VIEW boundary_z0 AS ( - SELECT geometry, 2 AS admin_level + SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_110m_admin_0_boundary_lines_land ); @@ -11,10 +11,10 @@ CREATE OR REPLACE VIEW boundary_z0 AS ( -- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z1 CREATE OR REPLACE VIEW boundary_z1 AS ( - SELECT geometry, 2 AS admin_level + SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_50m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level + SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_50m_admin_1_states_provinces_lines WHERE scalerank <= 2 ); @@ -24,10 +24,10 @@ CREATE OR REPLACE VIEW boundary_z1 AS ( -- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z3 CREATE OR REPLACE VIEW boundary_z3 AS ( - SELECT geometry, 2 AS admin_level + SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_50m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level + SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_50m_admin_1_states_provinces_lines ); @@ -36,10 +36,10 @@ CREATE OR REPLACE VIEW boundary_z3 AS ( -- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z4 CREATE OR REPLACE VIEW boundary_z4 AS ( - SELECT geometry, 2 AS admin_level + SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_10m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level + SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_10m_admin_1_states_provinces_lines_shp WHERE scalerank <= 3 AND featurecla = 'Adm-1 boundary' ); @@ -48,10 +48,10 @@ CREATE OR REPLACE VIEW boundary_z4 AS ( -- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z5 CREATE OR REPLACE VIEW boundary_z5 AS ( - SELECT geometry, 2 AS admin_level + SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_10m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level + SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_10m_admin_1_states_provinces_lines_shp WHERE scalerank <= 7 AND featurecla = 'Adm-1 boundary' ); @@ -60,10 +60,10 @@ CREATE OR REPLACE VIEW boundary_z5 AS ( -- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z6 CREATE OR REPLACE VIEW boundary_z6 AS ( - SELECT geometry, 2 AS admin_level + SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_10m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level + SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_10m_admin_1_states_provinces_lines_shp WHERE scalerank <= 9 AND featurecla = 'Adm-1 boundary' ); @@ -71,55 +71,55 @@ CREATE OR REPLACE VIEW boundary_z6 AS ( -- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z7 -- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z7 CREATE OR REPLACE VIEW boundary_z7 AS ( - SELECT geometry, 2 AS admin_level + SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime FROM ne_10m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level + SELECT geometry, 4 AS admin_level, NULL AS disputed, NULL AS maritime FROM ne_10m_admin_1_states_provinces_lines_shp WHERE featurecla = 'Adm-1 boundary' ); --- etldoc: osm_boundary_linestring_gen5 -> boundary_z8 +-- etldoc: osm_border_linestring_gen5 -> boundary_z8 CREATE OR REPLACE VIEW boundary_z8 AS ( - SELECT geometry, admin_level - FROM osm_boundary_linestring_gen5 + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen5 WHERE admin_level <= 4 AND ST_Length(geometry) > 1000 ); --- etldoc: osm_boundary_linestring_gen4 -> boundary_z9 +-- etldoc: osm_border_linestring_gen4 -> boundary_z9 CREATE OR REPLACE VIEW boundary_z9 AS ( - SELECT geometry, admin_level - FROM osm_boundary_linestring_gen4 + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen4 WHERE admin_level <= 6 ); --- etldoc: osm_boundary_linestring_gen3 -> boundary_z10 +-- etldoc: osm_border_linestring_gen3 -> boundary_z10 CREATE OR REPLACE VIEW boundary_z10 AS ( - SELECT geometry, admin_level - FROM osm_boundary_linestring_gen3 + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen3 WHERE admin_level <= 6 ); --- etldoc: osm_boundary_linestring_gen2 -> boundary_z11 +-- etldoc: osm_border_linestring_gen2 -> boundary_z11 CREATE OR REPLACE VIEW boundary_z11 AS ( - SELECT geometry, admin_level - FROM osm_boundary_linestring_gen2 + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen2 WHERE admin_level <= 8 ); --- etldoc: osm_boundary_linestring_gen1 -> boundary_z12 +-- etldoc: osm_border_linestring_gen1 -> boundary_z12 CREATE OR REPLACE VIEW boundary_z12 AS ( - SELECT geometry, admin_level - FROM osm_boundary_linestring_gen1 + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen1 ); -- etldoc: layer_boundary[shape=record fillcolor=lightpink, style="rounded,filled", -- etldoc: label=" layer_boundary | z0 | z1_2 | z3 | z4 | z5 | z6 | z7 | z8 | z9 | z10 | z11 | z12| z13+"] CREATE OR REPLACE FUNCTION layer_boundary (bbox geometry, zoom_level int) -RETURNS TABLE(geometry geometry, admin_level int) AS $$ - SELECT geometry, admin_level FROM ( +RETURNS TABLE(geometry geometry, admin_level int, disputed int, maritime int) AS $$ + SELECT geometry, admin_level, disputed::int, maritime::int FROM ( -- etldoc: boundary_z0 -> layer_boundary:z0 SELECT * FROM boundary_z0 WHERE geometry && bbox AND zoom_level = 0 UNION ALL diff --git a/layers/boundary/boundary.yaml b/layers/boundary/boundary.yaml index 29dec59..f4acdc2 100644 --- a/layers/boundary/boundary.yaml +++ b/layers/boundary/boundary.yaml @@ -15,12 +15,16 @@ layer: At low zoom levels the Natural Earth boundaries are mapped to the equivalent admin levels. disputed: description: | - Mark with `1` if the boundary is disputed. + Mark with `1` if the border is disputed. + values: [0, 1] + maritime: + description: | + Mark with `1` if it is a maritime border. values: [0, 1] buffer_size: 4 datasource: geometry_field: geometry - query: (SELECT geometry, admin_level FROM layer_boundary(!bbox!, z(!scale_denominator!))) AS t + query: (SELECT geometry, admin_level, disputed, maritime FROM layer_boundary(!bbox!, z(!scale_denominator!))) AS t schema: - ./boundary.sql datasources: From d9f791e11ed47fde72782f2f5a4da9876d1433c7 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 20:50:56 +0100 Subject: [PATCH 44/54] Mix maritime boundaries with NE and switch to OSM earlier --- layers/boundary/boundary.sql | 92 +++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/layers/boundary/boundary.sql b/layers/boundary/boundary.sql index d37524c..d0b7b2f 100644 --- a/layers/boundary/boundary.sql +++ b/layers/boundary/boundary.sql @@ -3,7 +3,7 @@ -- etldoc: ne_110m_admin_0_boundary_lines_land -> boundary_z0 CREATE OR REPLACE VIEW boundary_z0 AS ( - SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 2 AS admin_level, false AS disputed, false AS maritime FROM ne_110m_admin_0_boundary_lines_land ); @@ -11,10 +11,10 @@ CREATE OR REPLACE VIEW boundary_z0 AS ( -- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z1 CREATE OR REPLACE VIEW boundary_z1 AS ( - SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 2 AS admin_level, false AS disputed, false AS maritime FROM ne_50m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 4 AS admin_level, false AS disputed, false AS maritime FROM ne_50m_admin_1_states_provinces_lines WHERE scalerank <= 2 ); @@ -24,92 +24,98 @@ CREATE OR REPLACE VIEW boundary_z1 AS ( -- etldoc: ne_50m_admin_1_states_provinces_lines -> boundary_z3 CREATE OR REPLACE VIEW boundary_z3 AS ( - SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 2 AS admin_level, false AS disputed, false AS maritime FROM ne_50m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 4 AS admin_level, false AS disputed, false AS maritime FROM ne_50m_admin_1_states_provinces_lines ); -- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z4 -- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z4 +-- etldoc: osm_border_linestring_gen10 -> boundary_z4 CREATE OR REPLACE VIEW boundary_z4 AS ( - SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 2 AS admin_level, false AS disputed, false AS maritime FROM ne_10m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 4 AS admin_level, false AS disputed, false AS maritime FROM ne_10m_admin_1_states_provinces_lines_shp WHERE scalerank <= 3 AND featurecla = 'Adm-1 boundary' + UNION ALL + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen10 + WHERE maritime=true AND admin_level <= 2 ); -- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z5 -- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z5 +-- etldoc: osm_border_linestring_gen9 -> boundary_z5 CREATE OR REPLACE VIEW boundary_z5 AS ( - SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 2 AS admin_level, false AS disputed, false AS maritime FROM ne_10m_admin_0_boundary_lines_land UNION ALL - SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime + SELECT geometry, 4 AS admin_level, false AS disputed, false AS maritime FROM ne_10m_admin_1_states_provinces_lines_shp WHERE scalerank <= 7 AND featurecla = 'Adm-1 boundary' + UNION ALL + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen9 + WHERE maritime=true AND admin_level <= 2 ); --- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z6 --- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z6 - +-- etldoc: osm_border_linestring_gen8 -> boundary_z6 CREATE OR REPLACE VIEW boundary_z6 AS ( - SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime - FROM ne_10m_admin_0_boundary_lines_land - UNION ALL - SELECT geometry, 4 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime - FROM ne_10m_admin_1_states_provinces_lines_shp - WHERE scalerank <= 9 AND featurecla = 'Adm-1 boundary' + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen8 + WHERE admin_level <= 4 ); --- etldoc: ne_10m_admin_0_boundary_lines_land -> boundary_z7 --- etldoc: ne_10m_admin_1_states_provinces_lines_shp -> boundary_z7 +-- etldoc: osm_border_linestring_gen7 -> boundary_z7 CREATE OR REPLACE VIEW boundary_z7 AS ( - SELECT geometry, 2 AS admin_level, NULL::bool AS disputed, NULL::bool AS maritime - FROM ne_10m_admin_0_boundary_lines_land - UNION ALL - SELECT geometry, 4 AS admin_level, NULL AS disputed, NULL AS maritime - FROM ne_10m_admin_1_states_provinces_lines_shp - WHERE featurecla = 'Adm-1 boundary' - + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen7 + WHERE admin_level <= 4 ); --- etldoc: osm_border_linestring_gen5 -> boundary_z8 +-- etldoc: osm_border_linestring_gen6 -> boundary_z8 CREATE OR REPLACE VIEW boundary_z8 AS ( SELECT geometry, admin_level, disputed, maritime - FROM osm_border_linestring_gen5 - WHERE admin_level <= 4 AND ST_Length(geometry) > 1000 + FROM osm_border_linestring_gen6 + WHERE admin_level <= 4 ); --- etldoc: osm_border_linestring_gen4 -> boundary_z9 +-- etldoc: osm_border_linestring_gen5 -> boundary_z9 CREATE OR REPLACE VIEW boundary_z9 AS ( + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen5 + WHERE admin_level <= 6 +); + +-- etldoc: osm_border_linestring_gen4 -> boundary_z10 +CREATE OR REPLACE VIEW boundary_z10 AS ( SELECT geometry, admin_level, disputed, maritime FROM osm_border_linestring_gen4 WHERE admin_level <= 6 ); --- etldoc: osm_border_linestring_gen3 -> boundary_z10 -CREATE OR REPLACE VIEW boundary_z10 AS ( - SELECT geometry, admin_level, disputed, maritime - FROM osm_border_linestring_gen3 - WHERE admin_level <= 6 -); - --- etldoc: osm_border_linestring_gen2 -> boundary_z11 +-- etldoc: osm_border_linestring_gen3 -> boundary_z11 CREATE OR REPLACE VIEW boundary_z11 AS ( SELECT geometry, admin_level, disputed, maritime - FROM osm_border_linestring_gen2 + FROM osm_border_linestring_gen3 WHERE admin_level <= 8 ); --- etldoc: osm_border_linestring_gen1 -> boundary_z12 +-- etldoc: osm_border_linestring_gen2 -> boundary_z12 CREATE OR REPLACE VIEW boundary_z12 AS ( + SELECT geometry, admin_level, disputed, maritime + FROM osm_border_linestring_gen2 +); + +-- etldoc: osm_border_linestring_gen1 -> boundary_z12 +CREATE OR REPLACE VIEW boundary_z13 AS ( SELECT geometry, admin_level, disputed, maritime FROM osm_border_linestring_gen1 ); @@ -156,7 +162,7 @@ RETURNS TABLE(geometry geometry, admin_level int, disputed int, maritime int) AS -- etldoc: boundary_z12 -> layer_boundary:z12 SELECT * FROM boundary_z12 WHERE geometry && bbox AND zoom_level = 12 UNION ALL - -- etldoc: boundary_z12 -> layer_boundary:z13 - SELECT * FROM boundary_z12 WHERE geometry && bbox AND zoom_level >= 13 + -- etldoc: boundary_z13 -> layer_boundary:z13 + SELECT * FROM boundary_z13 WHERE geometry && bbox AND zoom_level >= 13 ) AS zoom_levels; $$ LANGUAGE SQL IMMUTABLE; From 8a99e49df70253c3d135bd34dfe71ca1fdc22fe9 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 20:54:14 +0100 Subject: [PATCH 45/54] Fix version of import-osmborder --- docker-compose.yml | 2 +- openmaptiles.yaml | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8e75af3..0476c1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: - ./build:/mapping - cache:/cache import-osmborder: - image: "openmaptiles/import-osmborder:0.1" + image: "openmaptiles/import-osmborder:0.2" env_file: .env links: - postgres diff --git a/openmaptiles.yaml b/openmaptiles.yaml index ded8574..6423619 100644 --- a/openmaptiles.yaml +++ b/openmaptiles.yaml @@ -1,19 +1,6 @@ tileset: layers: - - layers/water/water.yaml - - layers/waterway/waterway.yaml - - layers/landcover/landcover.yaml - - layers/landuse/landuse.yaml - - layers/park/park.yaml - layers/boundary/boundary.yaml - - layers/aeroway/aeroway.yaml - - layers/transportation/transportation.yaml - - layers/building/building.yaml - - layers/water_name/water_name.yaml - - layers/transportation_name/transportation_name.yaml - - layers/place/place.yaml - - layers/housenumber/housenumber.yaml - - layers/poi/poi.yaml name: OpenMapTiles version: 3.3.0 id: openmaptiles From dcfc14ab4cbb4cdc10e7da55ff1609342bb21566 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Mon, 16 Jan 2017 20:58:24 +0100 Subject: [PATCH 46/54] Readd missing layers to tileset --- openmaptiles.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/openmaptiles.yaml b/openmaptiles.yaml index 6423619..c207884 100644 --- a/openmaptiles.yaml +++ b/openmaptiles.yaml @@ -1,9 +1,22 @@ tileset: layers: + - layers/water/water.yaml + - layers/waterway/waterway.yaml + - layers/landcover/landcover.yaml + - layers/landuse/landuse.yaml + - layers/park/park.yaml - layers/boundary/boundary.yaml + - layers/aeroway/aeroway.yaml + - layers/transportation/transportation.yaml + - layers/building/building.yaml + - layers/water_name/water_name.yaml + - layers/transportation_name/transportation_name.yaml + - layers/place/place.yaml + - layers/housenumber/housenumber.yaml + - layers/poi/poi.yaml name: OpenMapTiles version: 3.3.0 - id: openmaptiles + id: openmaptiles description: "A tileset showcasing all layers in OpenMapTiles. http://openmaptiles.org" attribution: '© OpenMapTiles © OpenStreetMap contributors' center: [-12.2168, 28.6135, 4] From 6f842a685984c46b7e95b1543c19fcd581e98ba7 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Mon, 16 Jan 2017 20:54:10 -0500 Subject: [PATCH 47/54] Switch st_union to st_collect --- layers/transportation_name/merge_highways.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/layers/transportation_name/merge_highways.sql b/layers/transportation_name/merge_highways.sql index 7ad1f16..1be7840 100644 --- a/layers/transportation_name/merge_highways.sql +++ b/layers/transportation_name/merge_highways.sql @@ -20,14 +20,14 @@ CREATE MATERIALIZED VIEW osm_transportation_name_linestring AS ( member_osm_ids[0] AS osm_id, member_osm_ids, name, - ref, + ref, highway, z_order FROM ( SELECT - ST_LineMerge(ST_Union(geometry)) AS geometry, + ST_LineMerge(ST_Collect(geometry)) AS geometry, name, - ref, + ref, highway, min(z_order) AS z_order, array_agg(DISTINCT osm_id) AS member_osm_ids From f0786f6dea3fb5035590f1dc71ec27088eee4429 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Tue, 17 Jan 2017 07:18:03 +0100 Subject: [PATCH 48/54] Add border import to quickstart --- quickstart.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/quickstart.sh b/quickstart.sh index 1330d02..b7e8f65 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -194,6 +194,14 @@ echo " : Data license: http://openstreetmapdata.com/info/license " echo " : Thank you: http://openstreetmapdata.com/info/supporting " docker-compose run --rm import-water +echo " " +echo "-------------------------------------------------------------------------------------" +echo "====> : Start importing border data from http://openstreetmap.org into PostgreSQL " +echo " : Source code: https://github.com/openmaptiles/import-osmborder" +echo " : Data license: http://www.openstreetmap.org/copyright" +echo " : Thank you: https://github.com/pnorman/osmborder " +docker-compose run --rm import-osmborder + echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL " From 153976735fb673c600773b162db826e32f502f76 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Tue, 17 Jan 2017 09:03:40 -0500 Subject: [PATCH 49/54] ST_PointOnSurface instead of topoint --- layers/housenumber/housenumber_centroid.sql | 2 +- layers/place/island_polygon_update.sql | 2 +- layers/poi/poi_polygon_update.sql | 2 +- layers/water_name/water_point.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/layers/housenumber/housenumber_centroid.sql b/layers/housenumber/housenumber_centroid.sql index 29c687e..20e323c 100644 --- a/layers/housenumber/housenumber_centroid.sql +++ b/layers/housenumber/housenumber_centroid.sql @@ -4,7 +4,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON housenumber.updates; -- etldoc: osm_housenumber_point -> osm_housenumber_point CREATE OR REPLACE FUNCTION convert_housenumber_point() RETURNS VOID AS $$ BEGIN - UPDATE osm_housenumber_point SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; + UPDATE osm_housenumber_point SET geometry=ST_PointOnSurface(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; END; $$ LANGUAGE plpgsql; diff --git a/layers/place/island_polygon_update.sql b/layers/place/island_polygon_update.sql index ef937a5..91e6b36 100644 --- a/layers/place/island_polygon_update.sql +++ b/layers/place/island_polygon_update.sql @@ -4,7 +4,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON place_island.updates; -- etldoc: osm_island_polygon -> osm_island_polygon CREATE OR REPLACE FUNCTION convert_island_polygon_point() RETURNS VOID AS $$ BEGIN - UPDATE osm_island_polygon SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; + UPDATE osm_island_polygon SET geometry=ST_PointOnSurface(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; ANALYZE osm_island_polygon; END; $$ LANGUAGE plpgsql; diff --git a/layers/poi/poi_polygon_update.sql b/layers/poi/poi_polygon_update.sql index 22484ac..891ae2e 100644 --- a/layers/poi/poi_polygon_update.sql +++ b/layers/poi/poi_polygon_update.sql @@ -5,7 +5,7 @@ DROP TRIGGER IF EXISTS trigger_refresh ON poi.updates; CREATE OR REPLACE FUNCTION convert_poi_point() RETURNS VOID AS $$ BEGIN - UPDATE osm_poi_polygon SET geometry=topoint(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; + UPDATE osm_poi_polygon SET geometry=ST_PointOnSurface(geometry) WHERE ST_GeometryType(geometry) <> 'ST_Point'; ANALYZE osm_poi_polygon; END; $$ LANGUAGE plpgsql; diff --git a/layers/water_name/water_point.sql b/layers/water_name/water_point.sql index 4870879..9c46e7f 100644 --- a/layers/water_name/water_point.sql +++ b/layers/water_name/water_point.sql @@ -7,7 +7,7 @@ DROP MATERIALIZED VIEW IF EXISTS osm_water_point CASCADE; CREATE MATERIALIZED VIEW osm_water_point AS ( SELECT - wp.osm_id, topoint(wp.geometry) AS geometry, + wp.osm_id, ST_PointOnSurface(wp.geometry) AS geometry, wp.name, wp.name_en, ST_Area(wp.geometry) AS area FROM osm_water_polygon AS wp LEFT JOIN lake_centerline ll ON wp.osm_id = ll.osm_id From 8273bfbb629f3fc72e707486472673ecf5bcc694 Mon Sep 17 00:00:00 2001 From: stirringhalo Date: Tue, 17 Jan 2017 09:45:58 -0500 Subject: [PATCH 50/54] Switch to webmercarea --- layers/aeroway/mapping.yaml | 2 +- layers/building/mapping.yaml | 2 +- layers/landcover/mapping.yaml | 2 +- layers/landuse/mapping.yaml | 2 +- layers/park/mapping.yaml | 2 +- layers/place/mapping.yaml | 2 +- layers/water/mapping.yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/layers/aeroway/mapping.yaml b/layers/aeroway/mapping.yaml index 0d77037..2f1b90e 100644 --- a/layers/aeroway/mapping.yaml +++ b/layers/aeroway/mapping.yaml @@ -23,7 +23,7 @@ tables: key: aeroway type: string - name: area - type: pseudoarea + type: webmerc_area mapping: aeroway: - aerodrome diff --git a/layers/building/mapping.yaml b/layers/building/mapping.yaml index e4489f0..49783d9 100644 --- a/layers/building/mapping.yaml +++ b/layers/building/mapping.yaml @@ -16,7 +16,7 @@ tables: - name: geometry type: validated_geometry - name: area - type: pseudoarea + type: webmerc_area - name: building key: building type: string diff --git a/layers/landcover/mapping.yaml b/layers/landcover/mapping.yaml index ffa6d7d..337ac5e 100644 --- a/layers/landcover/mapping.yaml +++ b/layers/landcover/mapping.yaml @@ -58,7 +58,7 @@ tables: key: wetland type: string - name: area - type: pseudoarea + type: webmerc_area mapping: landuse: - allotments diff --git a/layers/landuse/mapping.yaml b/layers/landuse/mapping.yaml index 3773a38..7527e8a 100644 --- a/layers/landuse/mapping.yaml +++ b/layers/landuse/mapping.yaml @@ -39,7 +39,7 @@ tables: key: leisure type: string - name: area - type: pseudoarea + type: webmerc_area mapping: amenity: - school diff --git a/layers/park/mapping.yaml b/layers/park/mapping.yaml index 29fb1f0..0649d9a 100644 --- a/layers/park/mapping.yaml +++ b/layers/park/mapping.yaml @@ -67,7 +67,7 @@ tables: key: boundary type: string - name: area - type: pseudoarea + type: webmerc_area mapping: leisure: - nature_reserve diff --git a/layers/place/mapping.yaml b/layers/place/mapping.yaml index ab14e41..57cbce2 100644 --- a/layers/place/mapping.yaml +++ b/layers/place/mapping.yaml @@ -97,7 +97,7 @@ tables: - name: geometry type: geometry - name: area - type: pseudoarea + type: webmerc_area - *name - *name_en - *name_de diff --git a/layers/water/mapping.yaml b/layers/water/mapping.yaml index e923fb0..4d31fb0 100644 --- a/layers/water/mapping.yaml +++ b/layers/water/mapping.yaml @@ -40,7 +40,7 @@ tables: - name: geometry type: validated_geometry - name: area - type: pseudoarea + type: webmerc_area - key: name name: name type: string From d5350ee5350a1b9bdaf6a2f1b8d6614f53097221 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Wed, 18 Jan 2017 09:35:28 +0100 Subject: [PATCH 51/54] Correct rank query #123 --- layers/place/layer.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layers/place/layer.sql b/layers/place/layer.sql index 5998a6e..247d93b 100644 --- a/layers/place/layer.sql +++ b/layers/place/layer.sql @@ -21,7 +21,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class t osm_id, geometry, name, COALESCE(NULLIF(name_en, ''), name) AS name_en, 'country' AS class, "rank", NULL::int AS capital FROM osm_country_point - WHERE geometry && bbox AND "rank" <= zoom_level - 1 AND name <> '' + WHERE geometry && bbox AND "rank" <= zoom_level + 1 AND name <> '' UNION ALL -- etldoc: osm_state_point -> layer_place:z0_3 From 7f4ca37a6ab8b9b7e25bb225deba6bb92424f242 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Wed, 18 Jan 2017 10:19:39 +0100 Subject: [PATCH 52/54] Do not show seas at z0 --- layers/water_name/layer.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layers/water_name/layer.sql b/layers/water_name/layer.sql index b71b8cb..c79e70d 100644 --- a/layers/water_name/layer.sql +++ b/layers/water_name/layer.sql @@ -26,7 +26,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, name text, name_en text, class t FROM osm_marine_point WHERE geometry && bbox AND ( place = 'ocean' - OR (zoom_level <= "rank" AND "rank" IS NOT NULL) + OR (zoom_level >= 1 AND zoom_level <= "rank" AND "rank" IS NOT NULL) OR (zoom_level >= 8) ); $$ LANGUAGE SQL IMMUTABLE; From 2ec80a60aee73270346399e717f51fb979cf05a5 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Wed, 18 Jan 2017 13:39:33 +0100 Subject: [PATCH 53/54] Do not mount import to import-osmborder --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0476c1b..c4b9709 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,8 +41,6 @@ services: env_file: .env links: - postgres - volumes: - - ./data:/import import-osm-diff: image: "openmaptiles/import-osm:latest" env_file: .env From 95bcf9c1c13839e076fbd4e51bd423746f2dbbb8 Mon Sep 17 00:00:00 2001 From: Lukas Martinelli Date: Wed, 18 Jan 2017 13:58:55 +0100 Subject: [PATCH 54/54] Pull major railway up to z12 #77 --- layers/transportation/layer.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/layers/transportation/layer.sql b/layers/transportation/layer.sql index 65e24df..2765a1e 100644 --- a/layers/transportation/layer.sql +++ b/layers/transportation/layer.sql @@ -96,6 +96,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class text, ramp int, oneway int ) UNION ALL + -- etldoc: osm_railway_linestring -> layer_transportation:z12" -- etldoc: osm_railway_linestring -> layer_transportation:z13 -- etldoc: osm_railway_linestring -> layer_transportation:z14_ SELECT @@ -103,7 +104,7 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class text, ramp int, oneway int service_value(service) AS service, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_railway_linestring - WHERE zoom_level = 13 AND (railway='rail' AND service = '') + WHERE zoom_level BETWEEN 12 AND 13 AND (railway='rail' AND service = '') OR zoom_Level >= 14 UNION ALL