diff --git a/gen_etl_graph.sh b/gen_etl_graph.sh new file mode 100755 index 0000000..ddd3520 --- /dev/null +++ b/gen_etl_graph.sh @@ -0,0 +1,21 @@ + + +DOT=highway.dot + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +cat ./layers/highway/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +cat ./layers/highway/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/highway/ne_global_roads.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +cat ./layers/highway/highway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT + +echo " +} +" >> $DOT + +cat $DOT + +dot -Tpng $DOT > layer_highway_etl.png diff --git a/layers/highway/highway.sql b/layers/highway/highway.sql index a6c14a2..d233f8c 100644 --- a/layers/highway/highway.sql +++ b/layers/highway/highway.sql @@ -1,7 +1,19 @@ + + +-- etldoc: highway_sql[label="highway.sql", shape=note ] +-- etldoc: function_highway_is_link[label="FUNCTION highway_is_link"] +-- etldoc: highway_sql-> function_highway_is_link +-- etldoc: function_highway_is_link -> postgreSQL + CREATE OR REPLACE FUNCTION highway_is_link(highway TEXT) RETURNS BOOLEAN AS $$ SELECT highway LIKE '%_link'; $$ LANGUAGE SQL IMMUTABLE STRICT; + +-- etldoc: postgreSQL -> layer_highway +-- etldoc: highway_sql-> layer_highway +-- etldoc: layer_highway[shape=record fillcolor=lightpink, style="rounded,filled", +-- etldoc: label=" layer_highway (highway.sql) | z4-z7 | z8 | z9 | z10 | z11 | z12| z13| z14" ] ; CREATE OR REPLACE FUNCTION layer_highway(bbox geometry, zoom_level int) RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass text, properties highway_properties) AS $$ SELECT @@ -9,29 +21,41 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te to_highway_class(highway) AS class, highway AS subclass, to_highway_properties(is_bridge, is_tunnel, is_ford, is_ramp, is_oneway) AS properties FROM ( - SELECT + + -- etldoc: ne_10m_global_roads -> layer_highway:z4z7 + SELECT NULL::bigint AS osm_id, geometry, highway, FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway, 0 AS z_order FROM ne_10m_global_roads WHERE zoom_level BETWEEN 4 AND 7 AND scalerank <= 1 + zoom_level UNION ALL + + -- etldoc: osm_highway_linestring_gen4 -> layer_highway:z8 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring_gen4 WHERE zoom_level = 8 UNION ALL + + -- etldoc: osm_highway_linestring_gen3 -> layer_highway:z9 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring_gen3 WHERE zoom_level = 9 UNION ALL + + -- etldoc: osm_highway_linestring_gen2 -> layer_highway:z10 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring_gen2 WHERE zoom_level = 10 UNION ALL + + -- etldoc: osm_highway_linestring_gen1 -> layer_highway:z11 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring_gen1 WHERE zoom_level = 11 UNION ALL + + -- etldoc: osm_highway_linestring -> layer_highway:z12 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring WHERE zoom_level = 12 @@ -39,18 +63,25 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te AND NOT highway_is_link(highway) AND NOT is_area UNION ALL + + -- etldoc: osm_highway_linestring -> layer_highway:z13 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring WHERE zoom_level = 13 AND to_highway_class(highway) < 'path'::highway_class AND NOT is_area UNION ALL + + -- etldoc: osm_highway_linestring -> layer_highway:z14 SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order FROM osm_highway_linestring WHERE zoom_level >= 14 AND NOT is_area UNION ALL + -- NOTE: We limit the selection of polys because we need to be careful to net get false positives here because -- it is possible that closed linestrings appear both as highway linestrings and as polygon + -- etldoc: osm_highway_polygon -> layer_highway:z13 + -- etldoc: osm_highway_polygon -> layer_highway:z14 SELECT osm_id, geometry, highway, FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway, z_order FROM osm_highway_polygon -- We do not want underground pedestrian areas for now diff --git a/layers/highway/mapping.yaml b/layers/highway/mapping.yaml index b11bb57..1f76e47 100644 --- a/layers/highway/mapping.yaml +++ b/layers/highway/mapping.yaml @@ -1,21 +1,43 @@ + + + +# etldoc: imposm3 [fillcolor=yellow, style="rounded,filled", shape=diamond ]; +# etldoc: osm [fillcolor=lightblue, +# etldoc: style="rounded,filled", +# etldoc: shape=box , label="OpenStreetMap", +# etldoc: url="http://www.openstreetmap.org"]; +# etldoc: mapping_yml[label="mapping.yml", shape=note ] +# etldoc: mapping_yml -> imposm3 +# etldoc: osm -> imposm3 + generalized_tables: + +# etldoc: imposm3 -> osm_highway_linestring_gen4 highway_linestring_gen4: source: highway_linestring_gen3 sql_filter: highway IN ('motorway','trunk') AND NOT is_area tolerance: 200.0 + +# etldoc: imposm3 -> osm_highway_linestring_gen3 highway_linestring_gen3: source: highway_linestring_gen2 sql_filter: highway IN ('motorway','trunk', 'primary') AND NOT is_area tolerance: 120.0 + +# etldoc: imposm3 -> osm_highway_linestring_gen2 highway_linestring_gen2: source: highway_linestring_gen1 sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary') AND NOT is_area tolerance: 50.0 + +# etldoc: imposm3 -> osm_highway_linestring_gen1 highway_linestring_gen1: source: highway_linestring sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary', 'tertiary') AND NOT is_area tolerance: 20.0 + tables: +# etldoc: imposm3 -> osm_highway_linestring highway_linestring: type: linestring fields: @@ -85,6 +107,8 @@ tables: - corridor - crossing - pedestrian + +# etldoc: imposm3 -> osm_highway_polygon highway_polygon: type: polygon fields: diff --git a/layers/highway/ne_global_roads.sql b/layers/highway/ne_global_roads.sql index f52bb0f..42ed1bd 100644 --- a/layers/highway/ne_global_roads.sql +++ b/layers/highway/ne_global_roads.sql @@ -1,3 +1,10 @@ + + +-- etldoc: ne_global_roads_sql [label="ne_global_roads.sql", shape=note ] +-- etldoc: function_ne_highway[label="FUNCTION ne_highway"] +-- etldoc: ne_global_roads_sql -> function_ne_highway +-- etldoc: function_ne_highway -> postgreSQL + CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$ SELECT CASE type WHEN 'Major Highway' THEN 'motorway' @@ -7,13 +14,24 @@ CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$ END; $$ LANGUAGE SQL IMMUTABLE; + +-- etldoc: natural_earth [fillcolor=lightblue, style="rounded,filled", shape=box , label="Natural Earth" ]; +-- etldoc: natural_earth -> ne_10m_roads +-- etldoc: natural_earth -> ne_10m_roads_north_america + +-- etldoc: ne_global_roads_sql -> ne_10m_global_roads ; CREATE TABLE IF NOT EXISTS ne_10m_global_roads AS ( + + -- etldoc: ne_10m_roads -> ne_10m_global_roads SELECT geom AS geometry, scalerank, ne_highway(type) AS highway FROM ne_10m_roads WHERE continent <> 'North America' AND featurecla = 'Road' AND type IN ('Major Highway', 'Secondary Highway', 'Road') UNION ALL + + -- etldoc: function_ne_highway -> ne_10m_global_roads + -- etldoc: ne_10m_roads_north_america -> ne_10m_global_roads SELECT geom AS geometry, scalerank, ne_highway(type) AS highway FROM ne_10m_roads_north_america WHERE type IN ('Major Highway', 'Secondary Highway', 'Road') diff --git a/layers/highway/types.sql b/layers/highway/types.sql index f7346fa..a1e8443 100644 --- a/layers/highway/types.sql +++ b/layers/highway/types.sql @@ -1,4 +1,11 @@ + + +-- etldoc: types_sql[label="types.sql", shape=note ] + DO $$ +-- etldoc: type_highway_class[label="TYPE highway_class"] +-- etldoc: types_sql-> type_highway_class +-- etldoc: type_highway_class -> postgreSQL BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_class') THEN CREATE TYPE highway_class AS ENUM ('motorway', 'major_road', 'minor_road', 'path'); @@ -7,6 +14,9 @@ END $$; DO $$ +-- etldoc: type_highway_properties[label="TYPE highway_properties"] +-- etldoc: types_sql-> type_highway_properties +-- etldoc: type_highway_properties -> postgreSQL BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_properties') THEN CREATE TYPE highway_properties AS ENUM ('bridge:oneway', 'tunnel:oneway', 'ramp', 'ford', 'bridge', 'tunnel', 'oneway'); @@ -15,6 +25,9 @@ END $$; +-- etldoc: function_to_highway_class[label="FUNCTION to_highway_class"] +-- etldoc: types_sql-> function_to_highway_class +-- etldoc: function_to_highway_class -> postgreSQL CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class AS $$ SELECT CASE WHEN highway IN ('motorway', 'motorway_link') THEN 'motorway'::highway_class @@ -29,6 +42,9 @@ CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class END; $$ LANGUAGE SQL IMMUTABLE STRICT; +-- etldoc: function_to_highway_properties[label="FUNCTION to_highway_properties"] +-- etldoc: types_sql-> function_to_highway_properties +-- etldoc: function_to_highway_properties -> postgreSQL CREATE OR REPLACE FUNCTION to_highway_properties(is_bridge boolean, is_tunnel boolean, is_ford boolean, is_ramp boolean, is_oneway boolean) RETURNS highway_properties AS $$ SELECT CASE WHEN is_bridge AND is_oneway THEN 'bridge:oneway'::highway_properties