diff --git a/.gitignore b/.gitignore index 996b8cf..15d8c4a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ __pycache__/ *.osm *.o5m *.osm.pbf -.R* \ No newline at end of file +.R* +.direnv +entsoe/result +entsoe/data \ No newline at end of file diff --git a/entsoe/README.md b/entsoe/README.md index 771cd58..dbf9bb8 100644 --- a/entsoe/README.md +++ b/entsoe/README.md @@ -1,8 +1,8 @@ # Unofficial ENTSO-E dataset processed by GridKit -This dataset was generated based on a map extract from May 11, 2016. +This dataset was generated based on a map extract from May 25, 2018. This is an _unofficial_ extract of the -[ENTSO-E interactive map](https://www.entsoe.eu/map/Pages/default.aspx) +[ENTSO-E interactive map](https://www.entsoe.eu/data/map/) of the European power system (including to a limited extent North Africa and the Middle East). The dataset has been processed by GridKit to form complete topological connections. This dataset is neither @@ -14,12 +14,11 @@ This dataset may be inaccurate in several ways, notably: is known to choose topological clarity over geographical accuracy. Hence coordinates will not correspond exactly to reality. + Voltage levels are typically provided as ranges by ENTSO-E, of which - the lower bound has been reported in this dataset. Not all lines - - especially DC lines - contain voltage information. + the lower bound has been reported in this dataset. + Line structure conflicts are resolved by picking the first structure in the set + Transformers are _not present_ in the original ENTSO-E dataset, - there presence has been derived from the different voltages from + their presence has been derived from the different voltages from connected lines. + The connection between generators and busses is derived as the geographically nearest station at the lowest voltage level. This @@ -29,7 +28,6 @@ All users are advised to exercise caution in the use of this dataset. No liability is taken for inaccuracies. - ## Contents of dataset This dataset is provided as set of CSV files that describe the ENTSO-E @@ -53,39 +51,51 @@ Example code for reading the files: Describes terminals, vertices, or 'nodes' of the system + `bus_id`: the unique identifier for the bus -+ `station_id`: the substation or plant of this; a station may have - multiple buses, which are typically connected by transformers ++ `station_id`: unique identifier of its substation; a station may have multiple buses, which are typically connected by transformers + `voltage`: the operating voltage of this bus + `dc`: boolean ('t' or 'f'), describes whether the bus is a HVDC terminal (t) or a regular AC terminal (f) + `symbol`: type of station of this bus. ++ `under_construction`: boolean ('t' if station is currently under construction, + 'f' otherwise) + `tags`: _hstore_ encoded dictionary of 'extra' properties for this bus -+ `under_construction`: boolean ('t' if station is currently under - construction, 'f' otherwise) -+ `geometry`: location of the station in well-known-text format (WGS84) ++ `x`: longitude of its location ++ `y`: latitude of its location **NOTA BENE**: During the processing of the network, so called 'synthetic' stations may be inserted on locations where lines are apparantly connected. Such synthetic stations can be recognised because their symbol is always `joint`. +### lines.csv: + +Buses are connected by AC-lines: + ++ `line_id`: unique identifier for the line ++ `bus0`: first of the two connected buses ++ `bus1`: second of two connected buses ++ `voltage`: operating voltage of the line (identical to operating voltage of + the bus) ++ `circuits`: number of (independent) circuits in this link, each of which + typically has 3 cables. ++ `length`: length of line in km ++ `underground`: boolean, `t` if this is an underground cable, `f` for + an overhead line ++ `under_construction`: boolean, `t` for lines that are currently + under construction ++ `tags`: _hstore_ encoded dictionary of extra properties for this link ++ `geometry`: extent of this line in well-known-text format (WGS84) + ### links.csv: Connections between buses: + `link_id`: unique identifier for the link -+ `src_bus_id`: first of the two connected buses -+ `dst_bus_id`: second of two connected buses -+ `voltage`: operating voltage of the link (_must_ be identical to - operating voltage of the buses) -+ `circuits`: number of (independent) circuits in this link, each of - which typically has 3 cables (for AC lines). -+ `dc`: boolean, `t` if this is a HVDC line -+ `underground`: boolean, `t` if this is an underground cable, `f` for - an overhead line ++ `bus0`: first of the two connected buses ++ `bus1`: second of two connected buses ++ `length`: length of line in km + `under_construction`: boolean, `t` for lines that are currently under construction -+ `length_m`: length of line in meters + `tags`: _hstore_ encoded dictionary of extra properties for this link + `geometry`: extent of this line in well-known-text format (WGS84) @@ -95,26 +105,26 @@ Generators attached to the network. + `generator_id`: unique identifier for the generator + `bus_id`: the bus to which this generator is connected -+ `symbol`: type of generator -+ `capacity`: capacity of this generator (in megawatt) ++ `technology`: type of generator ++ `capacity`: capacity of this generator in MW + `tags`: _hstore_ encoded dictionary of extra attributes + `geometry`: location of generator in well-known text format (WGS84) ### transformers.csv -A transformer forms a link between buses which operate at distinct -voltages. **NOTA BENE**: Transformers _never_ originate from the -original dataset, and transformers are _only_ infered in 'real' -stations, never in synthetic ('joint') stations. +A transformer connects buses which operate at distinct voltages. **NOTA BENE**: +Transformers are _not_ represented in the original dataset, but instead have +been added at substations to connect AC transmission lines of distinct voltage +levels. + `transformer_id`: unique identifier -+ `symbol`: either `transformer` for AC-to-AC voltage transformers, or - `ac/dc` for AC-to-DC converters. -+ `src_bus_id`: first of the connected buses -+ `dst_bus_id`: second of connected buses -+ `src_voltage`: voltage of first bus -+ `dst_voltage`: voltage of second bus -+ `src_dc`: boolean, `t` if first bus is a DC terminal -+ `dst_dc`: boolean, `f` if second bus is a DC terminal -+ `geometry`: location of station of this transformer in well-known - text format (WGS84) ++ `bus0`: Bus at lower voltage level + `bus1`: Bus at higher voltage level + +### converters.csv + +Back-to-back converters connecting non-synchronized buses. + ++ `converter_id`: unique identifier ++ `bus0`: First bus + `bus1`: Second bus diff --git a/entsoe/abstraction.sql b/entsoe/abstraction.sql index bc967e5..880cf50 100644 --- a/entsoe/abstraction.sql +++ b/entsoe/abstraction.sql @@ -1,23 +1,35 @@ begin; drop sequence if exists network_bus_id; drop table if exists station_transformer; +drop table if exists external_converter_terminal; drop table if exists station_terminal; +drop table if exists network_line; +drop table if exists network_converter; drop table if exists network_link; drop table if exists network_transformer; drop table if exists network_generator; drop table if exists network_bus; --- split stations into busses +-- split stations into buses -- insert transformers -- export to text format create table station_terminal ( station_id integer not null, voltage integer null, dc boolean not null, + converter boolean not null, network_bus_id integer primary key ); +create table external_converter_terminal ( + terminal_id integer primary key, + terminal_location geometry(point,3857), + station_id integer not null, + network_bus_id integer references station_terminal (network_bus_id), + connection_line geometry(linestring,3857) +); + create index station_terminal_idx on station_terminal (station_id, voltage, dc); create table station_transformer ( @@ -34,29 +46,61 @@ create table station_transformer ( create sequence network_bus_id; with connected_line_structures as ( - select distinct station_id, voltage, dc_line + select distinct + station_id, voltage, dc_line, + n.topology_name in ('Converter Station', + 'Converter Station Back-to-back') as converter from topology_nodes n join line_structure l on l.line_id = any(n.line_id) order by station_id, voltage ) -insert into station_terminal (station_id, voltage, dc, network_bus_id) - select station_id, voltage, dc_line, nextval('network_bus_id') +insert into station_terminal (station_id, voltage, dc, converter, network_bus_id) + select station_id, voltage, dc_line, converter, nextval('network_bus_id') from connected_line_structures; -with terminal_bridges (station_id, src_bus_id, dst_bus_id, src_voltage, dst_voltage, src_dc, dst_dc) as ( - select distinct s.station_id, s.network_bus_id, d.network_bus_id, s.voltage, d.voltage, s.dc, d.dc - from station_terminal s - join station_terminal d on s.station_id = d.station_id and s.network_bus_id < d.network_bus_id - join topology_nodes n on s.station_id = n.station_id - where n.topology_name != 'joint' +insert into external_converter_terminal (terminal_id, terminal_location, + station_id, network_bus_id, connection_line) + select distinct on (t.station_id) + s.station_id, n.station_location, t.station_id, t.network_bus_id, + st_makeline(n.station_location, n.station_location) + from station_terminal s + join topology_nodes n on s.station_id = n.station_id + join station_terminal t on s.station_id = t.station_id and s.network_bus_id <> t.network_bus_id + where s.converter and s.dc + order by t.station_id, t.voltage desc ; + +with isolated_converter_terminal as ( + select s.station_id + from station_terminal s + join topology_nodes n on s.station_id = n.station_id + where converter + group by s.station_id + having count(s.station_id) < 2 ) -insert into station_transformer (transformer_id, station_id, src_bus_id, dst_bus_id, src_voltage, dst_voltage, src_dc, dst_dc) - select nextval('line_id'), station_id, - src_bus_id, dst_bus_id, - src_voltage, dst_voltage, - src_dc, dst_dc - from terminal_bridges b; - +insert into external_converter_terminal (terminal_id, terminal_location, + station_id, network_bus_id, connection_line) + select distinct on (fs.station_id) + t.station_id, t.station_location, f.station_id, fs.network_bus_id, + st_makeline(f.station_location, t.station_location) + from isolated_converter_terminal i + join topology_nodes t on t.station_id = i.station_id + join lateral ( + select station_id, station_location from topology_nodes n + where n.station_id != t.station_id + -- not to another HVDC station, or to a joint + and n.topology_name not in ( + 'joint', + 'Wind farm', + 'Converter Station', + 'Converter Station Back-to-back' + ) + -- indexed k-nearest neighbor + order by t.station_location <-> n.station_location limit 1 + -- TODO, this distorts lengths due to projection; maybe + -- better results with geography measurements + ) f on st_distance(t.station_location, f.station_location) < :hvdc_distance + join station_terminal fs on fs.station_id = f.station_id + order by fs.station_id, fs.voltage desc ; -- exported entities create table network_bus ( @@ -66,82 +110,140 @@ create table network_bus ( dc boolean, symbol text, under_construction boolean, - tags hstore, - geometry text + tags hstore, + x numeric, + y numeric + -- geometry geometry(Point,4326) ); create table network_link ( - link_id integer primary key, - src_bus_id integer references network_bus (bus_id), - dst_bus_id integer references network_bus (bus_id), - voltage integer, - circuits integer not null, - dc boolean not null, - underground boolean not null, - under_construction boolean not null, - length_m numeric, - tags hstore, - geometry text + link_id integer primary key, + bus0 integer references network_bus (bus_id), + bus1 integer references network_bus (bus_id), + "length" numeric, + underground boolean not null, + under_construction boolean not null, + tags hstore, + geometry text + -- geometry geometry(LineString, 4326) +); + +create table network_line ( + line_id integer primary key, + bus0 integer references network_bus (bus_id), + bus1 integer references network_bus (bus_id), + voltage integer, + circuits integer not null, + "length" numeric, + underground boolean not null, + under_construction boolean not null, + tags hstore, + geometry text + -- geometry geometry(LineString, 4326) ); create table network_generator ( generator_id integer primary key, bus_id integer not null references network_bus(bus_id), - symbol text, + technology text, capacity numeric, tags hstore, geometry text + -- geometry geometry(Point, 4326) +); + +create table network_converter ( + converter_id integer primary key, + bus0 integer not null references network_bus(bus_id), + bus1 integer not null references network_bus(bus_id) + -- geometry geometry(Point, 4326) ); create table network_transformer ( transformer_id integer primary key, - symbol text, - src_bus_id integer references network_bus(bus_id), - dst_bus_id integer references network_bus(bus_id), - src_voltage integer, - dst_voltage integer, - src_dc boolean, - dst_dc boolean, - geometry text + bus0 integer references network_bus(bus_id), + bus1 integer references network_bus(bus_id) + -- geometry geometry(Point, 4326) ); -insert into network_bus (bus_id, station_id, voltage, dc, symbol, under_construction, tags, geometry) +insert into network_bus (bus_id, station_id, voltage, dc, symbol, under_construction, tags, x, y) select t.network_bus_id, t.station_id, t.voltage, t.dc, n.topology_name, p.under_construction, - p.tags, st_astext(st_transform(n.station_location, 4326)) + p.tags, + st_x(st_transform(n.station_location, 4326)), + st_y(st_transform(n.station_location, 4326)) from topology_nodes n join station_terminal t on t.station_id = n.station_id - left join station_properties p on p.station_id = n.station_id; - -insert into network_link (link_id, src_bus_id, dst_bus_id, voltage, circuits, dc, underground, under_construction, length_m, tags, geometry) + left join station_properties p on p.station_id = n.station_id + left join external_converter_terminal e on e.terminal_id = t.station_id + where e.terminal_id is null or not t.dc or n.topology_name = 'Converter Station Back-to-back'; + -- buses for which external converter terminals exist are skipped + +-- DC lines connect external terminals directly +insert into network_link (link_id, bus0, bus1, underground, under_construction, + "length", tags, geometry) + select e.line_id, + coalesce(se.network_bus_id, s.network_bus_id), + coalesce(de.network_bus_id, d.network_bus_id), + l.underground, l.under_construction, + st_length(st_transform(e.line_extent, 4326)::geography), + l.tags, st_astext(st_transform(e.line_extent, 4326)) + from topology_edges e + join line_structure l on l.line_id = e.line_id + join station_terminal s on s.station_id = e.station_id[1] and s.dc + left join external_converter_terminal se on se.terminal_id = s.station_id + join station_terminal d on d.station_id = e.station_id[2] and d.dc + left join external_converter_terminal de on de.terminal_id = d.station_id + where l.dc_line; + +-- Back-to-back Converters appear as separate converters +insert into network_converter (converter_id, bus0, bus1) + select distinct + s.station_id, s.network_bus_id, coalesce(d.network_bus_id, de.network_bus_id) + -- st_astext(st_transform(n.station_location, 4326)) + from station_terminal s + left join station_terminal d on s.station_id = d.station_id and s.network_bus_id < d.network_bus_id + left join external_converter_terminal de on de.terminal_id = s.station_id + join topology_nodes n on s.station_id = n.station_id + where s.converter + and n.topology_name = 'Converter Station Back-to-back' + and coalesce(d.station_id, de.terminal_id) is not null; + +-- AC lines +insert into network_line (line_id, bus0, bus1, voltage, circuits, + underground, under_construction, "length", tags, geometry) select e.line_id, s.network_bus_id, d.network_bus_id, - l.voltage, l.circuits, l.dc_line, l.underground, l.under_construction, + l.voltage, l.circuits, l.underground, l.under_construction, st_length(st_transform(e.line_extent, 4326)::geography), l.tags, st_astext(st_transform(e.line_extent, 4326)) from topology_edges e join line_structure l on l.line_id = e.line_id join station_terminal s on s.station_id = e.station_id[1] - and (s.voltage = l.voltage or s.voltage is null and l.voltage is null) - and s.dc = l.dc_line + and s.voltage = l.voltage and not s.dc join station_terminal d on d.station_id = e.station_id[2] - and (d.voltage = l.voltage or s.voltage is null - and l.voltage is null) and d.dc = l.dc_line; - -insert into network_generator (generator_id, bus_id, symbol, capacity, tags, geometry) + and d.voltage = l.voltage and not d.dc + where not l.dc_line; + +-- Transformers +insert into network_transformer (transformer_id, bus0, bus1) + select nextval('line_id'), + s.network_bus_id, d.network_bus_id + -- st_astext(st_transform(n.station_location, 4326)) + from station_terminal s + join station_terminal d on s.station_id = d.station_id and s.network_bus_id < d.network_bus_id + join topology_nodes n on s.station_id = n.station_id + where not s.dc and not d.dc and n.topology_name != 'joint'; + +-- Generators +insert into network_generator (generator_id, bus_id, technology, capacity, tags, geometry) select g.generator_id, - (select network_bus_id from station_terminal t - where g.station_id = t.station_id order by voltage asc limit 1), - p.tags->'symbol', (p.tags->'mw')::numeric, p.tags - array['symbol','mw'], + ( select coalesce(te.network_bus_id, t.network_bus_id) + from station_terminal t + left join external_converter_terminal te on te.terminal_id = g.station_id + where g.station_id = t.station_id order by voltage asc limit 1), + p.tags->'symbol', (p.tags->'capacity')::numeric, p.tags - array['symbol','capacity'], st_astext(st_transform(p.location, 4326)) from topology_generators g join power_generator p on p.generator_id = g.generator_id; -insert into network_transformer (transformer_id, symbol, src_bus_id, dst_bus_id, src_voltage, dst_voltage, src_dc, dst_dc, geometry) - select t.transformer_id, - case when t.src_dc = t.dst_dc then 'transformer' else 'ac/dc' end, - t.src_bus_id, t.dst_bus_id, t.src_voltage, t.dst_voltage, t.src_dc, t.dst_dc, - st_astext(st_transform(n.station_location, 4326)) - from station_transformer t - join topology_nodes n - on t.station_id = n.station_id; commit; diff --git a/entsoe/data/.gitkeep b/entsoe/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/entsoe/download.sh b/entsoe/download.sh index a3c7722..3163392 100755 --- a/entsoe/download.sh +++ b/entsoe/download.sh @@ -1,6 +1,11 @@ #!/bin/bash +ZOOM=${1-6} + npm install vt-geojson -export MapboxAccessToken=pk.eyJ1IjoicnVzdHkiLCJhIjoib0FjUkJybyJ9.V9QoXck_1Z18MhpwyIE2Og -./node_modules/vt-geojson/cli.js rusty.cm0b8gzp -z 3 > rusty.cm0b8gzp-z3.geojson -./node_modules/vt-geojson/cli.js rusty.02rit83j -z 3 > rusty.02rit83j-z3.geojson + +export MapboxAccessToken=pk.eyJ1IjoiZW50c29lIiwiYSI6ImNpbWxxYXJocDAwMG53Ymx3N2JxNGhtZDYifQ.YjNgK9usqRNrzxWXnR152g +for key in entsoe.1nr7sorj entsoe.5m43hs6u; do + echo Getting ${key} + node_modules/vt-geojson/cli.js ${key} -z ${ZOOM} > data/${key}-z${ZOOM}.geojson +done diff --git a/entsoe/electric-properties.sql b/entsoe/electric-properties.sql index 77a1b31..4bdc9d7 100644 --- a/entsoe/electric-properties.sql +++ b/entsoe/electric-properties.sql @@ -35,8 +35,8 @@ create table line_structure_conflicts ( ); insert into station_properties (station_id, symbol, name, under_construction, tags) - select power_id, properties->'symbol', properties->'name_all', (properties->'under_construction')::boolean, - properties - array['symbol','name_all','under_construction'] + select power_id, properties->'symbol', properties->'name', (properties->'under_construction')::boolean, + properties - array['symbol','name','under_construction'] from feature_points f join source_objects o on o.import_id = f.import_id and o.power_type = 's'; @@ -44,10 +44,10 @@ insert into line_structure (line_id, voltage, circuits, under_construction, unde select power_id, substring(properties->'voltagelevel' from '^[0-9]+')::int, (properties->'numberofcircuits')::int, - (properties->'underconstruction')::boolean, + (properties->'under_construction')::boolean, (properties->'underground')::boolean, (properties->'current' = 'DC'), - properties - array['voltagelevel','numberofcircuits','shape_length','underconstruction','underground','current'] + properties - array['voltagelevel','numberofcircuits','shape_length','under_construction','underground','current'] from feature_lines f join source_objects o on o.import_id = f.import_id and o.power_type = 'l'; diff --git a/entsoe/entsoe.conf b/entsoe/entsoe.conf index 6b79b0d..8d0d530 100644 --- a/entsoe/entsoe.conf +++ b/entsoe/entsoe.conf @@ -1,4 +1,4 @@ -GRIDKIT_TERMINAL_RADIUS=750 +GRIDKIT_TERMINAL_RADIUS=4000 GRIDKIT_STATION_BUFFER=50 GRIDKIT_MERGE_DISTORTION=2500 -GRIDKIT_HVDC_DISTANCE=20000 +GRIDKIT_HVDC_DISTANCE=55000 diff --git a/entsoe/export.sh b/entsoe/export.sh index 1d6a416..0c72b18 100755 --- a/entsoe/export.sh +++ b/entsoe/export.sh @@ -1,7 +1,7 @@ #!/bin/bash -psql -c "COPY network_bus TO STDOUT WITH CSV HEADER QUOTE ''''" > buses.csv -psql -c "COPY network_link TO STDOUT WITH CSV HEADER QUOTE ''''" > links.csv -psql -c "COPY network_generator TO STDOUT WITH CSV HEADER QUOTE ''''" > generators.csv -psql -c "COPY network_transformer TO STDOUT WITH CSV HEADER QUOTE ''''" > transformers.csv - -zip entsoe.zip README.md buses.csv links.csv generators.csv transformers.csv +psql -c "COPY network_bus TO STDOUT WITH CSV HEADER QUOTE ''''" > result/buses.csv +psql -c "COPY network_line TO STDOUT WITH CSV HEADER QUOTE ''''" > result/lines.csv +psql -c "COPY network_link TO STDOUT WITH CSV HEADER QUOTE ''''" > result/links.csv +psql -c "COPY network_converter TO STDOUT WITH CSV HEADER QUOTE ''''" > result/converters.csv +psql -c "COPY network_transformer TO STDOUT WITH CSV HEADER QUOTE ''''" > result/transformers.csv +psql -c "COPY network_generator TO STDOUT WITH CSV HEADER QUOTE ''''" > result/generators.csv diff --git a/entsoe/fixup-annotated-lines.sql b/entsoe/fixup-annotated-lines.sql new file mode 100644 index 0000000..305babf --- /dev/null +++ b/entsoe/fixup-annotated-lines.sql @@ -0,0 +1,46 @@ +begin; + +drop table if exists sep_annotated_lines; +create table sep_annotated_lines ( + new_id integer primary key, + old_id integer, + voltage integer, + circuits integer +); + +with annotated_lines as ( + select e.line_id, s.tags->'text_' txt + from topology_edges e + join line_structure s on s.line_id = e.line_id + where s.tags->'text_' LIKE '%+%' +) +insert into sep_annotated_lines (new_id, old_id, voltage, circuits) + select nextval('line_id'), + line_id, + substring(txt from '\d+\+(\d+)')::int, + coalesce(substring(txt from '\d+\+\d+ \((\d+)x\d+\)')::int, 1) + from annotated_lines; + +insert into derived_objects (derived_id, derived_type, operation, source_id, source_type) + select new_id, 'l', 'separate', array[old_id], 'l' +from sep_annotated_lines; + +insert into topology_edges (line_id, station_id, line_extent, topology_name) + select new_id, station_id, line_extent, topology_name + from sep_annotated_lines s + join topology_edges e on s.old_id = e.line_id; + +insert into line_structure (line_id, voltage, circuits, dc_line, + underground, under_construction, tags) + select new_id, l.voltage, l.circuits, + dc_line, underground, under_construction, tags + from sep_annotated_lines l + join line_structure s on s.line_id = l.new_id; + +update topology_nodes n + set line_id = n.line_id || e.line_id + from sep_annotated_lines l + join topology_edges e on e.line_id = l.new_id + where n.station_id = any(e.station_id); + +commit; diff --git a/entsoe/fixup-mapbox-tiles.py b/entsoe/fixup-mapbox-tiles.py new file mode 100644 index 0000000..c16eaa2 --- /dev/null +++ b/entsoe/fixup-mapbox-tiles.py @@ -0,0 +1,202 @@ +import fiona +import numpy as np +import pandas as pd +import geopandas as gpd +import geojson +from shapely.geometry import Point, LineString + +from six import iteritems +from six.moves import reduce +from itertools import chain, count, permutations +import os, sys + +type_map = dict(MultiLineString="LineString", + LineString="LineString", + Point="Point") + + +def sort_features(fn, features): + with open(fn) as fp: + t = geojson.load(fp) + + offset = {t: fs[-1]['id']+1 for t, fs in iteritems(features)} + + for n, f in enumerate(t['features']): + t = type_map[f['geometry']['type']] + f['id'] = offset.setdefault(t, 0) + n + + features.setdefault(t, []).append(f) + + return features + + +def trim_fixedid(name): + return name[:-len('-fixedid')] if name.endswith('-fixedid') else name + + +def gen_outfn(fn, suffix, tmpdir=None): + name, ext = os.path.splitext(os.path.basename(fn)) + outfn = trim_fixedid(name) + suffix + ext + + if tmpdir is not None: + outfn = os.path.join(tmpdir, outfn) + else: + outfn = os.path.join(os.path.dirname(fn), outfn) + + if os.path.exists(outfn): + os.unlink(outfn) + + return outfn + + +def stitch_tiles(fn): + df = gpd.read_file(fn) + df = df.rename(columns={'OBJECTID': 'oid', + 'ogc_fid': 'oid', + 'underconstruction': 'under_construction', + 'UnderConst': 'under_construction', + 'Symbol': 'symbol', + 'ABR': 'country', + 'VoltageLev': 'voltagelevel', + 'T9_Code': 't9_code', + 'Visible': 'visible', + 'Current_': 'current', + 'NumberOfCi': 'numberofcircuits', + 'Undergroun': 'underground', + 'Tie_line': 'tie_line', + 'Text_': 'text_', + 'LengthKm': 'shape_length'}) + df['tie_line'] = df['tie_line'].fillna(0.).astype(int) + for f in ('visible', 'underground', 'under_construction'): + df[f] = df[f].astype(int) + df['numberofcircuits'] = df['numberofcircuits'].astype(int) + df['shape_length'] = df['shape_length'].astype(float) + df['symbol'] = df['symbol'].str.split(',').str[0] + uc_b = df['symbol'].str.endswith(' Under Construction') + df.loc[uc_b, 'symbol'] = df.loc[uc_b, 'symbol'].str[:-len(' Under Construction')] + + # Break MultiLineStrings + e = (df.loc[df.type == 'MultiLineString', 'geometry']) + if not e.empty: + extra = df.drop("geometry", axis=1).join( + pd.Series(chain(*e), np.repeat(e.index, e.map(len)), name="geometry"), + how="right" + ) + df = df[df.type != 'MultiLineString'].append(extra, ignore_index=True) + + def up_to_point(l, p, other): + if l.boundary[1].distance(other) > l.boundary[0].distance(other): + l = LineString(l.coords[::-1]) + for n, r in enumerate(l.coords): + if l.project(Point(r)) > l.project(p): + return l.coords[:n] + return l.coords[:] + + def stitch_lines(a, b): + if a.buffer(1e-2).contains(b): + return a + p = a.intersection(b) + if p.is_empty: + d = a.distance(b)/(2-1e-2) + assert d < .5e-2 + p = a.buffer(d).intersection(b.buffer(d)).centroid + p = p.representative_point() + return LineString(up_to_point(a, p, b) + up_to_point(b, p, a)[::-1]) + + def unique_lines(df): + dfbuf = df.buffer(1e-2) + for i, geom in df.geometry.iteritems(): + ndfbuf = dfbuf.drop(i) + if ndfbuf.contains(geom).any(): + dfbuf = ndfbuf + return df.loc[dfbuf.index] + + def stitch_where_possible(df): + if len(df) == 1: return df.geometry.iloc[0] + + df = unique_lines(df) + + for p in permutations(df.geometry.iloc[1:]): + try: + #print("Stitching {}: {} lines".format(df.ogc_fid.iloc[0], len(df))) + return reduce(stitch_lines, p, df.geometry.iloc[0]) + except AssertionError: + pass + else: + raise Exception("Could not stitch lines with `oid = {}`".format(df.oid.iloc[0])) + + stitched = df.groupby('oid').apply(stitch_where_possible) + df = gpd.GeoDataFrame(df.groupby('oid').first().assign(geometry=stitched)).reset_index() + + outfn = gen_outfn(fn, '-fixed') + df.set_index('oid', drop=False).to_file(outfn, driver='GeoJSON') + return outfn + +def strip_duplicate_stations(fn): + with open(fn) as fp: + f = geojson.load(fp) + + key_map = {'OBJECTID': 'oid', + 'ogc_fid': 'oid', + 'Under_cons': 'under_construction', + 'name_all': 'name', + 'Name_Eng': 'name', + 'Symbol': 'symbol', + 'Country': 'country', + 'Tie_line_s': 'tie_line_s', + 'Visible': 'visible', + 'MW': 'capacity', + 'mw': 'capacity'} + seen_oids = set() + features = [] + + for feature in f['features']: + prop = feature['properties'] + for old, new in key_map.items(): + if old in prop: + prop[new] = prop.pop(old) + + feature['id'] = oid = prop['oid'] + if oid in seen_oids: + continue + seen_oids.add(oid) + + if 'symbol' in prop: + prop['symbol'] = prop['symbol'].split(',', 1)[0] + + if 'TSO' in prop: + prop['TSO'] = prop['TSO'].strip() + + if 'EIC_code' in prop: + prop['EIC_code'] = prop['EIC_code'].strip() + + features.append(feature) + + f['features'] = features + + outfn = gen_outfn(fn, '-fixed') + with open(outfn, 'w') as fp: + geojson.dump(f, fp) + return outfn + +if __name__ == '__main__': + base = sys.argv[1] + files = sys.argv[2:] + + features = dict() + for fn in files: + sort_features(fn, features) + + for geom_type, fs in iteritems(features): + fixed_id_fn = gen_outfn(base + '.geojson', '-{}-fixedid'.format(geom_type)) + with open(fixed_id_fn, 'w') as fp: + geojson.dump({'type': 'FeatureCollection', 'features': fs}, fp) + + if geom_type == 'LineString': + fixed_fn = stitch_tiles(fixed_id_fn) + print("Stitched tiles into {}.".format(fixed_fn)) + elif geom_type == 'Point': + fixed_fn = strip_duplicate_stations(fixed_id_fn) + print("Stripped station duplicates into {}.".format(fixed_fn)) + else: + print("Incompatible geometry type {} in {}.".format(geom_type, fixed_id_fn)) diff --git a/entsoe/gridkit-start.sql b/entsoe/gridkit-start.sql index 0162c1c..439b7f0 100644 --- a/entsoe/gridkit-start.sql +++ b/entsoe/gridkit-start.sql @@ -85,9 +85,7 @@ insert into power_generator (generator_id, location, tags) from feature_points where properties->'symbol' not in ( 'Substation', - 'Substation, under construction', 'Converter Station', - 'Converter Station, under construction', 'Converter Station Back-to-Back' ); diff --git a/entsoe/import.sh b/entsoe/import.sh index 6ab798e..43fbd73 100755 --- a/entsoe/import.sh +++ b/entsoe/import.sh @@ -1,3 +1,7 @@ #!/bin/bash -python ../util/geojson-to-postgis.py *.geojson +BASE=${1-entsoe} +ZOOM=${2-6} + +python fixup-mapbox-tiles.py data/${BASE} data/${BASE}*-z${ZOOM}.geojson +python ../util/geojson-to-postgis.py data/${BASE}-*-fixed.geojson diff --git a/entsoe/requirements.in b/entsoe/requirements.in new file mode 100644 index 0000000..8fb4768 --- /dev/null +++ b/entsoe/requirements.in @@ -0,0 +1,7 @@ +psycopg2-binary +fiona +pandas +geopandas +geojson +shapely +six \ No newline at end of file diff --git a/entsoe/requirements.txt b/entsoe/requirements.txt new file mode 100644 index 0000000..af14d1a --- /dev/null +++ b/entsoe/requirements.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile +# +aenum==2.2.3 # via pyproj +attrs==19.3.0 # via fiona +click-plugins==1.1.1 # via fiona +click==7.0 # via click-plugins, cligj, fiona +cligj==0.5.0 # via fiona +enum34==1.1.6 # via fiona +fiona==1.8.13 +geojson==2.5.0 +geopandas==0.6.2 +munch==2.5.0 # via fiona +numpy==1.16.6 # via pandas +pandas==0.24.2 +psycopg2-binary==2.8.4 +pyproj==2.2.2 # via geopandas +python-dateutil==2.8.1 # via pandas +pytz==2019.3 # via pandas +shapely==1.6.4.post2 +six==1.13.0 diff --git a/entsoe/result/.gitkeep b/entsoe/result/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/entsoe/run.sh b/entsoe/run.sh index f22980f..9b640b6 100755 --- a/entsoe/run.sh +++ b/entsoe/run.sh @@ -15,18 +15,18 @@ psql -f ../src/spatial-6-merge-lines.sql psql -f ../src/topology-1-connections.sql psql -f ../src/topology-2-dangling-joints.sql -psql -v merge_distortion=$GRIDKIT_MERGE_DISTORTION \ - -f ../src/topology-3-redundant-splits.sql -psql -f ../src/topology-4-redundant-joints.sql +# psql -v merge_distortion=$GRIDKIT_MERGE_DISTORTION \ +# -f ../src/topology-3-redundant-splits.sql +# psql -f ../src/topology-4-redundant-joints.sql psql -f electric-properties.sql # Fix edges which should not have been merged -psql -f fixup-merge.sql +# (only makes sense in conjunction with topology-3-redundant-splits) +#psql -f fixup-merge.sql +psql -f fixup-annotated-lines.sql -psql -f abstraction.sql -# Add transformers between DC terminals and stations psql -v hvdc_distance=$GRIDKIT_HVDC_DISTANCE \ - -f fixup-hvdc.sql + -f abstraction.sql bash ./export.sh echo "All done" diff --git a/entsoe/runall_in_docker.sh b/entsoe/runall_in_docker.sh new file mode 100755 index 0000000..5ab1648 --- /dev/null +++ b/entsoe/runall_in_docker.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +BASE=$(realpath $PWD/..) +PGUSER=postgres +PGPASSWORD=mysecretpassword + +echo +echo "Download geojson from mapbox" +echo "============================" +docker run -it -v $BASE:/app:rw -w /app/entsoe node:8.16.0-alpine sh download.sh + +echo +echo "Start postgis database server" +echo "=============================" +docker run --name gridkit-postgis -e POSTGRES_PASSWORD=$PGPASSWORD -d mdillon/postgis + +echo +echo "Import into postgis" +echo "===================" +docker run -it --link gridkit-postgis:postgres --rm \ + -v $BASE:/app:rw -w /app/entsoe \ + -e PGUSER=$PGUSER -e PGPASSWORD=$PGPASSWORD \ + python:2 \ + bash -c 'pip install -r requirements.txt && env PGHOST="$POSTGRES_PORT_5432_TCP_ADDR" PGPORT="$POSTGRES_PORT_5432_TCP_PORT" ./import.sh' + +echo +echo "Run Gridkit and export into result directory" +echo "============================================" +docker run -it --link gridkit-postgis:postgres --rm \ + -v $BASE:/app:rw -w /app/entsoe \ + -e PGUSER=$PGUSER -e PGPASSWORD=$PGPASSWORD \ + mdillon/postgis \ + bash -c 'env PGHOST="$POSTGRES_PORT_5432_TCP_ADDR" PGPORT="$POSTGRES_PORT_5432_TCP_PORT" ./run.sh' + +echo +echo "Tear down postgis server" +echo "========================" +docker stop gridkit-postgis +docker rm gridkit-postgis \ No newline at end of file diff --git a/src/spatial-3-attachment-joints.sql b/src/spatial-3-attachment-joints.sql index 3569ded..a5a72ed 100644 --- a/src/spatial-3-attachment-joints.sql +++ b/src/spatial-3-attachment-joints.sql @@ -92,7 +92,7 @@ update attached_lines and not st_dwithin(attachments, st_startpoint(old_extent), 1); update attached_lines - set new_extent = st_addpoint(new_extent, st_closestpoint(attachments, st_endpoint(new_extent)), -1) + set new_extent = st_addpoint(new_extent, st_closestpoint(attachments, st_endpoint(new_extent))) where st_contains(terminals, st_endpoint(old_extent)) and not st_dwithin(attachments, st_endpoint(old_extent), 1); diff --git a/src/spatial-5-terminal-joints.sql b/src/spatial-5-terminal-joints.sql index 553f597..1b1a2cb 100644 --- a/src/spatial-5-terminal-joints.sql +++ b/src/spatial-5-terminal-joints.sql @@ -34,11 +34,11 @@ insert into extended_lines (line_id, line_pt, old_extent, new_extent, locations) update extended_lines set new_extent = case when line_pt[1] = 1 then st_addpoint(new_extent, st_closestpoint(locations, st_startpoint(new_extent)), 0) - else st_addpoint(new_extent, st_closestpoint(locations, st_endpoint(new_extent)), -1) end; + else st_addpoint(new_extent, st_closestpoint(locations, st_endpoint(new_extent))) end; update extended_lines set new_extent = case when line_pt[2] = 1 then st_addpoint(new_extent, st_closestpoint(locations, st_startpoint(new_extent)), 0) - else st_addpoint(new_extent, st_closestpoint(locations, st_endpoint(new_extent)), -1) end + else st_addpoint(new_extent, st_closestpoint(locations, st_endpoint(new_extent))) end where array_length(line_pt, 1) = 2; insert into power_station (station_id, power_name, area)