From e6d2b33cebc815f02588adb01ec061cb5982057c Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Tue, 12 Dec 2017 13:15:15 +0100 Subject: [PATCH 01/19] entsoe: Add new map sources to download and import Also increase zoom level leading to smaller distortions --- entsoe/download.sh | 10 +++++++--- entsoe/import.sh | 5 ++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/entsoe/download.sh b/entsoe/download.sh index a3c7722..d794676 100755 --- a/entsoe/download.sh +++ b/entsoe/download.sh @@ -1,6 +1,10 @@ #!/bin/bash +ZOOM=${1-6} + npm install vt-geojson -export MapboxAccessToken=pk.eyJ1IjoicnVzdHkiLCJhIjoib0FjUkJybyJ9.V9QoXck_1Z18MhpwyIE2Og -./node_modules/vt-geojson/cli.js rusty.cm0b8gzp -z 3 > rusty.cm0b8gzp-z3.geojson -./node_modules/vt-geojson/cli.js rusty.02rit83j -z 3 > rusty.02rit83j-z3.geojson +export MapboxAccessToken=pk.eyJ1IjoiZW50c29lIiwiYSI6ImNpbWxxYXJocDAwMG53Ymx3N2JxNGhtZDYifQ.YjNgK9usqRNrzxWXnR152g +for key in rusty.cm0b8gzp rusty.02rit83j entsoe.1nr7sorj entsoe.5m43hs6u; do + echo Getting ${key} + node_modules/vt-geojson/cli.js ${key} -z ${ZOOM} > ${key}-z${ZOOM}.geojson +done diff --git a/entsoe/import.sh b/entsoe/import.sh index 6ab798e..6681173 100755 --- a/entsoe/import.sh +++ b/entsoe/import.sh @@ -1,3 +1,6 @@ #!/bin/bash -python ../util/geojson-to-postgis.py *.geojson +BASE=${1-rusty} +ZOOM=${2-6} + +python ../util/geojson-to-postgis.py ${BASE}-*-fixed.geojson From fcc1ae5bd3be3fee61a714e874465ef5a8bc9cad Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Tue, 12 Dec 2017 13:19:28 +0100 Subject: [PATCH 02/19] entsoe: Add preprocessing python script for stitching vector tiles Removes the need for late topological cleanup like lines being covered twice. --- entsoe/electric-properties.sql | 8 +- entsoe/fixup-mapbox-tiles.py | 171 +++++++++++++++++++++++++++++++++ entsoe/gridkit-start.sql | 2 - entsoe/import.sh | 1 + entsoe/run.sh | 10 +- 5 files changed, 180 insertions(+), 12 deletions(-) create mode 100644 entsoe/fixup-mapbox-tiles.py diff --git a/entsoe/electric-properties.sql b/entsoe/electric-properties.sql index 77a1b31..4bdc9d7 100644 --- a/entsoe/electric-properties.sql +++ b/entsoe/electric-properties.sql @@ -35,8 +35,8 @@ create table line_structure_conflicts ( ); insert into station_properties (station_id, symbol, name, under_construction, tags) - select power_id, properties->'symbol', properties->'name_all', (properties->'under_construction')::boolean, - properties - array['symbol','name_all','under_construction'] + select power_id, properties->'symbol', properties->'name', (properties->'under_construction')::boolean, + properties - array['symbol','name','under_construction'] from feature_points f join source_objects o on o.import_id = f.import_id and o.power_type = 's'; @@ -44,10 +44,10 @@ insert into line_structure (line_id, voltage, circuits, under_construction, unde select power_id, substring(properties->'voltagelevel' from '^[0-9]+')::int, (properties->'numberofcircuits')::int, - (properties->'underconstruction')::boolean, + (properties->'under_construction')::boolean, (properties->'underground')::boolean, (properties->'current' = 'DC'), - properties - array['voltagelevel','numberofcircuits','shape_length','underconstruction','underground','current'] + properties - array['voltagelevel','numberofcircuits','shape_length','under_construction','underground','current'] from feature_lines f join source_objects o on o.import_id = f.import_id and o.power_type = 'l'; diff --git a/entsoe/fixup-mapbox-tiles.py b/entsoe/fixup-mapbox-tiles.py new file mode 100644 index 0000000..aa7777e --- /dev/null +++ b/entsoe/fixup-mapbox-tiles.py @@ -0,0 +1,171 @@ +import fiona +import pandas as pd +import geopandas as gpd +import geojson +from shapely.geometry import Point, LineString + +from six import iteritems +from six.moves import reduce +from itertools import count, permutations +import os, sys + +type_map = dict(MultiLineString="LineString", + LineString="LineString", + Point="Point") + + +def sort_features(fn, features): + with open(fn) as fp: + t = geojson.load(fp) + + offset = {t: fs[-1]['id']+1 for t, fs in iteritems(features)} + + for n, f in enumerate(t['features']): + t = type_map[f['geometry']['type']] + f['id'] = offset.setdefault(t, 0) + n + features.setdefault(t, []).append(f) + + return features + + +def trim_fixedid(name): + return name[:-len('-fixedid')] if name.endswith('-fixedid') else name + + +def gen_outfn(fn, suffix, tmpdir=None): + name, ext = os.path.splitext(os.path.basename(fn)) + outfn = trim_fixedid(name) + suffix + ext + + if tmpdir is not None: + outfn = os.path.join(tmpdir, outfn) + + if os.path.exists(outfn): + os.unlink(outfn) + + return outfn + + +def stitch_tiles(fn): + df = gpd.read_file(fn) + df = df.rename(columns={'OBJECTID': 'oid', + 'ogc_fid': 'oid', + 'underconstruction': 'under_construction', + 'UnderConst': 'under_construction', + 'Symbol': 'symbol', + 'ABR': 'country', + 'VoltageLev': 'voltagelevel', + 'T9_Code': 't9_code', + 'Visible': 'visible', + 'Current_': 'current', + 'NumberOfCi': 'numberofcircuits', + 'Undergroun': 'underground', + 'Tie_line': 'tie_line', + 'Text_': 'text_', + 'LengthKm': 'shape_length'}) + for f in ('visible', 'underground', 'tie_line', 'under_construction'): + df[f] = df[f].astype(int) + df['numberofcircuits'] = df['numberofcircuits'].astype(int) + df['shape_length'] = df['shape_length'].astype(float) + df['symbol'] = df['symbol'].str.split(',').str[0] + uc_b = df['symbol'].str.endswith(' Under Construction') + df.loc[uc_b, 'symbol'] = df.loc[uc_b, 'symbol'].str[:-len(' Under Construction')] + + # Break MultiLineStrings + extra = [] + for i in count(): + e = (df[df.type == 'MultiLineString'] + .assign(geometry=lambda df: df.geometry.str[i]) + .dropna(subset=['geometry'])) + if e.empty: break + extra.append(e) + df = df[df.type != 'MultiLineString'].append(pd.concat(extra), ignore_index=True) + + def up_to_point(l, p, other): + if l.boundary[1].distance(other) > l.boundary[0].distance(other): + l = LineString(l.coords[::-1]) + for n, r in enumerate(l.coords): + if l.project(Point(r)) > l.project(p): + return l.coords[:n] + return l.coords[:] + + def stitch_lines(a, b): + if a.buffer(1e-2).contains(b): + return a + if a.boundary.intersects(b.boundary): + return a if a.length > b.length else b + p = a.intersection(b) + if p.is_empty: + d = a.distance(b)/(2-1e-2) + assert d < 1e-3 + p = a.buffer(d).intersection(b.buffer(d)).centroid + p = p.representative_point() + return LineString(up_to_point(a, p, b) + up_to_point(b, p, a)[::-1]) + + def unique_lines(df): + dfbuf = df.buffer(1e-2) + for i, geom in df.geometry.iteritems(): + ndfbuf = dfbuf.drop(i) + if ndfbuf.contains(geom).any(): + dfbuf = ndfbuf + return df.loc[dfbuf.index] + + def stitch_where_possible(df): + if len(df) == 1: return df.geometry.iloc[0] + + df = unique_lines(df) + + for p in permutations(df.geometry.iloc[1:]): + try: + #print("Stitching {}: {} lines".format(df.ogc_fid.iloc[0], len(df))) + return reduce(stitch_lines, p, df.geometry.iloc[0]) + except AssertionError: + pass + + stitched = df.groupby('oid').apply(stitch_where_possible) + df = gpd.GeoDataFrame(df.groupby('oid').first().assign(geometry=stitched)).reset_index() + + outfn = gen_outfn(fn, '-fixed') + df.set_index('oid', drop=False).to_file(outfn, driver='GeoJSON') + return outfn + +def strip_duplicate_stations(fn): + df = gpd.read_file(fn) + df = df.rename(columns={'OBJECTID': 'oid', + 'ogc_fid': 'oid', + 'Under_cons': 'under_construction', + 'name_all': 'name', + 'Name_Eng': 'name', + 'Symbol': 'symbol', + 'Country': 'country', + 'Tie_line_s': 'tie_line_s', + 'Visible': 'visible', + 'MW': 'capacity', + 'mw': 'capacity'}) + + outfn = gen_outfn(fn, '-fixed') + (df[~df.duplicated(subset=['oid'], keep='first')] + .set_index('oid', drop=False) + .to_file(outfn, driver='GeoJSON')) + return outfn + +if __name__ == '__main__': + base = sys.argv[1] + files = sys.argv[2:] + + features = dict() + for fn in files: + sort_features(fn, features) + + for geom_type, fs in iteritems(features): + fixed_id_fn = gen_outfn(base + '.geojson', '-{}-fixedid'.format(geom_type)) + with open(fixed_id_fn, 'w') as fp: + geojson.dump({'type': 'FeatureCollection', 'features': fs}, fp) + + if geom_type == 'LineString': + fixed_fn = stitch_tiles(fixed_id_fn) + print("Stitched tiles into {}.".format(fixed_fn)) + elif geom_type == 'Point': + fixed_fn = strip_duplicate_stations(fixed_id_fn) + print("Stripped station duplicates into {}.".format(fixed_fn)) + else: + print("Incompatible geometry type {} in {}.".format(geom_type, fixed_id_fn)) diff --git a/entsoe/gridkit-start.sql b/entsoe/gridkit-start.sql index 0162c1c..439b7f0 100644 --- a/entsoe/gridkit-start.sql +++ b/entsoe/gridkit-start.sql @@ -85,9 +85,7 @@ insert into power_generator (generator_id, location, tags) from feature_points where properties->'symbol' not in ( 'Substation', - 'Substation, under construction', 'Converter Station', - 'Converter Station, under construction', 'Converter Station Back-to-Back' ); diff --git a/entsoe/import.sh b/entsoe/import.sh index 6681173..8be1cd9 100755 --- a/entsoe/import.sh +++ b/entsoe/import.sh @@ -3,4 +3,5 @@ BASE=${1-rusty} ZOOM=${2-6} +python fixup-mapbox-tiles.py ${BASE} ${BASE}*-z${ZOOM}.geojson python ../util/geojson-to-postgis.py ${BASE}-*-fixed.geojson diff --git a/entsoe/run.sh b/entsoe/run.sh index f22980f..b8af8f3 100755 --- a/entsoe/run.sh +++ b/entsoe/run.sh @@ -15,18 +15,16 @@ psql -f ../src/spatial-6-merge-lines.sql psql -f ../src/topology-1-connections.sql psql -f ../src/topology-2-dangling-joints.sql -psql -v merge_distortion=$GRIDKIT_MERGE_DISTORTION \ - -f ../src/topology-3-redundant-splits.sql -psql -f ../src/topology-4-redundant-joints.sql +# psql -v merge_distortion=$GRIDKIT_MERGE_DISTORTION \ +# -f ../src/topology-3-redundant-splits.sql +# psql -f ../src/topology-4-redundant-joints.sql psql -f electric-properties.sql # Fix edges which should not have been merged psql -f fixup-merge.sql -psql -f abstraction.sql -# Add transformers between DC terminals and stations psql -v hvdc_distance=$GRIDKIT_HVDC_DISTANCE \ - -f fixup-hvdc.sql + -f abstraction.sql bash ./export.sh echo "All done" From 1393c1745ccd8665dda08dc6334e2b3c94324c4c Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Tue, 12 Dec 2017 13:24:59 +0100 Subject: [PATCH 03/19] entsoe: Sync abstraction table structure with pypsa needs --- entsoe/abstraction.sql | 227 ++++++++++++++++++++++++++++------------- entsoe/entsoe.conf | 4 +- 2 files changed, 160 insertions(+), 71 deletions(-) diff --git a/entsoe/abstraction.sql b/entsoe/abstraction.sql index bc967e5..9b9cf5d 100644 --- a/entsoe/abstraction.sql +++ b/entsoe/abstraction.sql @@ -1,23 +1,35 @@ begin; drop sequence if exists network_bus_id; drop table if exists station_transformer; +drop table if exists external_converter_terminal; drop table if exists station_terminal; +drop table if exists network_line; +drop table if exists network_converter; drop table if exists network_link; drop table if exists network_transformer; drop table if exists network_generator; drop table if exists network_bus; --- split stations into busses +-- split stations into buses -- insert transformers -- export to text format create table station_terminal ( station_id integer not null, voltage integer null, dc boolean not null, + converter boolean not null, network_bus_id integer primary key ); +create table external_converter_terminal ( + terminal_id integer primary key, + terminal_location geometry(point,3857), + station_id integer not null, + network_bus_id integer references station_terminal (network_bus_id), + connection_line geometry(linestring,3857) +); + create index station_terminal_idx on station_terminal (station_id, voltage, dc); create table station_transformer ( @@ -34,29 +46,50 @@ create table station_transformer ( create sequence network_bus_id; with connected_line_structures as ( - select distinct station_id, voltage, dc_line + select distinct + station_id, voltage, dc_line, + n.topology_name in ('Converter Station', + 'Converter Station Back-to-back') as converter from topology_nodes n join line_structure l on l.line_id = any(n.line_id) order by station_id, voltage ) -insert into station_terminal (station_id, voltage, dc, network_bus_id) - select station_id, voltage, dc_line, nextval('network_bus_id') +insert into station_terminal (station_id, voltage, dc, converter, network_bus_id) + select station_id, voltage, dc_line, converter, nextval('network_bus_id') from connected_line_structures; -with terminal_bridges (station_id, src_bus_id, dst_bus_id, src_voltage, dst_voltage, src_dc, dst_dc) as ( - select distinct s.station_id, s.network_bus_id, d.network_bus_id, s.voltage, d.voltage, s.dc, d.dc - from station_terminal s - join station_terminal d on s.station_id = d.station_id and s.network_bus_id < d.network_bus_id - join topology_nodes n on s.station_id = n.station_id - where n.topology_name != 'joint' +with isolated_converter_terminal as ( + select s.station_id + from station_terminal s + join topology_nodes n on s.station_id = n.station_id + where converter + group by s.station_id + having count(s.station_id) < 2 ) -insert into station_transformer (transformer_id, station_id, src_bus_id, dst_bus_id, src_voltage, dst_voltage, src_dc, dst_dc) - select nextval('line_id'), station_id, - src_bus_id, dst_bus_id, - src_voltage, dst_voltage, - src_dc, dst_dc - from terminal_bridges b; - +insert into external_converter_terminal (terminal_id, terminal_location, + station_id, network_bus_id, connection_line) + select distinct on (fs.station_id) + t.station_id, t.station_location, f.station_id, fs.network_bus_id, + st_makeline(f.station_location, t.station_location) + from isolated_converter_terminal i + join topology_nodes t on t.station_id = i.station_id + join lateral ( + select station_id, station_location from topology_nodes n + where n.station_id != t.station_id + -- not to another HVDC station, or to a joint + and n.topology_name not in ( + 'joint', + 'Wind farm', + 'Converter Station', + 'Converter Station Back-to-back' + ) + -- indexed k-nearest neighbor + order by t.station_location <-> n.station_location limit 1 + -- TODO, this distorts lengths due to projection; maybe + -- better results with geography measurements + ) f on st_distance(t.station_location, f.station_location) < :hvdc_distance + join station_terminal fs on fs.station_id = f.station_id + order by fs.station_id, fs.voltage desc ; -- exported entities create table network_bus ( @@ -66,82 +99,138 @@ create table network_bus ( dc boolean, symbol text, under_construction boolean, - tags hstore, - geometry text + tags hstore, + x numeric, + y numeric + -- geometry geometry(Point,4326) ); create table network_link ( - link_id integer primary key, - src_bus_id integer references network_bus (bus_id), - dst_bus_id integer references network_bus (bus_id), - voltage integer, - circuits integer not null, - dc boolean not null, - underground boolean not null, - under_construction boolean not null, - length_m numeric, - tags hstore, - geometry text + link_id integer primary key, + bus0 integer references network_bus (bus_id), + bus1 integer references network_bus (bus_id), + "length" numeric, + underground boolean not null, + under_construction boolean not null, + tags hstore, + geometry text + -- geometry geometry(LineString, 4326) +); + +create table network_line ( + line_id integer primary key, + bus0 integer references network_bus (bus_id), + bus1 integer references network_bus (bus_id), + voltage integer, + circuits integer not null, + "length" numeric, + underground boolean not null, + under_construction boolean not null, + tags hstore, + geometry text + -- geometry geometry(LineString, 4326) ); create table network_generator ( generator_id integer primary key, bus_id integer not null references network_bus(bus_id), - symbol text, + technology text, capacity numeric, - tags hstore, - geometry text + tags hstore + -- geometry geometry(Point, 4326) +); + +create table network_converter ( + converter_id integer primary key, + bus0 integer not null references network_bus(bus_id), + bus1 integer not null references network_bus(bus_id) + -- geometry geometry(Point, 4326) ); create table network_transformer ( transformer_id integer primary key, - symbol text, - src_bus_id integer references network_bus(bus_id), - dst_bus_id integer references network_bus(bus_id), - src_voltage integer, - dst_voltage integer, - src_dc boolean, - dst_dc boolean, - geometry text + bus0 integer references network_bus(bus_id), + bus1 integer references network_bus(bus_id) + -- geometry geometry(Point, 4326) ); -insert into network_bus (bus_id, station_id, voltage, dc, symbol, under_construction, tags, geometry) +insert into network_bus (bus_id, station_id, voltage, dc, symbol, under_construction, tags, x, y) select t.network_bus_id, t.station_id, t.voltage, t.dc, n.topology_name, p.under_construction, - p.tags, st_astext(st_transform(n.station_location, 4326)) + p.tags, + st_x(st_transform(n.station_location, 4326)), + st_y(st_transform(n.station_location, 4326)) from topology_nodes n join station_terminal t on t.station_id = n.station_id - left join station_properties p on p.station_id = n.station_id; - -insert into network_link (link_id, src_bus_id, dst_bus_id, voltage, circuits, dc, underground, under_construction, length_m, tags, geometry) + left join station_properties p on p.station_id = n.station_id + left join external_converter_terminal e on e.terminal_id = t.station_id + where e.terminal_id is null or n.topology_name = 'Converter Station Back-to-back'; + -- buses for which external converter terminals exist are skipped + +-- DC lines connect external terminals directly +insert into network_link (link_id, bus0, bus1, underground, under_construction, + "length", tags, geometry) + select e.line_id, + coalesce(se.network_bus_id, s.network_bus_id), + coalesce(de.network_bus_id, d.network_bus_id), + l.underground, l.under_construction, + st_length(st_transform(e.line_extent, 4326)::geography), + l.tags, st_astext(st_transform(e.line_extent, 4326)) + from topology_edges e + join line_structure l on l.line_id = e.line_id + join station_terminal s on s.station_id = e.station_id[1] and s.dc + left join external_converter_terminal se on se.terminal_id = s.station_id + join station_terminal d on d.station_id = e.station_id[2] and d.dc + left join external_converter_terminal de on de.terminal_id = d.station_id + where l.dc_line; + +-- Back-to-back Converters appear as separate converters +insert into network_converter (converter_id, bus0, bus1) + select distinct + s.station_id, s.network_bus_id, coalesce(d.network_bus_id, de.network_bus_id) + -- st_astext(st_transform(n.station_location, 4326)) + from station_terminal s + left join station_terminal d on s.station_id = d.station_id and s.network_bus_id < d.network_bus_id + left join external_converter_terminal de on de.terminal_id = s.station_id + join topology_nodes n on s.station_id = n.station_id + where s.converter + and n.topology_name = 'Converter Station Back-to-back' + and coalesce(d.station_id, de.terminal_id) is not null; + +-- AC lines +insert into network_line (line_id, bus0, bus1, voltage, circuits, + underground, under_construction, "length", tags, geometry) select e.line_id, s.network_bus_id, d.network_bus_id, - l.voltage, l.circuits, l.dc_line, l.underground, l.under_construction, + l.voltage, l.circuits, l.underground, l.under_construction, st_length(st_transform(e.line_extent, 4326)::geography), l.tags, st_astext(st_transform(e.line_extent, 4326)) from topology_edges e join line_structure l on l.line_id = e.line_id join station_terminal s on s.station_id = e.station_id[1] - and (s.voltage = l.voltage or s.voltage is null and l.voltage is null) - and s.dc = l.dc_line + and s.voltage = l.voltage and not s.dc join station_terminal d on d.station_id = e.station_id[2] - and (d.voltage = l.voltage or s.voltage is null - and l.voltage is null) and d.dc = l.dc_line; - -insert into network_generator (generator_id, bus_id, symbol, capacity, tags, geometry) - select g.generator_id, - (select network_bus_id from station_terminal t - where g.station_id = t.station_id order by voltage asc limit 1), - p.tags->'symbol', (p.tags->'mw')::numeric, p.tags - array['symbol','mw'], - st_astext(st_transform(p.location, 4326)) - from topology_generators g - join power_generator p on p.generator_id = g.generator_id; - -insert into network_transformer (transformer_id, symbol, src_bus_id, dst_bus_id, src_voltage, dst_voltage, src_dc, dst_dc, geometry) - select t.transformer_id, - case when t.src_dc = t.dst_dc then 'transformer' else 'ac/dc' end, - t.src_bus_id, t.dst_bus_id, t.src_voltage, t.dst_voltage, t.src_dc, t.dst_dc, - st_astext(st_transform(n.station_location, 4326)) - from station_transformer t - join topology_nodes n - on t.station_id = n.station_id; + and d.voltage = l.voltage and not d.dc + where not l.dc_line; + +-- Transformers +insert into network_transformer (transformer_id, bus0, bus1) + select nextval('line_id'), + s.network_bus_id, d.network_bus_id + -- st_astext(st_transform(n.station_location, 4326)) + from station_terminal s + join station_terminal d on s.station_id = d.station_id and s.network_bus_id < d.network_bus_id + join topology_nodes n on s.station_id = n.station_id + where not s.dc and not d.dc and n.topology_name != 'joint'; + +-- Generators +-- insert into network_generator (generator_id, bus_id, technology, capacity, tags, geometry) +-- select g.generator_id, +-- (select network_bus_id from station_terminal t +-- where g.station_id = t.station_id order by voltage asc limit 1), +-- p.tags->'symbol', (p.tags->'mw')::numeric, p.tags - array['symbol','mw'], +-- st_transform(p.location, 4326) +-- from topology_generators g +-- join power_generator p on p.generator_id = g.generator_id; + + commit; diff --git a/entsoe/entsoe.conf b/entsoe/entsoe.conf index 6b79b0d..8d0d530 100644 --- a/entsoe/entsoe.conf +++ b/entsoe/entsoe.conf @@ -1,4 +1,4 @@ -GRIDKIT_TERMINAL_RADIUS=750 +GRIDKIT_TERMINAL_RADIUS=4000 GRIDKIT_STATION_BUFFER=50 GRIDKIT_MERGE_DISTORTION=2500 -GRIDKIT_HVDC_DISTANCE=20000 +GRIDKIT_HVDC_DISTANCE=55000 From 9909796174c2045225a4556087c7a296f5e3af51 Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Tue, 12 Dec 2017 13:26:27 +0100 Subject: [PATCH 04/19] entsoe: Split annotated lines into separate circuits Some lines have a text_ attribute like 380+220 signifying there is an extra 220kV circuit on the transmission line, which has to be treated separately. --- entsoe/fixup-annotated-lines.sql | 46 ++++++++++++++++++++++++++++++++ entsoe/run.sh | 1 + 2 files changed, 47 insertions(+) create mode 100644 entsoe/fixup-annotated-lines.sql diff --git a/entsoe/fixup-annotated-lines.sql b/entsoe/fixup-annotated-lines.sql new file mode 100644 index 0000000..305babf --- /dev/null +++ b/entsoe/fixup-annotated-lines.sql @@ -0,0 +1,46 @@ +begin; + +drop table if exists sep_annotated_lines; +create table sep_annotated_lines ( + new_id integer primary key, + old_id integer, + voltage integer, + circuits integer +); + +with annotated_lines as ( + select e.line_id, s.tags->'text_' txt + from topology_edges e + join line_structure s on s.line_id = e.line_id + where s.tags->'text_' LIKE '%+%' +) +insert into sep_annotated_lines (new_id, old_id, voltage, circuits) + select nextval('line_id'), + line_id, + substring(txt from '\d+\+(\d+)')::int, + coalesce(substring(txt from '\d+\+\d+ \((\d+)x\d+\)')::int, 1) + from annotated_lines; + +insert into derived_objects (derived_id, derived_type, operation, source_id, source_type) + select new_id, 'l', 'separate', array[old_id], 'l' +from sep_annotated_lines; + +insert into topology_edges (line_id, station_id, line_extent, topology_name) + select new_id, station_id, line_extent, topology_name + from sep_annotated_lines s + join topology_edges e on s.old_id = e.line_id; + +insert into line_structure (line_id, voltage, circuits, dc_line, + underground, under_construction, tags) + select new_id, l.voltage, l.circuits, + dc_line, underground, under_construction, tags + from sep_annotated_lines l + join line_structure s on s.line_id = l.new_id; + +update topology_nodes n + set line_id = n.line_id || e.line_id + from sep_annotated_lines l + join topology_edges e on e.line_id = l.new_id + where n.station_id = any(e.station_id); + +commit; diff --git a/entsoe/run.sh b/entsoe/run.sh index b8af8f3..10fd7a6 100755 --- a/entsoe/run.sh +++ b/entsoe/run.sh @@ -22,6 +22,7 @@ psql -f ../src/topology-2-dangling-joints.sql psql -f electric-properties.sql # Fix edges which should not have been merged psql -f fixup-merge.sql +psql -f fixup-annotated-lines.sql psql -v hvdc_distance=$GRIDKIT_HVDC_DISTANCE \ -f abstraction.sql From 8df226b71d5a1a8c786d176719a38e5970c054bc Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Tue, 12 Dec 2017 13:57:23 +0100 Subject: [PATCH 05/19] entsoe: Export all new tables as csv files --- entsoe/export.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/entsoe/export.sh b/entsoe/export.sh index 1d6a416..345f7e5 100755 --- a/entsoe/export.sh +++ b/entsoe/export.sh @@ -1,7 +1,9 @@ #!/bin/bash psql -c "COPY network_bus TO STDOUT WITH CSV HEADER QUOTE ''''" > buses.csv +psql -c "COPY network_line TO STDOUT WITH CSV HEADER QUOTE ''''" > lines.csv psql -c "COPY network_link TO STDOUT WITH CSV HEADER QUOTE ''''" > links.csv -psql -c "COPY network_generator TO STDOUT WITH CSV HEADER QUOTE ''''" > generators.csv +psql -c "COPY network_converter TO STDOUT WITH CSV HEADER QUOTE ''''" > converters.csv psql -c "COPY network_transformer TO STDOUT WITH CSV HEADER QUOTE ''''" > transformers.csv +# psql -c "COPY network_generator TO STDOUT WITH CSV HEADER QUOTE ''''" > generators.csv -zip entsoe.zip README.md buses.csv links.csv generators.csv transformers.csv +zip entsoe.zip README.md buses.csv lines.csv links.csv converters.csv transformers.csv generators.csv From 5d1c9c73522736898416956b2ed52a4024cc225f Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Tue, 12 Dec 2017 14:20:32 +0100 Subject: [PATCH 06/19] entsoe: Remove under construction annotation in python script --- entsoe/fixup-mapbox-tiles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/entsoe/fixup-mapbox-tiles.py b/entsoe/fixup-mapbox-tiles.py index aa7777e..867fbed 100644 --- a/entsoe/fixup-mapbox-tiles.py +++ b/entsoe/fixup-mapbox-tiles.py @@ -141,6 +141,7 @@ def strip_duplicate_stations(fn): 'Visible': 'visible', 'MW': 'capacity', 'mw': 'capacity'}) + df['symbol'] = df['symbol'].str.split(',').str[0] outfn = gen_outfn(fn, '-fixed') (df[~df.duplicated(subset=['oid'], keep='first')] From cb58b89c106a229d501fc52bc0aaa4d8ac9ab134 Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Thu, 14 Dec 2017 14:33:10 +0100 Subject: [PATCH 07/19] entsoe: Fix skipping converters --- entsoe/abstraction.sql | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/entsoe/abstraction.sql b/entsoe/abstraction.sql index 9b9cf5d..c1ced20 100644 --- a/entsoe/abstraction.sql +++ b/entsoe/abstraction.sql @@ -58,6 +58,17 @@ insert into station_terminal (station_id, voltage, dc, converter, network_bus_id select station_id, voltage, dc_line, converter, nextval('network_bus_id') from connected_line_structures; +insert into external_converter_terminal (terminal_id, terminal_location, + station_id, network_bus_id, connection_line) + select distinct on (t.station_id) + s.station_id, n.station_location, t.station_id, t.network_bus_id, + st_makeline(n.station_location, n.station_location) + from station_terminal s + join topology_nodes n on s.station_id = n.station_id + join station_terminal t on s.station_id = t.station_id and s.network_bus_id <> t.network_bus_id + where s.converter and s.dc + order by t.station_id, t.voltage desc ; + with isolated_converter_terminal as ( select s.station_id from station_terminal s @@ -163,7 +174,7 @@ insert into network_bus (bus_id, station_id, voltage, dc, symbol, under_construc join station_terminal t on t.station_id = n.station_id left join station_properties p on p.station_id = n.station_id left join external_converter_terminal e on e.terminal_id = t.station_id - where e.terminal_id is null or n.topology_name = 'Converter Station Back-to-back'; + where e.terminal_id is null or not t.dc or n.topology_name = 'Converter Station Back-to-back'; -- buses for which external converter terminals exist are skipped -- DC lines connect external terminals directly From c8aeaa10217722eaa9b7efe4f35c28f31c248c9e Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Sat, 26 May 2018 09:23:21 +0200 Subject: [PATCH 08/19] entsoe: Include generators in abstraction process --- entsoe/abstraction.sql | 28 +++++++++++++++------------- entsoe/export.sh | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/entsoe/abstraction.sql b/entsoe/abstraction.sql index c1ced20..880cf50 100644 --- a/entsoe/abstraction.sql +++ b/entsoe/abstraction.sql @@ -23,10 +23,10 @@ create table station_terminal ( ); create table external_converter_terminal ( - terminal_id integer primary key, + terminal_id integer primary key, terminal_location geometry(point,3857), - station_id integer not null, - network_bus_id integer references station_terminal (network_bus_id), + station_id integer not null, + network_bus_id integer references station_terminal (network_bus_id), connection_line geometry(linestring,3857) ); @@ -147,7 +147,8 @@ create table network_generator ( bus_id integer not null references network_bus(bus_id), technology text, capacity numeric, - tags hstore + tags hstore, + geometry text -- geometry geometry(Point, 4326) ); @@ -233,15 +234,16 @@ insert into network_transformer (transformer_id, bus0, bus1) where not s.dc and not d.dc and n.topology_name != 'joint'; -- Generators --- insert into network_generator (generator_id, bus_id, technology, capacity, tags, geometry) --- select g.generator_id, --- (select network_bus_id from station_terminal t --- where g.station_id = t.station_id order by voltage asc limit 1), --- p.tags->'symbol', (p.tags->'mw')::numeric, p.tags - array['symbol','mw'], --- st_transform(p.location, 4326) --- from topology_generators g --- join power_generator p on p.generator_id = g.generator_id; - +insert into network_generator (generator_id, bus_id, technology, capacity, tags, geometry) + select g.generator_id, + ( select coalesce(te.network_bus_id, t.network_bus_id) + from station_terminal t + left join external_converter_terminal te on te.terminal_id = g.station_id + where g.station_id = t.station_id order by voltage asc limit 1), + p.tags->'symbol', (p.tags->'capacity')::numeric, p.tags - array['symbol','capacity'], + st_astext(st_transform(p.location, 4326)) + from topology_generators g + join power_generator p on p.generator_id = g.generator_id; commit; diff --git a/entsoe/export.sh b/entsoe/export.sh index 345f7e5..6cd6520 100755 --- a/entsoe/export.sh +++ b/entsoe/export.sh @@ -4,6 +4,6 @@ psql -c "COPY network_line TO STDOUT WITH CSV HEADER QUOTE ''''" > lines.csv psql -c "COPY network_link TO STDOUT WITH CSV HEADER QUOTE ''''" > links.csv psql -c "COPY network_converter TO STDOUT WITH CSV HEADER QUOTE ''''" > converters.csv psql -c "COPY network_transformer TO STDOUT WITH CSV HEADER QUOTE ''''" > transformers.csv -# psql -c "COPY network_generator TO STDOUT WITH CSV HEADER QUOTE ''''" > generators.csv +psql -c "COPY network_generator TO STDOUT WITH CSV HEADER QUOTE ''''" > generators.csv zip entsoe.zip README.md buses.csv lines.csv links.csv converters.csv transformers.csv generators.csv From a505f60feaf1d29a179f7ef0757bb5821f31b754 Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Sat, 26 May 2018 09:24:03 +0200 Subject: [PATCH 09/19] entsoe: Adapt parameters of tile stitching to new entsoe map --- entsoe/fixup-mapbox-tiles.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/entsoe/fixup-mapbox-tiles.py b/entsoe/fixup-mapbox-tiles.py index 867fbed..ed416c6 100644 --- a/entsoe/fixup-mapbox-tiles.py +++ b/entsoe/fixup-mapbox-tiles.py @@ -23,6 +23,7 @@ def sort_features(fn, features): for n, f in enumerate(t['features']): t = type_map[f['geometry']['type']] f['id'] = offset.setdefault(t, 0) + n + features.setdefault(t, []).append(f) return features @@ -62,7 +63,8 @@ def stitch_tiles(fn): 'Tie_line': 'tie_line', 'Text_': 'text_', 'LengthKm': 'shape_length'}) - for f in ('visible', 'underground', 'tie_line', 'under_construction'): + df['tie_line'] = df['tie_line'].fillna(0.).astype(int) + for f in ('visible', 'underground', 'under_construction'): df[f] = df[f].astype(int) df['numberofcircuits'] = df['numberofcircuits'].astype(int) df['shape_length'] = df['shape_length'].astype(float) @@ -91,12 +93,10 @@ def up_to_point(l, p, other): def stitch_lines(a, b): if a.buffer(1e-2).contains(b): return a - if a.boundary.intersects(b.boundary): - return a if a.length > b.length else b p = a.intersection(b) if p.is_empty: d = a.distance(b)/(2-1e-2) - assert d < 1e-3 + assert d < .5e-2 p = a.buffer(d).intersection(b.buffer(d)).centroid p = p.representative_point() return LineString(up_to_point(a, p, b) + up_to_point(b, p, a)[::-1]) @@ -120,6 +120,8 @@ def stitch_where_possible(df): return reduce(stitch_lines, p, df.geometry.iloc[0]) except AssertionError: pass + else: + raise Exception("Could not stitch lines with `oid = {}`".format(df.oid.iloc[0])) stitched = df.groupby('oid').apply(stitch_where_possible) df = gpd.GeoDataFrame(df.groupby('oid').first().assign(geometry=stitched)).reset_index() From e5f753601eff66d48bd86c67d50fd1400bc017a2 Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Sat, 26 May 2018 09:26:08 +0200 Subject: [PATCH 10/19] entsoe: Use new entsoe mapbox account as default source --- entsoe/download.sh | 1 + entsoe/import.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/entsoe/download.sh b/entsoe/download.sh index d794676..0669381 100755 --- a/entsoe/download.sh +++ b/entsoe/download.sh @@ -3,6 +3,7 @@ ZOOM=${1-6} npm install vt-geojson + export MapboxAccessToken=pk.eyJ1IjoiZW50c29lIiwiYSI6ImNpbWxxYXJocDAwMG53Ymx3N2JxNGhtZDYifQ.YjNgK9usqRNrzxWXnR152g for key in rusty.cm0b8gzp rusty.02rit83j entsoe.1nr7sorj entsoe.5m43hs6u; do echo Getting ${key} diff --git a/entsoe/import.sh b/entsoe/import.sh index 8be1cd9..dfb572e 100755 --- a/entsoe/import.sh +++ b/entsoe/import.sh @@ -1,6 +1,6 @@ #!/bin/bash -BASE=${1-rusty} +BASE=${1-entsoe} ZOOM=${2-6} python fixup-mapbox-tiles.py ${BASE} ${BASE}*-z${ZOOM}.geojson From d7e1693a1981497ad369207fa1079f5920ff780f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 26 Aug 2018 14:02:32 +0200 Subject: [PATCH 11/19] entsoe: Update README to reflect new data structure --- entsoe/README.md | 84 +++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/entsoe/README.md b/entsoe/README.md index 771cd58..dbf9bb8 100644 --- a/entsoe/README.md +++ b/entsoe/README.md @@ -1,8 +1,8 @@ # Unofficial ENTSO-E dataset processed by GridKit -This dataset was generated based on a map extract from May 11, 2016. +This dataset was generated based on a map extract from May 25, 2018. This is an _unofficial_ extract of the -[ENTSO-E interactive map](https://www.entsoe.eu/map/Pages/default.aspx) +[ENTSO-E interactive map](https://www.entsoe.eu/data/map/) of the European power system (including to a limited extent North Africa and the Middle East). The dataset has been processed by GridKit to form complete topological connections. This dataset is neither @@ -14,12 +14,11 @@ This dataset may be inaccurate in several ways, notably: is known to choose topological clarity over geographical accuracy. Hence coordinates will not correspond exactly to reality. + Voltage levels are typically provided as ranges by ENTSO-E, of which - the lower bound has been reported in this dataset. Not all lines - - especially DC lines - contain voltage information. + the lower bound has been reported in this dataset. + Line structure conflicts are resolved by picking the first structure in the set + Transformers are _not present_ in the original ENTSO-E dataset, - there presence has been derived from the different voltages from + their presence has been derived from the different voltages from connected lines. + The connection between generators and busses is derived as the geographically nearest station at the lowest voltage level. This @@ -29,7 +28,6 @@ All users are advised to exercise caution in the use of this dataset. No liability is taken for inaccuracies. - ## Contents of dataset This dataset is provided as set of CSV files that describe the ENTSO-E @@ -53,39 +51,51 @@ Example code for reading the files: Describes terminals, vertices, or 'nodes' of the system + `bus_id`: the unique identifier for the bus -+ `station_id`: the substation or plant of this; a station may have - multiple buses, which are typically connected by transformers ++ `station_id`: unique identifier of its substation; a station may have multiple buses, which are typically connected by transformers + `voltage`: the operating voltage of this bus + `dc`: boolean ('t' or 'f'), describes whether the bus is a HVDC terminal (t) or a regular AC terminal (f) + `symbol`: type of station of this bus. ++ `under_construction`: boolean ('t' if station is currently under construction, + 'f' otherwise) + `tags`: _hstore_ encoded dictionary of 'extra' properties for this bus -+ `under_construction`: boolean ('t' if station is currently under - construction, 'f' otherwise) -+ `geometry`: location of the station in well-known-text format (WGS84) ++ `x`: longitude of its location ++ `y`: latitude of its location **NOTA BENE**: During the processing of the network, so called 'synthetic' stations may be inserted on locations where lines are apparantly connected. Such synthetic stations can be recognised because their symbol is always `joint`. +### lines.csv: + +Buses are connected by AC-lines: + ++ `line_id`: unique identifier for the line ++ `bus0`: first of the two connected buses ++ `bus1`: second of two connected buses ++ `voltage`: operating voltage of the line (identical to operating voltage of + the bus) ++ `circuits`: number of (independent) circuits in this link, each of which + typically has 3 cables. ++ `length`: length of line in km ++ `underground`: boolean, `t` if this is an underground cable, `f` for + an overhead line ++ `under_construction`: boolean, `t` for lines that are currently + under construction ++ `tags`: _hstore_ encoded dictionary of extra properties for this link ++ `geometry`: extent of this line in well-known-text format (WGS84) + ### links.csv: Connections between buses: + `link_id`: unique identifier for the link -+ `src_bus_id`: first of the two connected buses -+ `dst_bus_id`: second of two connected buses -+ `voltage`: operating voltage of the link (_must_ be identical to - operating voltage of the buses) -+ `circuits`: number of (independent) circuits in this link, each of - which typically has 3 cables (for AC lines). -+ `dc`: boolean, `t` if this is a HVDC line -+ `underground`: boolean, `t` if this is an underground cable, `f` for - an overhead line ++ `bus0`: first of the two connected buses ++ `bus1`: second of two connected buses ++ `length`: length of line in km + `under_construction`: boolean, `t` for lines that are currently under construction -+ `length_m`: length of line in meters + `tags`: _hstore_ encoded dictionary of extra properties for this link + `geometry`: extent of this line in well-known-text format (WGS84) @@ -95,26 +105,26 @@ Generators attached to the network. + `generator_id`: unique identifier for the generator + `bus_id`: the bus to which this generator is connected -+ `symbol`: type of generator -+ `capacity`: capacity of this generator (in megawatt) ++ `technology`: type of generator ++ `capacity`: capacity of this generator in MW + `tags`: _hstore_ encoded dictionary of extra attributes + `geometry`: location of generator in well-known text format (WGS84) ### transformers.csv -A transformer forms a link between buses which operate at distinct -voltages. **NOTA BENE**: Transformers _never_ originate from the -original dataset, and transformers are _only_ infered in 'real' -stations, never in synthetic ('joint') stations. +A transformer connects buses which operate at distinct voltages. **NOTA BENE**: +Transformers are _not_ represented in the original dataset, but instead have +been added at substations to connect AC transmission lines of distinct voltage +levels. + `transformer_id`: unique identifier -+ `symbol`: either `transformer` for AC-to-AC voltage transformers, or - `ac/dc` for AC-to-DC converters. -+ `src_bus_id`: first of the connected buses -+ `dst_bus_id`: second of connected buses -+ `src_voltage`: voltage of first bus -+ `dst_voltage`: voltage of second bus -+ `src_dc`: boolean, `t` if first bus is a DC terminal -+ `dst_dc`: boolean, `f` if second bus is a DC terminal -+ `geometry`: location of station of this transformer in well-known - text format (WGS84) ++ `bus0`: Bus at lower voltage level + `bus1`: Bus at higher voltage level + +### converters.csv + +Back-to-back converters connecting non-synchronized buses. + ++ `converter_id`: unique identifier ++ `bus0`: First bus + `bus1`: Second bus From 839592855d887661c633015f31d2565935a43baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 12 Jan 2020 22:42:09 +0100 Subject: [PATCH 12/19] entsoe: remove old mapbox handles --- entsoe/download.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entsoe/download.sh b/entsoe/download.sh index 0669381..3163392 100755 --- a/entsoe/download.sh +++ b/entsoe/download.sh @@ -5,7 +5,7 @@ ZOOM=${1-6} npm install vt-geojson export MapboxAccessToken=pk.eyJ1IjoiZW50c29lIiwiYSI6ImNpbWxxYXJocDAwMG53Ymx3N2JxNGhtZDYifQ.YjNgK9usqRNrzxWXnR152g -for key in rusty.cm0b8gzp rusty.02rit83j entsoe.1nr7sorj entsoe.5m43hs6u; do +for key in entsoe.1nr7sorj entsoe.5m43hs6u; do echo Getting ${key} - node_modules/vt-geojson/cli.js ${key} -z ${ZOOM} > ${key}-z${ZOOM}.geojson + node_modules/vt-geojson/cli.js ${key} -z ${ZOOM} > data/${key}-z${ZOOM}.geojson done From 73021b9b80b0aa6f45109e9adfbc4b998c7e3588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 12 Jan 2020 22:43:30 +0100 Subject: [PATCH 13/19] entsoe: Fix pandas bit-rot in fixup-mapbox-tiles --- entsoe/fixup-mapbox-tiles.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/entsoe/fixup-mapbox-tiles.py b/entsoe/fixup-mapbox-tiles.py index ed416c6..24e482b 100644 --- a/entsoe/fixup-mapbox-tiles.py +++ b/entsoe/fixup-mapbox-tiles.py @@ -1,4 +1,5 @@ import fiona +import numpy as np import pandas as pd import geopandas as gpd import geojson @@ -6,7 +7,7 @@ from six import iteritems from six.moves import reduce -from itertools import count, permutations +from itertools import chain, count, permutations import os, sys type_map = dict(MultiLineString="LineString", @@ -39,6 +40,8 @@ def gen_outfn(fn, suffix, tmpdir=None): if tmpdir is not None: outfn = os.path.join(tmpdir, outfn) + else: + outfn = os.path.join(os.path.dirname(fn), outfn) if os.path.exists(outfn): os.unlink(outfn) @@ -73,14 +76,13 @@ def stitch_tiles(fn): df.loc[uc_b, 'symbol'] = df.loc[uc_b, 'symbol'].str[:-len(' Under Construction')] # Break MultiLineStrings - extra = [] - for i in count(): - e = (df[df.type == 'MultiLineString'] - .assign(geometry=lambda df: df.geometry.str[i]) - .dropna(subset=['geometry'])) - if e.empty: break - extra.append(e) - df = df[df.type != 'MultiLineString'].append(pd.concat(extra), ignore_index=True) + e = (df.loc[df.type == 'MultiLineString', 'geometry']) + if not e.empty: + extra = df.drop("geometry", axis=1).join( + pd.Series(chain(*e), np.repeat(e.index, e.map(len)), name="geometry"), + how="right" + ) + df = df[df.type != 'MultiLineString'].append(extra, ignore_index=True) def up_to_point(l, p, other): if l.boundary[1].distance(other) > l.boundary[0].distance(other): From 95b455004f26309496126896003b2055b2844082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 12 Jan 2020 22:44:33 +0100 Subject: [PATCH 14/19] entsoe: Don't use geopandas unless necessary GeoPandas introduced null properties interfering with station capacities. --- entsoe/fixup-mapbox-tiles.py | 58 ++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/entsoe/fixup-mapbox-tiles.py b/entsoe/fixup-mapbox-tiles.py index 24e482b..c16eaa2 100644 --- a/entsoe/fixup-mapbox-tiles.py +++ b/entsoe/fixup-mapbox-tiles.py @@ -133,24 +133,50 @@ def stitch_where_possible(df): return outfn def strip_duplicate_stations(fn): - df = gpd.read_file(fn) - df = df.rename(columns={'OBJECTID': 'oid', - 'ogc_fid': 'oid', - 'Under_cons': 'under_construction', - 'name_all': 'name', - 'Name_Eng': 'name', - 'Symbol': 'symbol', - 'Country': 'country', - 'Tie_line_s': 'tie_line_s', - 'Visible': 'visible', - 'MW': 'capacity', - 'mw': 'capacity'}) - df['symbol'] = df['symbol'].str.split(',').str[0] + with open(fn) as fp: + f = geojson.load(fp) + + key_map = {'OBJECTID': 'oid', + 'ogc_fid': 'oid', + 'Under_cons': 'under_construction', + 'name_all': 'name', + 'Name_Eng': 'name', + 'Symbol': 'symbol', + 'Country': 'country', + 'Tie_line_s': 'tie_line_s', + 'Visible': 'visible', + 'MW': 'capacity', + 'mw': 'capacity'} + seen_oids = set() + features = [] + + for feature in f['features']: + prop = feature['properties'] + for old, new in key_map.items(): + if old in prop: + prop[new] = prop.pop(old) + + feature['id'] = oid = prop['oid'] + if oid in seen_oids: + continue + seen_oids.add(oid) + + if 'symbol' in prop: + prop['symbol'] = prop['symbol'].split(',', 1)[0] + + if 'TSO' in prop: + prop['TSO'] = prop['TSO'].strip() + + if 'EIC_code' in prop: + prop['EIC_code'] = prop['EIC_code'].strip() + + features.append(feature) + + f['features'] = features outfn = gen_outfn(fn, '-fixed') - (df[~df.duplicated(subset=['oid'], keep='first')] - .set_index('oid', drop=False) - .to_file(outfn, driver='GeoJSON')) + with open(outfn, 'w') as fp: + geojson.dump(f, fp) return outfn if __name__ == '__main__': From f64e65ab36b4e4345d7b41fcc708c8a0f54018e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 12 Jan 2020 22:46:51 +0100 Subject: [PATCH 15/19] entsoe: Move data files to separate directories --- entsoe/data/.gitkeep | 0 entsoe/export.sh | 14 ++++++-------- entsoe/import.sh | 4 ++-- entsoe/result/.gitkeep | 0 4 files changed, 8 insertions(+), 10 deletions(-) create mode 100644 entsoe/data/.gitkeep create mode 100644 entsoe/result/.gitkeep diff --git a/entsoe/data/.gitkeep b/entsoe/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/entsoe/export.sh b/entsoe/export.sh index 6cd6520..0c72b18 100755 --- a/entsoe/export.sh +++ b/entsoe/export.sh @@ -1,9 +1,7 @@ #!/bin/bash -psql -c "COPY network_bus TO STDOUT WITH CSV HEADER QUOTE ''''" > buses.csv -psql -c "COPY network_line TO STDOUT WITH CSV HEADER QUOTE ''''" > lines.csv -psql -c "COPY network_link TO STDOUT WITH CSV HEADER QUOTE ''''" > links.csv -psql -c "COPY network_converter TO STDOUT WITH CSV HEADER QUOTE ''''" > converters.csv -psql -c "COPY network_transformer TO STDOUT WITH CSV HEADER QUOTE ''''" > transformers.csv -psql -c "COPY network_generator TO STDOUT WITH CSV HEADER QUOTE ''''" > generators.csv - -zip entsoe.zip README.md buses.csv lines.csv links.csv converters.csv transformers.csv generators.csv +psql -c "COPY network_bus TO STDOUT WITH CSV HEADER QUOTE ''''" > result/buses.csv +psql -c "COPY network_line TO STDOUT WITH CSV HEADER QUOTE ''''" > result/lines.csv +psql -c "COPY network_link TO STDOUT WITH CSV HEADER QUOTE ''''" > result/links.csv +psql -c "COPY network_converter TO STDOUT WITH CSV HEADER QUOTE ''''" > result/converters.csv +psql -c "COPY network_transformer TO STDOUT WITH CSV HEADER QUOTE ''''" > result/transformers.csv +psql -c "COPY network_generator TO STDOUT WITH CSV HEADER QUOTE ''''" > result/generators.csv diff --git a/entsoe/import.sh b/entsoe/import.sh index dfb572e..43fbd73 100755 --- a/entsoe/import.sh +++ b/entsoe/import.sh @@ -3,5 +3,5 @@ BASE=${1-entsoe} ZOOM=${2-6} -python fixup-mapbox-tiles.py ${BASE} ${BASE}*-z${ZOOM}.geojson -python ../util/geojson-to-postgis.py ${BASE}-*-fixed.geojson +python fixup-mapbox-tiles.py data/${BASE} data/${BASE}*-z${ZOOM}.geojson +python ../util/geojson-to-postgis.py data/${BASE}-*-fixed.geojson diff --git a/entsoe/result/.gitkeep b/entsoe/result/.gitkeep new file mode 100644 index 0000000..e69de29 From 3960210c474cb530989dfab1ee4e106c7d7aa11f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 12 Jan 2020 22:48:32 +0100 Subject: [PATCH 16/19] Fix invalid offset postgis error ST_ADDPOINT seems to have dropped support for using an offset of -1 for appending a new point, instead one has to omit the argument. --- src/spatial-3-attachment-joints.sql | 2 +- src/spatial-5-terminal-joints.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/spatial-3-attachment-joints.sql b/src/spatial-3-attachment-joints.sql index 3569ded..a5a72ed 100644 --- a/src/spatial-3-attachment-joints.sql +++ b/src/spatial-3-attachment-joints.sql @@ -92,7 +92,7 @@ update attached_lines and not st_dwithin(attachments, st_startpoint(old_extent), 1); update attached_lines - set new_extent = st_addpoint(new_extent, st_closestpoint(attachments, st_endpoint(new_extent)), -1) + set new_extent = st_addpoint(new_extent, st_closestpoint(attachments, st_endpoint(new_extent))) where st_contains(terminals, st_endpoint(old_extent)) and not st_dwithin(attachments, st_endpoint(old_extent), 1); diff --git a/src/spatial-5-terminal-joints.sql b/src/spatial-5-terminal-joints.sql index 553f597..1b1a2cb 100644 --- a/src/spatial-5-terminal-joints.sql +++ b/src/spatial-5-terminal-joints.sql @@ -34,11 +34,11 @@ insert into extended_lines (line_id, line_pt, old_extent, new_extent, locations) update extended_lines set new_extent = case when line_pt[1] = 1 then st_addpoint(new_extent, st_closestpoint(locations, st_startpoint(new_extent)), 0) - else st_addpoint(new_extent, st_closestpoint(locations, st_endpoint(new_extent)), -1) end; + else st_addpoint(new_extent, st_closestpoint(locations, st_endpoint(new_extent))) end; update extended_lines set new_extent = case when line_pt[2] = 1 then st_addpoint(new_extent, st_closestpoint(locations, st_startpoint(new_extent)), 0) - else st_addpoint(new_extent, st_closestpoint(locations, st_endpoint(new_extent)), -1) end + else st_addpoint(new_extent, st_closestpoint(locations, st_endpoint(new_extent))) end where array_length(line_pt, 1) = 2; insert into power_station (station_id, power_name, area) From 0d8262f50cdce509fe0045bfd8159df58761303f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 12 Jan 2020 22:48:55 +0100 Subject: [PATCH 17/19] Add data directories to .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 996b8cf..15d8c4a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ __pycache__/ *.osm *.o5m *.osm.pbf -.R* \ No newline at end of file +.R* +.direnv +entsoe/result +entsoe/data \ No newline at end of file From 5c95144e53d1ebaa9bea58010649580eea769437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 12 Jan 2020 22:51:09 +0100 Subject: [PATCH 18/19] entsoe: Remove broken gridkit rule --- entsoe/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/entsoe/run.sh b/entsoe/run.sh index 10fd7a6..9b640b6 100755 --- a/entsoe/run.sh +++ b/entsoe/run.sh @@ -21,7 +21,8 @@ psql -f ../src/topology-2-dangling-joints.sql psql -f electric-properties.sql # Fix edges which should not have been merged -psql -f fixup-merge.sql +# (only makes sense in conjunction with topology-3-redundant-splits) +#psql -f fixup-merge.sql psql -f fixup-annotated-lines.sql psql -v hvdc_distance=$GRIDKIT_HVDC_DISTANCE \ From 46f23aaa05c93f67ab4888530ffe733a6b85a3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6rsch?= Date: Sun, 12 Jan 2020 22:52:24 +0100 Subject: [PATCH 19/19] entsoe: Add runall_in_docker script and requirements --- entsoe/requirements.in | 7 +++++++ entsoe/requirements.txt | 24 +++++++++++++++++++++++ entsoe/runall_in_docker.sh | 39 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 entsoe/requirements.in create mode 100644 entsoe/requirements.txt create mode 100755 entsoe/runall_in_docker.sh diff --git a/entsoe/requirements.in b/entsoe/requirements.in new file mode 100644 index 0000000..8fb4768 --- /dev/null +++ b/entsoe/requirements.in @@ -0,0 +1,7 @@ +psycopg2-binary +fiona +pandas +geopandas +geojson +shapely +six \ No newline at end of file diff --git a/entsoe/requirements.txt b/entsoe/requirements.txt new file mode 100644 index 0000000..af14d1a --- /dev/null +++ b/entsoe/requirements.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile +# +aenum==2.2.3 # via pyproj +attrs==19.3.0 # via fiona +click-plugins==1.1.1 # via fiona +click==7.0 # via click-plugins, cligj, fiona +cligj==0.5.0 # via fiona +enum34==1.1.6 # via fiona +fiona==1.8.13 +geojson==2.5.0 +geopandas==0.6.2 +munch==2.5.0 # via fiona +numpy==1.16.6 # via pandas +pandas==0.24.2 +psycopg2-binary==2.8.4 +pyproj==2.2.2 # via geopandas +python-dateutil==2.8.1 # via pandas +pytz==2019.3 # via pandas +shapely==1.6.4.post2 +six==1.13.0 diff --git a/entsoe/runall_in_docker.sh b/entsoe/runall_in_docker.sh new file mode 100755 index 0000000..5ab1648 --- /dev/null +++ b/entsoe/runall_in_docker.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +BASE=$(realpath $PWD/..) +PGUSER=postgres +PGPASSWORD=mysecretpassword + +echo +echo "Download geojson from mapbox" +echo "============================" +docker run -it -v $BASE:/app:rw -w /app/entsoe node:8.16.0-alpine sh download.sh + +echo +echo "Start postgis database server" +echo "=============================" +docker run --name gridkit-postgis -e POSTGRES_PASSWORD=$PGPASSWORD -d mdillon/postgis + +echo +echo "Import into postgis" +echo "===================" +docker run -it --link gridkit-postgis:postgres --rm \ + -v $BASE:/app:rw -w /app/entsoe \ + -e PGUSER=$PGUSER -e PGPASSWORD=$PGPASSWORD \ + python:2 \ + bash -c 'pip install -r requirements.txt && env PGHOST="$POSTGRES_PORT_5432_TCP_ADDR" PGPORT="$POSTGRES_PORT_5432_TCP_PORT" ./import.sh' + +echo +echo "Run Gridkit and export into result directory" +echo "============================================" +docker run -it --link gridkit-postgis:postgres --rm \ + -v $BASE:/app:rw -w /app/entsoe \ + -e PGUSER=$PGUSER -e PGPASSWORD=$PGPASSWORD \ + mdillon/postgis \ + bash -c 'env PGHOST="$POSTGRES_PORT_5432_TCP_ADDR" PGPORT="$POSTGRES_PORT_5432_TCP_PORT" ./run.sh' + +echo +echo "Tear down postgis server" +echo "========================" +docker stop gridkit-postgis +docker rm gridkit-postgis \ No newline at end of file