From d4d24c49fc70ae211da5c2a69e4346a51ebe7d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 7 Jan 2021 11:30:00 +0100 Subject: [PATCH 01/13] Add initial Alembic structure Except for some style fixes, these files where generated by running `alembic init alembic` inside "src/egon/data". --- src/egon/data/alembic/README | 1 + src/egon/data/alembic/env.py | 75 ++++++++++++++++++++++++++++ src/egon/data/alembic/script.py.mako | 24 +++++++++ 3 files changed, 100 insertions(+) create mode 100644 src/egon/data/alembic/README create mode 100644 src/egon/data/alembic/env.py create mode 100644 src/egon/data/alembic/script.py.mako diff --git a/src/egon/data/alembic/README b/src/egon/data/alembic/README new file mode 100644 index 000000000..2500aa1bc --- /dev/null +++ b/src/egon/data/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. diff --git a/src/egon/data/alembic/env.py b/src/egon/data/alembic/env.py new file mode 100644 index 000000000..c872dab3b --- /dev/null +++ b/src/egon/data/alembic/env.py @@ -0,0 +1,75 @@ +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/src/egon/data/alembic/script.py.mako b/src/egon/data/alembic/script.py.mako new file mode 100644 index 000000000..2c0156303 --- /dev/null +++ b/src/egon/data/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} From c2250bea6851e68262541acad626815f0126109a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Thu, 7 Jan 2021 11:42:04 +0100 Subject: [PATCH 02/13] Add "alembic.ini" This is also the result of running `alembic init alembic`, but I forgot to add the file in the previous commit, because I only committed the contents of the alembic directory, whereas the "alembic.ini" file got placed directly outside of it. --- src/egon/data/alembic.ini | 85 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/egon/data/alembic.ini diff --git a/src/egon/data/alembic.ini b/src/egon/data/alembic.ini new file mode 100644 index 000000000..bfcc3c7f9 --- /dev/null +++ b/src/egon/data/alembic.ini @@ -0,0 +1,85 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks=black +# black.type=console_scripts +# black.entrypoint=black +# black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S From f60f7cffa6536352b8a651fed2983613157d84f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 7 Jan 2021 12:23:27 +0100 Subject: [PATCH 03/13] Use egon_data engine creator for Alembic --- src/egon/data/alembic/env.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/egon/data/alembic/env.py b/src/egon/data/alembic/env.py index c872dab3b..8cad495ab 100644 --- a/src/egon/data/alembic/env.py +++ b/src/egon/data/alembic/env.py @@ -1,7 +1,8 @@ from logging.config import fileConfig from alembic import context -from sqlalchemy import engine_from_config, pool + +from egon.data.db import engine # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -54,11 +55,7 @@ def run_migrations_online(): and associate a connection with the context. """ - connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) + connectable = engine() with connectable.connect() as connection: context.configure( From 83f334ccf781082d195d72b5f2b2ca9c65c2914e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 7 Jan 2021 13:05:55 +0100 Subject: [PATCH 04/13] Add ORM for OpenStreeMap tables This includes only ORMs for target tables in schema openstreetmap --- src/egon/data/orm/__init__.py | 0 src/egon/data/orm/openstreetmap.py | 418 +++++++++++++++++++++++++++++ 2 files changed, 418 insertions(+) create mode 100644 src/egon/data/orm/__init__.py create mode 100644 src/egon/data/orm/openstreetmap.py diff --git a/src/egon/data/orm/__init__.py b/src/egon/data/orm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/egon/data/orm/openstreetmap.py b/src/egon/data/orm/openstreetmap.py new file mode 100644 index 000000000..d6dc4a1fc --- /dev/null +++ b/src/egon/data/orm/openstreetmap.py @@ -0,0 +1,418 @@ +from geoalchemy2.types import Geometry +from sqlalchemy import ( + ARRAY, + BigInteger, + Column, + Float, + Integer, + SmallInteger, + Text, + text, +) +from sqlalchemy.dialects.postgresql import HSTORE +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() +metadata = Base.metadata + + +class OsmLine(Base): + __tablename__ = "osm_line" + __table_args__ = { + "schema": "openstreetmap", + } + + osm_id = Column(BigInteger) + access = Column(Text) + addr_housename = Column("addr:housename", Text) + addr_housenumber = Column("addr:housenumber", Text) + addr_interpolation = Column("addr:interpolation", Text) + admin_level = Column(Text) + aerialway = Column(Text) + aeroway = Column(Text) + amenity = Column(Text) + area = Column(Text) + barrier = Column(Text) + bicycle = Column(Text) + brand = Column(Text) + bridge = Column(Text) + boundary = Column(Text) + building = Column(Text) + construction = Column(Text) + covered = Column(Text) + culvert = Column(Text) + cutting = Column(Text) + denomination = Column(Text) + disused = Column(Text) + embankment = Column(Text) + foot = Column(Text) + generator_source = Column("generator:source", Text) + harbour = Column(Text) + highway = Column(Text) + historic = Column(Text) + horse = Column(Text) + intermittent = Column(Text) + junction = Column(Text) + landuse = Column(Text) + layer = Column(Text) + leisure = Column(Text) + line = Column(Text) + lock = Column(Text) + man_made = Column(Text) + military = Column(Text) + motorcar = Column(Text) + name = Column(Text) + natural = Column(Text) + office = Column(Text) + oneway = Column(Text) + operator = Column(Text) + place = Column(Text) + population = Column(Text) + power = Column(Text) + power_source = Column(Text) + public_transport = Column(Text) + railway = Column(Text) + ref = Column(Text) + religion = Column(Text) + route = Column(Text) + service = Column(Text) + sidewalk = Column(Text) + shop = Column(Text) + sport = Column(Text) + surface = Column(Text) + toll = Column(Text) + tourism = Column(Text) + tower_type = Column("tower:type", Text) + tracktype = Column(Text) + tunnel = Column(Text) + water = Column(Text) + waterway = Column(Text) + wetland = Column(Text) + width = Column(Text) + wood = Column(Text) + z_order = Column(Integer) + way_area = Column(Float) + abandoned_aeroway = Column("abandoned:aeroway", Text) + abandoned_amenity = Column("abandoned:amenity", Text) + abandoned_building = Column("abandoned:building", Text) + abandoned_landuse = Column("abandoned:landuse", Text) + abandoned_power = Column("abandoned:power", Text) + area_highway = Column("area:highway", Text) + tags = Column(HSTORE(Text()), index=True) + geom = Column(Geometry("LINESTRING", 3857), index=True) + gid = Column( + Integer, + primary_key=True, + server_default=text( + "nextval('openstreetmap.osm_line_gid_seq'::regclass)" + ), + ) + + +class OsmNode(Base): + __tablename__ = "osm_nodes" + __table_args__ = { + "schema": "openstreetmap", + } + + id = Column(BigInteger, primary_key=True) + lat = Column(Integer, nullable=False) + lon = Column(Integer, nullable=False) + + +class OsmPoint(Base): + __tablename__ = "osm_point" + __table_args__ = { + "schema": "openstreetmap", + } + + osm_id = Column(BigInteger) + access = Column(Text) + addr_housename = Column("addr:housename", Text) + addr_housenumber = Column("addr:housenumber", Text) + addr_interpolation = Column("addr:interpolation", Text) + admin_level = Column(Text) + aerialway = Column(Text) + aeroway = Column(Text) + amenity = Column(Text) + area = Column(Text) + barrier = Column(Text) + bicycle = Column(Text) + brand = Column(Text) + bridge = Column(Text) + boundary = Column(Text) + building = Column(Text) + capital = Column(Text) + construction = Column(Text) + covered = Column(Text) + culvert = Column(Text) + cutting = Column(Text) + denomination = Column(Text) + disused = Column(Text) + ele = Column(Text) + embankment = Column(Text) + foot = Column(Text) + generator_source = Column("generator:source", Text) + harbour = Column(Text) + highway = Column(Text) + historic = Column(Text) + horse = Column(Text) + intermittent = Column(Text) + junction = Column(Text) + landuse = Column(Text) + layer = Column(Text) + leisure = Column(Text) + line = Column(Text) + lock = Column(Text) + man_made = Column(Text) + military = Column(Text) + motorcar = Column(Text) + name = Column(Text) + natural = Column(Text) + office = Column(Text) + oneway = Column(Text) + operator = Column(Text) + place = Column(Text) + population = Column(Text) + power = Column(Text) + power_source = Column(Text) + public_transport = Column(Text) + railway = Column(Text) + ref = Column(Text) + religion = Column(Text) + route = Column(Text) + service = Column(Text) + sidewalk = Column(Text) + shop = Column(Text) + sport = Column(Text) + surface = Column(Text) + toll = Column(Text) + tourism = Column(Text) + tower_type = Column("tower:type", Text) + tunnel = Column(Text) + water = Column(Text) + waterway = Column(Text) + wetland = Column(Text) + width = Column(Text) + wood = Column(Text) + z_order = Column(Integer) + tags = Column(HSTORE(Text()), index=True) + geom = Column(Geometry("POINT", 3857), index=True) + gid = Column( + Integer, + primary_key=True, + server_default=text( + "nextval('openstreetmap.osm_point_gid_seq'::regclass)" + ), + ) + + +class OsmPolygon(Base): + __tablename__ = "osm_polygon" + __table_args__ = { + "schema": "openstreetmap", + } + + osm_id = Column(BigInteger) + access = Column(Text) + addr_housename = Column("addr:housename", Text) + addr_housenumber = Column("addr:housenumber", Text) + addr_interpolation = Column("addr:interpolation", Text) + admin_level = Column(Text) + aerialway = Column(Text) + aeroway = Column(Text) + amenity = Column(Text) + area = Column(Text) + barrier = Column(Text) + bicycle = Column(Text) + brand = Column(Text) + bridge = Column(Text) + boundary = Column(Text) + building = Column(Text) + construction = Column(Text) + covered = Column(Text) + culvert = Column(Text) + cutting = Column(Text) + denomination = Column(Text) + disused = Column(Text) + embankment = Column(Text) + foot = Column(Text) + generator_source = Column("generator:source", Text) + harbour = Column(Text) + highway = Column(Text) + historic = Column(Text) + horse = Column(Text) + intermittent = Column(Text) + junction = Column(Text) + landuse = Column(Text) + layer = Column(Text) + leisure = Column(Text) + line = Column(Text) + lock = Column(Text) + man_made = Column(Text) + military = Column(Text) + motorcar = Column(Text) + name = Column(Text) + natural = Column(Text) + office = Column(Text) + oneway = Column(Text) + operator = Column(Text) + place = Column(Text) + population = Column(Text) + power = Column(Text) + power_source = Column(Text) + public_transport = Column(Text) + railway = Column(Text) + ref = Column(Text) + religion = Column(Text) + route = Column(Text) + service = Column(Text) + sidewalk = Column(Text) + shop = Column(Text) + sport = Column(Text) + surface = Column(Text) + toll = Column(Text) + tourism = Column(Text) + tower_type = Column("tower:type", Text) + tracktype = Column(Text) + tunnel = Column(Text) + water = Column(Text) + waterway = Column(Text) + wetland = Column(Text) + width = Column(Text) + wood = Column(Text) + z_order = Column(Integer) + way_area = Column(Float) + abandoned_aeroway = Column("abandoned:aeroway", Text) + abandoned_amenity = Column("abandoned:amenity", Text) + abandoned_building = Column("abandoned:building", Text) + abandoned_landuse = Column("abandoned:landuse", Text) + abandoned_power = Column("abandoned:power", Text) + area_highway = Column("area:highway", Text) + tags = Column(HSTORE(Text()), index=True) + geom = Column(Geometry(srid=3857), index=True) + gid = Column( + Integer, + primary_key=True, + server_default=text( + "nextval('openstreetmap.osm_polygon_gid_seq'::regclass)" + ), + ) + + +class OsmRel(Base): + __tablename__ = "osm_rels" + __table_args__ = { + "schema": "openstreetmap", + } + + id = Column(BigInteger, primary_key=True) + way_off = Column(SmallInteger) + rel_off = Column(SmallInteger) + parts = Column(ARRAY(BigInteger()), index=True) + members = Column(ARRAY(Text())) + tags = Column(ARRAY(Text())) + + +class OsmRoad(Base): + __tablename__ = "osm_roads" + __table_args__ = { + "schema": "openstreetmap", + } + + osm_id = Column(BigInteger) + access = Column(Text) + addr_housename = Column("addr:housename", Text) + addr_housenumber = Column("addr:housenumber", Text) + addr_interpolation = Column("addr:interpolation", Text) + admin_level = Column(Text) + aerialway = Column(Text) + aeroway = Column(Text) + amenity = Column(Text) + area = Column(Text) + barrier = Column(Text) + bicycle = Column(Text) + brand = Column(Text) + bridge = Column(Text) + boundary = Column(Text) + building = Column(Text) + construction = Column(Text) + covered = Column(Text) + culvert = Column(Text) + cutting = Column(Text) + denomination = Column(Text) + disused = Column(Text) + embankment = Column(Text) + foot = Column(Text) + generator_source = Column("generator:source", Text) + harbour = Column(Text) + highway = Column(Text) + historic = Column(Text) + horse = Column(Text) + intermittent = Column(Text) + junction = Column(Text) + landuse = Column(Text) + layer = Column(Text) + leisure = Column(Text) + line = Column(Text) + lock = Column(Text) + man_made = Column(Text) + military = Column(Text) + motorcar = Column(Text) + name = Column(Text) + natural = Column(Text) + office = Column(Text) + oneway = Column(Text) + operator = Column(Text) + place = Column(Text) + population = Column(Text) + power = Column(Text) + power_source = Column(Text) + public_transport = Column(Text) + railway = Column(Text) + ref = Column(Text) + religion = Column(Text) + route = Column(Text) + service = Column(Text) + sidewalk = Column(Text) + shop = Column(Text) + sport = Column(Text) + surface = Column(Text) + toll = Column(Text) + tourism = Column(Text) + tower_type = Column("tower:type", Text) + tracktype = Column(Text) + tunnel = Column(Text) + water = Column(Text) + waterway = Column(Text) + wetland = Column(Text) + width = Column(Text) + wood = Column(Text) + z_order = Column(Integer) + way_area = Column(Float) + abandoned_aeroway = Column("abandoned:aeroway", Text) + abandoned_amenity = Column("abandoned:amenity", Text) + abandoned_building = Column("abandoned:building", Text) + abandoned_landuse = Column("abandoned:landuse", Text) + abandoned_power = Column("abandoned:power", Text) + area_highway = Column("area:highway", Text) + tags = Column(HSTORE(Text()), index=True) + geom = Column(Geometry("LINESTRING", 3857), index=True) + gid = Column( + Integer, + primary_key=True, + server_default=text( + "nextval('openstreetmap.osm_roads_gid_seq'::regclass)" + ), + ) + + +class OsmWay(Base): + __tablename__ = "osm_ways" + __table_args__ = { + "schema": "openstreetmap", + } + + id = Column(BigInteger, primary_key=True) + nodes = Column(ARRAY(BigInteger()), nullable=False, index=True) + tags = Column(ARRAY(Text())) From d34ef5d420ba78aa58afea044abad7283b69e4de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guido=20Ple=C3=9Fmann?= Date: Thu, 7 Jan 2021 13:10:26 +0100 Subject: [PATCH 05/13] Add ORM for VG250 tables in boundaries schema --- src/egon/data/orm/boundaries.py | 386 ++++++++++++++++++++++++++++++++ 1 file changed, 386 insertions(+) create mode 100644 src/egon/data/orm/boundaries.py diff --git a/src/egon/data/orm/boundaries.py b/src/egon/data/orm/boundaries.py new file mode 100644 index 000000000..348affb30 --- /dev/null +++ b/src/egon/data/orm/boundaries.py @@ -0,0 +1,386 @@ +from geoalchemy2.types import Geometry +from sqlalchemy import ( + ARRAY, + BigInteger, + Boolean, + Column, + Float, + Integer, + Numeric, + String, + Table, + Text, + text, +) +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() +metadata = Base.metadata + + +t_bkg_vg250_statistics_view = Table( + "bkg_vg250_statistics_view", + metadata, + Column("id", Text), + Column("area_sum_ha", Integer), + schema="boundaries", +) + + +class Vg250Gem(Base): + __tablename__ = "vg250_gem" + __table_args__ = { + "schema": "boundaries", + } + + gid = Column(BigInteger, primary_key=True, index=True) + ade = Column(BigInteger) + gf = Column(BigInteger) + bsg = Column(BigInteger) + ars = Column(Text) + ags = Column(Text) + sdv_ars = Column(Text) + gen = Column(Text) + bez = Column(Text) + ibz = Column(BigInteger) + bem = Column(Text) + nbd = Column(Text) + sn_l = Column(Text) + sn_r = Column(Text) + sn_k = Column(Text) + sn_v1 = Column(Text) + sn_v2 = Column(Text) + sn_g = Column(Text) + fk_s3 = Column(Text) + nuts = Column(Text) + ars_0 = Column(Text) + ags_0 = Column(Text) + wsk = Column(Text) + debkg_id = Column(Text) + rs = Column(Text) + sdv_rs = Column(Text) + rs_0 = Column(Text) + geometry = Column(Geometry(srid=4326), index=True) + + +class Vg250GemClean(Base): + __tablename__ = "vg250_gem_clean" + __table_args__ = {"schema": "boundaries"} + + id = Column( + Integer, + primary_key=True, + server_default=text( + "nextval('boundaries.vg250_gem_clean_id_seq'::regclass)" + ), + ) + old_id = Column(Integer) + gen = Column(Text) + bez = Column(Text) + bem = Column(Text) + nuts = Column(String(5)) + rs_0 = Column(String(12)) + ags_0 = Column(String(12)) + area_ha = Column(Numeric) + count_hole = Column(Integer) + path = Column(ARRAY(Integer())) + is_hole = Column(Boolean) + geometry = Column(Geometry("POLYGON", 3035), index=True) + + +t_vg250_gem_hole = Table( + "vg250_gem_hole", + metadata, + Column("id", Integer, unique=True), + Column("old_id", Integer), + Column("gen", Text), + Column("bez", Text), + Column("bem", Text), + Column("nuts", String(5)), + Column("rs_0", String(12)), + Column("ags_0", String(12)), + Column("area_ha", Numeric), + Column("count_hole", Integer), + Column("path", ARRAY(Integer())), + Column("is_hole", Boolean), + Column("geometry", Geometry("POLYGON", 3035), index=True), + schema="boundaries", +) + + +t_vg250_gem_valid = Table( + "vg250_gem_valid", + metadata, + Column("id", BigInteger, unique=True), + Column("old_id", Integer), + Column("gen", Text), + Column("bez", Text), + Column("bem", Text), + Column("nuts", String(5)), + Column("rs_0", String(12)), + Column("ags_0", String(12)), + Column("area_ha", Float(53)), + Column("geometry", Geometry("POLYGON", 3035), index=True), + schema="boundaries", +) + + +class Vg250Kr(Base): + __tablename__ = "vg250_krs" + __table_args__ = { + "schema": "boundaries", + } + + gid = Column(BigInteger, primary_key=True, index=True) + ade = Column(BigInteger) + gf = Column(BigInteger) + bsg = Column(BigInteger) + ars = Column(Text) + ags = Column(Text) + sdv_ars = Column(Text) + gen = Column(Text) + bez = Column(Text) + ibz = Column(BigInteger) + bem = Column(Text) + nbd = Column(Text) + sn_l = Column(Text) + sn_r = Column(Text) + sn_k = Column(Text) + sn_v1 = Column(Text) + sn_v2 = Column(Text) + sn_g = Column(Text) + fk_s3 = Column(Text) + nuts = Column(Text) + ars_0 = Column(Text) + ags_0 = Column(Text) + wsk = Column(Text) + debkg_id = Column(Text) + rs = Column(Text) + sdv_rs = Column(Text) + rs_0 = Column(Text) + geometry = Column(Geometry(srid=4326), index=True) + + +t_vg250_krs_area = Table( + "vg250_krs_area", + metadata, + Column("id", Integer, unique=True), + Column("gen", Text), + Column("bez", Text), + Column("nuts", String(5)), + Column("rs_0", String(12)), + Column("ags_0", String(12)), + Column("area_ha", Float(53)), + Column("geometry", Geometry, index=True), + schema="boundaries", +) + + +class Vg250Lan(Base): + __tablename__ = "vg250_lan" + __table_args__ = { + "schema": "boundaries", + } + + gid = Column(BigInteger, primary_key=True, index=True) + ade = Column(BigInteger) + gf = Column(BigInteger) + bsg = Column(BigInteger) + ars = Column(Text) + ags = Column(Text) + sdv_ars = Column(Text) + gen = Column(Text) + bez = Column(Text) + ibz = Column(BigInteger) + bem = Column(Text) + nbd = Column(Text) + sn_l = Column(Text) + sn_r = Column(Text) + sn_k = Column(Text) + sn_v1 = Column(Text) + sn_v2 = Column(Text) + sn_g = Column(Text) + fk_s3 = Column(Text) + nuts = Column(Text) + ars_0 = Column(Text) + ags_0 = Column(Text) + wsk = Column(Text) + debkg_id = Column(Text) + rs = Column(Text) + sdv_rs = Column(Text) + rs_0 = Column(Text) + geometry = Column(Geometry(srid=4326), index=True) + + +t_vg250_lan_nuts_id = Table( + "vg250_lan_nuts_id", + metadata, + Column("ags_0", Text), + Column("gen", Text), + Column("nuts", Text), + Column("geometry", Geometry), + schema="boundaries", +) + + +t_vg250_lan_union = Table( + "vg250_lan_union", + metadata, + Column("ags_0", String(12), unique=True), + Column("gen", Text), + Column("geometry", Geometry("MULTIPOLYGON", 3035), index=True), + schema="boundaries", +) + + +class Vg250Rbz(Base): + __tablename__ = "vg250_rbz" + __table_args__ = { + "schema": "boundaries", + } + + gid = Column(BigInteger, primary_key=True, index=True) + ade = Column(BigInteger) + gf = Column(BigInteger) + bsg = Column(BigInteger) + ars = Column(Text) + ags = Column(Text) + sdv_ars = Column(Text) + gen = Column(Text) + bez = Column(Text) + ibz = Column(BigInteger) + bem = Column(Text) + nbd = Column(Text) + sn_l = Column(Text) + sn_r = Column(Text) + sn_k = Column(Text) + sn_v1 = Column(Text) + sn_v2 = Column(Text) + sn_g = Column(Text) + fk_s3 = Column(Text) + nuts = Column(Text) + ars_0 = Column(Text) + ags_0 = Column(Text) + wsk = Column(Text) + debkg_id = Column(Text) + rs = Column(Text) + sdv_rs = Column(Text) + rs_0 = Column(Text) + geometry = Column(Geometry(srid=4326), index=True) + + +class Vg250Sta(Base): + __tablename__ = "vg250_sta" + __table_args__ = { + "schema": "boundaries", + } + + gid = Column(BigInteger, primary_key=True, index=True) + ade = Column(BigInteger) + gf = Column(BigInteger) + bsg = Column(BigInteger) + ars = Column(Text) + ags = Column(Text) + sdv_ars = Column(Text) + gen = Column(Text) + bez = Column(Text) + ibz = Column(BigInteger) + bem = Column(Text) + nbd = Column(Text) + sn_l = Column(Text) + sn_r = Column(Text) + sn_k = Column(Text) + sn_v1 = Column(Text) + sn_v2 = Column(Text) + sn_g = Column(Text) + fk_s3 = Column(Text) + nuts = Column(Text) + ars_0 = Column(Text) + ags_0 = Column(Text) + wsk = Column(Text) + debkg_id = Column(Text) + rs = Column(Text) + sdv_rs = Column(Text) + rs_0 = Column(Text) + geometry = Column(Geometry(srid=4326), index=True) + + +t_vg250_sta_bbox = Table( + "vg250_sta_bbox", + metadata, + Column("gid", Integer, unique=True), + Column("bez", Text), + Column("area_ha", Float(53)), + Column("geometry", Geometry("POLYGON", 3035), index=True), + schema="boundaries", +) + + +t_vg250_sta_invalid_geometry = Table( + "vg250_sta_invalid_geometry", + metadata, + Column("gid", BigInteger, unique=True), + Column("error", Boolean), + Column("error_reason", String), + Column("geometry", Geometry("POINT", 3035), index=True), + schema="boundaries", +) + + +t_vg250_sta_tiny_buffer = Table( + "vg250_sta_tiny_buffer", + metadata, + Column("gid", Integer, unique=True), + Column("bez", Text), + Column("gf", Float(53)), + Column("area_ha", Float(53)), + Column("geometry", Geometry("MULTIPOLYGON", 3035), index=True), + schema="boundaries", +) + + +t_vg250_sta_union = Table( + "vg250_sta_union", + metadata, + Column("gid", Integer, unique=True), + Column("bez", Text), + Column("area_ha", Float(53)), + Column("geometry", Geometry("MULTIPOLYGON", 3035), index=True), + schema="boundaries", +) + + +class Vg250Vwg(Base): + __tablename__ = "vg250_vwg" + __table_args__ = { + "schema": "boundaries", + } + + gid = Column(BigInteger, primary_key=True, index=True) + ade = Column(BigInteger) + gf = Column(BigInteger) + bsg = Column(BigInteger) + ars = Column(Text) + ags = Column(Text) + sdv_ars = Column(Text) + gen = Column(Text) + bez = Column(Text) + ibz = Column(BigInteger) + bem = Column(Text) + nbd = Column(Text) + sn_l = Column(Text) + sn_r = Column(Text) + sn_k = Column(Text) + sn_v1 = Column(Text) + sn_v2 = Column(Text) + sn_g = Column(Text) + fk_s3 = Column(Text) + nuts = Column(Text) + ars_0 = Column(Text) + ags_0 = Column(Text) + wsk = Column(Text) + debkg_id = Column(Text) + rs = Column(Text) + sdv_rs = Column(Text) + rs_0 = Column(Text) + geometry = Column(Geometry(srid=4326), index=True) From c4504d72f6afcd81754348f090f1dd14d10efc81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 8 Jan 2021 14:59:17 +0100 Subject: [PATCH 06/13] Configure revision script filenames Just using the revision id doesn't convey too much information about the script when looking at the filename. It especially doesn't convey any information with respect to the relative ordering among scripts. Adding the time stamp to the script's filename at least gives an idea about this and it also gives a hint on how recent the change is. One even might use this information to figure how whether old migrations should be pruned in the future. When one talks about timestamps, it's of course good to always have them in UTC. That way, they are comparable even without specifying time zone information, which would just add unnecessary noise here. I also changed the character separating the different parts of the filename from underscores to periods, because those are already used to separate the suffix from the rest of the filename and using a smaller number if different characters is easier on the eyes, IMHO. Last but not least, I relaxed the length at which the revision message to 53 characters. This is the maximum length of the first line of a Git commit message, i.e. 50 characters, with a little bit of leeway. --- src/egon/data/alembic.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/alembic.ini b/src/egon/data/alembic.ini index bfcc3c7f9..75618559a 100644 --- a/src/egon/data/alembic.ini +++ b/src/egon/data/alembic.ini @@ -5,17 +5,17 @@ script_location = alembic # template used to generate migration files -# file_template = %%(rev)s_%%(slug)s +file_template = %%(year)d-%%(month).2d-%%(day).2dT%%(hour).2d:%%(minute).2d.%%(rev)s.%%(slug)s # timezone to use when rendering the date # within the migration file as well as the filename. # string value is passed to dateutil.tz.gettz() # leave blank for localtime -# timezone = +timezone = UTC # max length of characters to apply to the # "slug" field -# truncate_slug_length = 40 +truncate_slug_length = 53 # set to 'true' to run the environment during # the 'revision' command, regardless of autogenerate From 97980b9e7e7da9f26fba7067f7b9778fa5d8d223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 8 Jan 2021 15:11:24 +0100 Subject: [PATCH 07/13] Reformat newly generated migration scripts Since we use `black` and `isort` to ensure a consistent style in our Python code, running newly generated migration scripts through them ensures that not code violating our style guide gets automatically generated. Note that using Alembic's `console_scripts` entry point type unfortunately only works with Python console scripts installed in the same virtual environment as the `alembic` in use. If that starts to become a problem, we have to implement a custom write hook in "env.py" which calls arbitrary external programs. --- src/egon/data/alembic.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/egon/data/alembic.ini b/src/egon/data/alembic.ini index 75618559a..55d7d4516 100644 --- a/src/egon/data/alembic.ini +++ b/src/egon/data/alembic.ini @@ -44,10 +44,11 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # detail and examples # format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks=black -# black.type=console_scripts -# black.entrypoint=black -# black.options=-l 79 +hooks = isort, black +black.type = console_scripts +black.entrypoint = black +isort.type = console_scripts +isort.entrypoint = isort # Logging configuration [loggers] From 8e93c7b5f4e07480ccf9222eb7839f966ffc4694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 8 Jan 2021 17:28:38 +0100 Subject: [PATCH 08/13] Prefix migration scripts with a sequential number Since this moves the migration script, this hook has to be run after every other hook. Everything else should be explained in the comments and docstrings. We also have to set `revision_environment` to `true` because the "env.py" file contains our hook definition which always has to be loaded in order to move a generated migration script. --- src/egon/data/alembic.ini | 5 ++-- src/egon/data/alembic/env.py | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/egon/data/alembic.ini b/src/egon/data/alembic.ini index 55d7d4516..0540d40a8 100644 --- a/src/egon/data/alembic.ini +++ b/src/egon/data/alembic.ini @@ -19,7 +19,7 @@ truncate_slug_length = 53 # set to 'true' to run the environment during # the 'revision' command, regardless of autogenerate -# revision_environment = false +revision_environment = true # set to 'true' to allow .pyc and .pyo files without # a source .py file to be detected as revisions in the @@ -44,11 +44,12 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # detail and examples # format using "black" - use the console_scripts runner, against the "black" entrypoint -hooks = isort, black +hooks = isort, black, sequentialize black.type = console_scripts black.entrypoint = black isort.type = console_scripts isort.entrypoint = isort +sequentialize.type = sequentialize # Logging configuration [loggers] diff --git a/src/egon/data/alembic/env.py b/src/egon/data/alembic/env.py index 8cad495ab..5ae44c359 100644 --- a/src/egon/data/alembic/env.py +++ b/src/egon/data/alembic/env.py @@ -1,6 +1,9 @@ from logging.config import fileConfig +from pathlib import Path +import re from alembic import context +from alembic.script import ScriptDirectory, write_hooks from egon.data.db import engine @@ -24,6 +27,49 @@ # ... etc. +def script(filename=None, revision=None): + """Return the script object matching the parameters. + + Either the script object having the given :py:obj:`filename` or having + the given :py:obj:`revision`, is returned. If both are given, both have + to match. + + Supplying at least one is mandatory. + """ + assert filename is not None or revision is not None + scripts = ScriptDirectory.from_config(config) + return [ + s + for s in scripts.walk_revisions() + if (filename is None or s.path == filename) + and (revision is None or s.revision == revision) + ][0] + + +@write_hooks.register("sequentialize") +def assign_sequential_version_number(filename, options): + """Prefix the migration script's name with a sequential version number. + + The version number is generated by incrementing the version number of + the down revision by one. Since the intention is to get a sequential + ordering when sorting the migration scripts alphabetically, this + sequential number is right aligned and padded with zeroes to a length of + three characters. + """ + + current = script(filename=filename) + path = Path(filename) + down = ( + "-001." # Gets incremented to 0 in the line below. + if current.down_revision is None + else Path(script(revision=current.down_revision).path).name + ) + version = int(re.match(r"^(-?\d{3})\..*", down)[1]) + 1 + prefix = f"{version:0>3}." + path.rename(path.parent / (prefix + path.name)) + print(f'Prefixed new migration script with "{prefix}".') + + def run_migrations_offline(): """Run migrations in 'offline' mode. From db0f9b42c678eede37a7894fd3c5a8d84dd39d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 8 Jan 2021 17:44:32 +0100 Subject: [PATCH 09/13] Add the base migration The file was generated using the command ``` alembic revision -m "Do nothing for the base migration" ``` inside the "src/egon/data/alembic" directory. This is important, as this directory contains the "alembic.ini" file which Alembic needs to operate. As the docstring states, this migration is initially empty in order to provide a blank slate to build upon and to go back to. --- ...fcefd.do_nothing_for_the_base_migration.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/egon/data/alembic/versions/000.2021-01-08T16:37.9b9584efcefd.do_nothing_for_the_base_migration.py diff --git a/src/egon/data/alembic/versions/000.2021-01-08T16:37.9b9584efcefd.do_nothing_for_the_base_migration.py b/src/egon/data/alembic/versions/000.2021-01-08T16:37.9b9584efcefd.do_nothing_for_the_base_migration.py new file mode 100644 index 000000000..558d7f7d0 --- /dev/null +++ b/src/egon/data/alembic/versions/000.2021-01-08T16:37.9b9584efcefd.do_nothing_for_the_base_migration.py @@ -0,0 +1,25 @@ +"""Do nothing for the base migration + +This migration intentionally does nothing. Downgrading to this revision +should reset the database to a clean state or at least undo everything +done via Alembic. + +Revision ID: 9b9584efcefd +Revises: +Create Date: 2021-01-08 16:37:31.238609+00:00 + +""" + +# Revision identifiers, used by Alembic. +revision = "9b9584efcefd" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From df82f29761fda1b75a69ff1304c125060742ea82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 8 Jan 2021 19:26:14 +0100 Subject: [PATCH 10/13] Add an Alembic hook fixing minor issues --- src/egon/data/alembic.ini | 3 ++- src/egon/data/alembic/env.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/egon/data/alembic.ini b/src/egon/data/alembic.ini index 0540d40a8..27f60230a 100644 --- a/src/egon/data/alembic.ini +++ b/src/egon/data/alembic.ini @@ -44,7 +44,8 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # detail and examples # format using "black" - use the console_scripts runner, against the "black" entrypoint -hooks = isort, black, sequentialize +hooks = fixup, isort, black, sequentialize +fixup.type = fixup black.type = console_scripts black.entrypoint = black isort.type = console_scripts diff --git a/src/egon/data/alembic/env.py b/src/egon/data/alembic/env.py index 5ae44c359..8dc46e82b 100644 --- a/src/egon/data/alembic/env.py +++ b/src/egon/data/alembic/env.py @@ -1,3 +1,4 @@ +from fileinput import FileInput from logging.config import fileConfig from pathlib import Path import re @@ -27,6 +28,21 @@ # ... etc. +@write_hooks.register("fixup") +def fix_minor_issues(filename, options): + """Fix minor issues like grammar in comments or trailing whitespace.""" + fixes = [] + with FileInput(filename, inplace=True) as script: + for line in script: + if re.match(r"# revision identifiers, used by Alembic\.", line): + print("# Revision identifiers, used by Alembic.") + fixes.append('"# revision ..." -> "# Revision ..."') + else: + print(line, end="") + for fix in fixes: + print(f"Fixed: {fix}.") + + def script(filename=None, revision=None): """Return the script object matching the parameters. From 00f4af33e8afeb50e04f901f1d982e97cc2161e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Fri, 8 Jan 2021 20:13:23 +0100 Subject: [PATCH 11/13] Add a migration creating database extensions The extensions in question are hstore and postgis. These are currently added via a Docker entry point. Since this already created problems and since it also wouldn't work when not using Docker, moving it into a migration is both, a good proof of concept and a fix for an open issue. --- ..._hstore_and_postgis_database_extensions.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/egon/data/alembic/versions/001.2021-01-08T18:50.79833aa6bc04.create_the_hstore_and_postgis_database_extensions.py diff --git a/src/egon/data/alembic/versions/001.2021-01-08T18:50.79833aa6bc04.create_the_hstore_and_postgis_database_extensions.py b/src/egon/data/alembic/versions/001.2021-01-08T18:50.79833aa6bc04.create_the_hstore_and_postgis_database_extensions.py new file mode 100644 index 000000000..73f2b4547 --- /dev/null +++ b/src/egon/data/alembic/versions/001.2021-01-08T18:50.79833aa6bc04.create_the_hstore_and_postgis_database_extensions.py @@ -0,0 +1,24 @@ +"""Create the hstore and postgis database extensions + +Revision ID: 79833aa6bc04 +Revises: 9b9584efcefd +Create Date: 2021-01-08 18:50:07.951487+00:00 + +""" +from alembic import op + +# Revision identifiers, used by Alembic. +revision = "79833aa6bc04" +down_revision = "9b9584efcefd" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("CREATE EXTENSION IF NOT EXISTS hstore;") + op.execute("CREATE EXTENSION IF NOT EXISTS postgis;") + + +def downgrade(): + op.execute("DROP EXTENSION IF EXISTS postgis;") + op.execute("DROP EXTENSION IF EXISTS hstore;") From c30bc1b75f5c61ca678c8a328caab6a4a07ab3f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Sun, 10 Jan 2021 19:45:53 +0100 Subject: [PATCH 12/13] Run all migrations during `initdb` Since the migrations currently also contain one that creates the database extensions we need, this also fixes #76 and the Docker entry point is removed with this commit. --- src/egon/data/airflow/docker-compose.yml | 1 - src/egon/data/airflow/entrypoints/create_postgis_extension.sql | 2 -- src/egon/data/airflow/tasks.py | 3 +++ 3 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 src/egon/data/airflow/entrypoints/create_postgis_extension.sql diff --git a/src/egon/data/airflow/docker-compose.yml b/src/egon/data/airflow/docker-compose.yml index 94bc4c385..b64a8c2fc 100644 --- a/src/egon/data/airflow/docker-compose.yml +++ b/src/egon/data/airflow/docker-compose.yml @@ -15,4 +15,3 @@ services: POSTGRES_PASSWORD: data volumes: - $HOME/docker/volumes/postgres/egon-data:/var/lib/postgresql/data - - ./entrypoints:/docker-entrypoint-initdb.d/ diff --git a/src/egon/data/airflow/entrypoints/create_postgis_extension.sql b/src/egon/data/airflow/entrypoints/create_postgis_extension.sql deleted file mode 100644 index e3fcb15f5..000000000 --- a/src/egon/data/airflow/entrypoints/create_postgis_extension.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE EXTENSION hstore; -CREATE EXTENSION postgis; diff --git a/src/egon/data/airflow/tasks.py b/src/egon/data/airflow/tasks.py index b624fa85c..c61594a39 100644 --- a/src/egon/data/airflow/tasks.py +++ b/src/egon/data/airflow/tasks.py @@ -1,6 +1,8 @@ import os.path import subprocess +import egon.data + def initdb(): """ Initialize the local database used for data processing. """ @@ -8,3 +10,4 @@ def initdb(): ["docker-compose", "up", "-d", "--build"], cwd=os.path.dirname(__file__), ) + subprocess.run(["alembic", "upgrade", "head"], cwd=egon.data.__path__[0]) From 853bc1c67b57efa45edcad64eda02b40ad46f0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20G=C3=BCnther?= Date: Sun, 10 Jan 2021 21:54:08 +0100 Subject: [PATCH 13/13] Add the OpenStreetMap migration script The script was automatically generated by setting `target_metadata` in "src/egon/data/alembic/env.py" to `egon.data.orm.openstreetmap.metadata` and running ``` alembic revision \ --autogenerate \ -m "Create the OpenStreetMap database structure" ``` The script needs some adjustments though, which I'm putting into a separate commit for documentation purposes. --- src/egon/data/alembic/env.py | 3 +- ...te_the_openstreetmap_database_structure.py | 591 ++++++++++++++++++ 2 files changed, 593 insertions(+), 1 deletion(-) create mode 100644 src/egon/data/alembic/versions/002.2021-01-10T20:52.3aea51f86ebf.create_the_openstreetmap_database_structure.py diff --git a/src/egon/data/alembic/env.py b/src/egon/data/alembic/env.py index 8dc46e82b..af5281ef1 100644 --- a/src/egon/data/alembic/env.py +++ b/src/egon/data/alembic/env.py @@ -7,6 +7,7 @@ from alembic.script import ScriptDirectory, write_hooks from egon.data.db import engine +from egon.data.orm.openstreetmap import metadata # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -20,7 +21,7 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -target_metadata = None +target_metadata = metadata # other values from the config, defined by the needs of env.py, # can be acquired: diff --git a/src/egon/data/alembic/versions/002.2021-01-10T20:52.3aea51f86ebf.create_the_openstreetmap_database_structure.py b/src/egon/data/alembic/versions/002.2021-01-10T20:52.3aea51f86ebf.create_the_openstreetmap_database_structure.py new file mode 100644 index 000000000..0c941107e --- /dev/null +++ b/src/egon/data/alembic/versions/002.2021-01-10T20:52.3aea51f86ebf.create_the_openstreetmap_database_structure.py @@ -0,0 +1,591 @@ +"""Create the OpenStreetMap database structure + +Revision ID: 3aea51f86ebf +Revises: 79833aa6bc04 +Create Date: 2021-01-10 20:52:22.884864+00:00 + +""" +from alembic import op +from sqlalchemy.dialects import postgresql +import sqlalchemy as sa + +# Revision identifiers, used by Alembic. +revision = "3aea51f86ebf" +down_revision = "79833aa6bc04" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "osm_line", + sa.Column("osm_id", sa.BigInteger(), nullable=True), + sa.Column("access", sa.Text(), nullable=True), + sa.Column("addr:housename", sa.Text(), nullable=True), + sa.Column("addr:housenumber", sa.Text(), nullable=True), + sa.Column("addr:interpolation", sa.Text(), nullable=True), + sa.Column("admin_level", sa.Text(), nullable=True), + sa.Column("aerialway", sa.Text(), nullable=True), + sa.Column("aeroway", sa.Text(), nullable=True), + sa.Column("amenity", sa.Text(), nullable=True), + sa.Column("area", sa.Text(), nullable=True), + sa.Column("barrier", sa.Text(), nullable=True), + sa.Column("bicycle", sa.Text(), nullable=True), + sa.Column("brand", sa.Text(), nullable=True), + sa.Column("bridge", sa.Text(), nullable=True), + sa.Column("boundary", sa.Text(), nullable=True), + sa.Column("building", sa.Text(), nullable=True), + sa.Column("construction", sa.Text(), nullable=True), + sa.Column("covered", sa.Text(), nullable=True), + sa.Column("culvert", sa.Text(), nullable=True), + sa.Column("cutting", sa.Text(), nullable=True), + sa.Column("denomination", sa.Text(), nullable=True), + sa.Column("disused", sa.Text(), nullable=True), + sa.Column("embankment", sa.Text(), nullable=True), + sa.Column("foot", sa.Text(), nullable=True), + sa.Column("generator:source", sa.Text(), nullable=True), + sa.Column("harbour", sa.Text(), nullable=True), + sa.Column("highway", sa.Text(), nullable=True), + sa.Column("historic", sa.Text(), nullable=True), + sa.Column("horse", sa.Text(), nullable=True), + sa.Column("intermittent", sa.Text(), nullable=True), + sa.Column("junction", sa.Text(), nullable=True), + sa.Column("landuse", sa.Text(), nullable=True), + sa.Column("layer", sa.Text(), nullable=True), + sa.Column("leisure", sa.Text(), nullable=True), + sa.Column("line", sa.Text(), nullable=True), + sa.Column("lock", sa.Text(), nullable=True), + sa.Column("man_made", sa.Text(), nullable=True), + sa.Column("military", sa.Text(), nullable=True), + sa.Column("motorcar", sa.Text(), nullable=True), + sa.Column("name", sa.Text(), nullable=True), + sa.Column("natural", sa.Text(), nullable=True), + sa.Column("office", sa.Text(), nullable=True), + sa.Column("oneway", sa.Text(), nullable=True), + sa.Column("operator", sa.Text(), nullable=True), + sa.Column("place", sa.Text(), nullable=True), + sa.Column("population", sa.Text(), nullable=True), + sa.Column("power", sa.Text(), nullable=True), + sa.Column("power_source", sa.Text(), nullable=True), + sa.Column("public_transport", sa.Text(), nullable=True), + sa.Column("railway", sa.Text(), nullable=True), + sa.Column("ref", sa.Text(), nullable=True), + sa.Column("religion", sa.Text(), nullable=True), + sa.Column("route", sa.Text(), nullable=True), + sa.Column("service", sa.Text(), nullable=True), + sa.Column("sidewalk", sa.Text(), nullable=True), + sa.Column("shop", sa.Text(), nullable=True), + sa.Column("sport", sa.Text(), nullable=True), + sa.Column("surface", sa.Text(), nullable=True), + sa.Column("toll", sa.Text(), nullable=True), + sa.Column("tourism", sa.Text(), nullable=True), + sa.Column("tower:type", sa.Text(), nullable=True), + sa.Column("tracktype", sa.Text(), nullable=True), + sa.Column("tunnel", sa.Text(), nullable=True), + sa.Column("water", sa.Text(), nullable=True), + sa.Column("waterway", sa.Text(), nullable=True), + sa.Column("wetland", sa.Text(), nullable=True), + sa.Column("width", sa.Text(), nullable=True), + sa.Column("wood", sa.Text(), nullable=True), + sa.Column("z_order", sa.Integer(), nullable=True), + sa.Column("way_area", sa.Float(), nullable=True), + sa.Column("abandoned:aeroway", sa.Text(), nullable=True), + sa.Column("abandoned:amenity", sa.Text(), nullable=True), + sa.Column("abandoned:building", sa.Text(), nullable=True), + sa.Column("abandoned:landuse", sa.Text(), nullable=True), + sa.Column("abandoned:power", sa.Text(), nullable=True), + sa.Column("area:highway", sa.Text(), nullable=True), + sa.Column( + "tags", postgresql.HSTORE(text_type=sa.Text()), nullable=True + ), + sa.Column( + "geom", + geoalchemy2.types.Geometry(geometry_type="LINESTRING", srid=3857), + nullable=True, + ), + sa.Column( + "gid", + sa.Integer(), + server_default=sa.text( + "nextval('openstreetmap.osm_line_gid_seq'::regclass)" + ), + nullable=False, + ), + sa.PrimaryKeyConstraint("gid"), + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_line_geom"), + "osm_line", + ["geom"], + unique=False, + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_line_tags"), + "osm_line", + ["tags"], + unique=False, + schema="openstreetmap", + ) + op.create_table( + "osm_nodes", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("lat", sa.Integer(), nullable=False), + sa.Column("lon", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("id"), + schema="openstreetmap", + ) + op.create_table( + "osm_point", + sa.Column("osm_id", sa.BigInteger(), nullable=True), + sa.Column("access", sa.Text(), nullable=True), + sa.Column("addr:housename", sa.Text(), nullable=True), + sa.Column("addr:housenumber", sa.Text(), nullable=True), + sa.Column("addr:interpolation", sa.Text(), nullable=True), + sa.Column("admin_level", sa.Text(), nullable=True), + sa.Column("aerialway", sa.Text(), nullable=True), + sa.Column("aeroway", sa.Text(), nullable=True), + sa.Column("amenity", sa.Text(), nullable=True), + sa.Column("area", sa.Text(), nullable=True), + sa.Column("barrier", sa.Text(), nullable=True), + sa.Column("bicycle", sa.Text(), nullable=True), + sa.Column("brand", sa.Text(), nullable=True), + sa.Column("bridge", sa.Text(), nullable=True), + sa.Column("boundary", sa.Text(), nullable=True), + sa.Column("building", sa.Text(), nullable=True), + sa.Column("capital", sa.Text(), nullable=True), + sa.Column("construction", sa.Text(), nullable=True), + sa.Column("covered", sa.Text(), nullable=True), + sa.Column("culvert", sa.Text(), nullable=True), + sa.Column("cutting", sa.Text(), nullable=True), + sa.Column("denomination", sa.Text(), nullable=True), + sa.Column("disused", sa.Text(), nullable=True), + sa.Column("ele", sa.Text(), nullable=True), + sa.Column("embankment", sa.Text(), nullable=True), + sa.Column("foot", sa.Text(), nullable=True), + sa.Column("generator:source", sa.Text(), nullable=True), + sa.Column("harbour", sa.Text(), nullable=True), + sa.Column("highway", sa.Text(), nullable=True), + sa.Column("historic", sa.Text(), nullable=True), + sa.Column("horse", sa.Text(), nullable=True), + sa.Column("intermittent", sa.Text(), nullable=True), + sa.Column("junction", sa.Text(), nullable=True), + sa.Column("landuse", sa.Text(), nullable=True), + sa.Column("layer", sa.Text(), nullable=True), + sa.Column("leisure", sa.Text(), nullable=True), + sa.Column("line", sa.Text(), nullable=True), + sa.Column("lock", sa.Text(), nullable=True), + sa.Column("man_made", sa.Text(), nullable=True), + sa.Column("military", sa.Text(), nullable=True), + sa.Column("motorcar", sa.Text(), nullable=True), + sa.Column("name", sa.Text(), nullable=True), + sa.Column("natural", sa.Text(), nullable=True), + sa.Column("office", sa.Text(), nullable=True), + sa.Column("oneway", sa.Text(), nullable=True), + sa.Column("operator", sa.Text(), nullable=True), + sa.Column("place", sa.Text(), nullable=True), + sa.Column("population", sa.Text(), nullable=True), + sa.Column("power", sa.Text(), nullable=True), + sa.Column("power_source", sa.Text(), nullable=True), + sa.Column("public_transport", sa.Text(), nullable=True), + sa.Column("railway", sa.Text(), nullable=True), + sa.Column("ref", sa.Text(), nullable=True), + sa.Column("religion", sa.Text(), nullable=True), + sa.Column("route", sa.Text(), nullable=True), + sa.Column("service", sa.Text(), nullable=True), + sa.Column("sidewalk", sa.Text(), nullable=True), + sa.Column("shop", sa.Text(), nullable=True), + sa.Column("sport", sa.Text(), nullable=True), + sa.Column("surface", sa.Text(), nullable=True), + sa.Column("toll", sa.Text(), nullable=True), + sa.Column("tourism", sa.Text(), nullable=True), + sa.Column("tower:type", sa.Text(), nullable=True), + sa.Column("tunnel", sa.Text(), nullable=True), + sa.Column("water", sa.Text(), nullable=True), + sa.Column("waterway", sa.Text(), nullable=True), + sa.Column("wetland", sa.Text(), nullable=True), + sa.Column("width", sa.Text(), nullable=True), + sa.Column("wood", sa.Text(), nullable=True), + sa.Column("z_order", sa.Integer(), nullable=True), + sa.Column( + "tags", postgresql.HSTORE(text_type=sa.Text()), nullable=True + ), + sa.Column( + "geom", + geoalchemy2.types.Geometry(geometry_type="POINT", srid=3857), + nullable=True, + ), + sa.Column( + "gid", + sa.Integer(), + server_default=sa.text( + "nextval('openstreetmap.osm_point_gid_seq'::regclass)" + ), + nullable=False, + ), + sa.PrimaryKeyConstraint("gid"), + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_point_geom"), + "osm_point", + ["geom"], + unique=False, + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_point_tags"), + "osm_point", + ["tags"], + unique=False, + schema="openstreetmap", + ) + op.create_table( + "osm_polygon", + sa.Column("osm_id", sa.BigInteger(), nullable=True), + sa.Column("access", sa.Text(), nullable=True), + sa.Column("addr:housename", sa.Text(), nullable=True), + sa.Column("addr:housenumber", sa.Text(), nullable=True), + sa.Column("addr:interpolation", sa.Text(), nullable=True), + sa.Column("admin_level", sa.Text(), nullable=True), + sa.Column("aerialway", sa.Text(), nullable=True), + sa.Column("aeroway", sa.Text(), nullable=True), + sa.Column("amenity", sa.Text(), nullable=True), + sa.Column("area", sa.Text(), nullable=True), + sa.Column("barrier", sa.Text(), nullable=True), + sa.Column("bicycle", sa.Text(), nullable=True), + sa.Column("brand", sa.Text(), nullable=True), + sa.Column("bridge", sa.Text(), nullable=True), + sa.Column("boundary", sa.Text(), nullable=True), + sa.Column("building", sa.Text(), nullable=True), + sa.Column("construction", sa.Text(), nullable=True), + sa.Column("covered", sa.Text(), nullable=True), + sa.Column("culvert", sa.Text(), nullable=True), + sa.Column("cutting", sa.Text(), nullable=True), + sa.Column("denomination", sa.Text(), nullable=True), + sa.Column("disused", sa.Text(), nullable=True), + sa.Column("embankment", sa.Text(), nullable=True), + sa.Column("foot", sa.Text(), nullable=True), + sa.Column("generator:source", sa.Text(), nullable=True), + sa.Column("harbour", sa.Text(), nullable=True), + sa.Column("highway", sa.Text(), nullable=True), + sa.Column("historic", sa.Text(), nullable=True), + sa.Column("horse", sa.Text(), nullable=True), + sa.Column("intermittent", sa.Text(), nullable=True), + sa.Column("junction", sa.Text(), nullable=True), + sa.Column("landuse", sa.Text(), nullable=True), + sa.Column("layer", sa.Text(), nullable=True), + sa.Column("leisure", sa.Text(), nullable=True), + sa.Column("line", sa.Text(), nullable=True), + sa.Column("lock", sa.Text(), nullable=True), + sa.Column("man_made", sa.Text(), nullable=True), + sa.Column("military", sa.Text(), nullable=True), + sa.Column("motorcar", sa.Text(), nullable=True), + sa.Column("name", sa.Text(), nullable=True), + sa.Column("natural", sa.Text(), nullable=True), + sa.Column("office", sa.Text(), nullable=True), + sa.Column("oneway", sa.Text(), nullable=True), + sa.Column("operator", sa.Text(), nullable=True), + sa.Column("place", sa.Text(), nullable=True), + sa.Column("population", sa.Text(), nullable=True), + sa.Column("power", sa.Text(), nullable=True), + sa.Column("power_source", sa.Text(), nullable=True), + sa.Column("public_transport", sa.Text(), nullable=True), + sa.Column("railway", sa.Text(), nullable=True), + sa.Column("ref", sa.Text(), nullable=True), + sa.Column("religion", sa.Text(), nullable=True), + sa.Column("route", sa.Text(), nullable=True), + sa.Column("service", sa.Text(), nullable=True), + sa.Column("sidewalk", sa.Text(), nullable=True), + sa.Column("shop", sa.Text(), nullable=True), + sa.Column("sport", sa.Text(), nullable=True), + sa.Column("surface", sa.Text(), nullable=True), + sa.Column("toll", sa.Text(), nullable=True), + sa.Column("tourism", sa.Text(), nullable=True), + sa.Column("tower:type", sa.Text(), nullable=True), + sa.Column("tracktype", sa.Text(), nullable=True), + sa.Column("tunnel", sa.Text(), nullable=True), + sa.Column("water", sa.Text(), nullable=True), + sa.Column("waterway", sa.Text(), nullable=True), + sa.Column("wetland", sa.Text(), nullable=True), + sa.Column("width", sa.Text(), nullable=True), + sa.Column("wood", sa.Text(), nullable=True), + sa.Column("z_order", sa.Integer(), nullable=True), + sa.Column("way_area", sa.Float(), nullable=True), + sa.Column("abandoned:aeroway", sa.Text(), nullable=True), + sa.Column("abandoned:amenity", sa.Text(), nullable=True), + sa.Column("abandoned:building", sa.Text(), nullable=True), + sa.Column("abandoned:landuse", sa.Text(), nullable=True), + sa.Column("abandoned:power", sa.Text(), nullable=True), + sa.Column("area:highway", sa.Text(), nullable=True), + sa.Column( + "tags", postgresql.HSTORE(text_type=sa.Text()), nullable=True + ), + sa.Column( + "geom", geoalchemy2.types.Geometry(srid=3857), nullable=True + ), + sa.Column( + "gid", + sa.Integer(), + server_default=sa.text( + "nextval('openstreetmap.osm_polygon_gid_seq'::regclass)" + ), + nullable=False, + ), + sa.PrimaryKeyConstraint("gid"), + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_polygon_geom"), + "osm_polygon", + ["geom"], + unique=False, + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_polygon_tags"), + "osm_polygon", + ["tags"], + unique=False, + schema="openstreetmap", + ) + op.create_table( + "osm_rels", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("way_off", sa.SmallInteger(), nullable=True), + sa.Column("rel_off", sa.SmallInteger(), nullable=True), + sa.Column("parts", sa.ARRAY(sa.BigInteger()), nullable=True), + sa.Column("members", sa.ARRAY(sa.Text()), nullable=True), + sa.Column("tags", sa.ARRAY(sa.Text()), nullable=True), + sa.PrimaryKeyConstraint("id"), + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_rels_parts"), + "osm_rels", + ["parts"], + unique=False, + schema="openstreetmap", + ) + op.create_table( + "osm_roads", + sa.Column("osm_id", sa.BigInteger(), nullable=True), + sa.Column("access", sa.Text(), nullable=True), + sa.Column("addr:housename", sa.Text(), nullable=True), + sa.Column("addr:housenumber", sa.Text(), nullable=True), + sa.Column("addr:interpolation", sa.Text(), nullable=True), + sa.Column("admin_level", sa.Text(), nullable=True), + sa.Column("aerialway", sa.Text(), nullable=True), + sa.Column("aeroway", sa.Text(), nullable=True), + sa.Column("amenity", sa.Text(), nullable=True), + sa.Column("area", sa.Text(), nullable=True), + sa.Column("barrier", sa.Text(), nullable=True), + sa.Column("bicycle", sa.Text(), nullable=True), + sa.Column("brand", sa.Text(), nullable=True), + sa.Column("bridge", sa.Text(), nullable=True), + sa.Column("boundary", sa.Text(), nullable=True), + sa.Column("building", sa.Text(), nullable=True), + sa.Column("construction", sa.Text(), nullable=True), + sa.Column("covered", sa.Text(), nullable=True), + sa.Column("culvert", sa.Text(), nullable=True), + sa.Column("cutting", sa.Text(), nullable=True), + sa.Column("denomination", sa.Text(), nullable=True), + sa.Column("disused", sa.Text(), nullable=True), + sa.Column("embankment", sa.Text(), nullable=True), + sa.Column("foot", sa.Text(), nullable=True), + sa.Column("generator:source", sa.Text(), nullable=True), + sa.Column("harbour", sa.Text(), nullable=True), + sa.Column("highway", sa.Text(), nullable=True), + sa.Column("historic", sa.Text(), nullable=True), + sa.Column("horse", sa.Text(), nullable=True), + sa.Column("intermittent", sa.Text(), nullable=True), + sa.Column("junction", sa.Text(), nullable=True), + sa.Column("landuse", sa.Text(), nullable=True), + sa.Column("layer", sa.Text(), nullable=True), + sa.Column("leisure", sa.Text(), nullable=True), + sa.Column("line", sa.Text(), nullable=True), + sa.Column("lock", sa.Text(), nullable=True), + sa.Column("man_made", sa.Text(), nullable=True), + sa.Column("military", sa.Text(), nullable=True), + sa.Column("motorcar", sa.Text(), nullable=True), + sa.Column("name", sa.Text(), nullable=True), + sa.Column("natural", sa.Text(), nullable=True), + sa.Column("office", sa.Text(), nullable=True), + sa.Column("oneway", sa.Text(), nullable=True), + sa.Column("operator", sa.Text(), nullable=True), + sa.Column("place", sa.Text(), nullable=True), + sa.Column("population", sa.Text(), nullable=True), + sa.Column("power", sa.Text(), nullable=True), + sa.Column("power_source", sa.Text(), nullable=True), + sa.Column("public_transport", sa.Text(), nullable=True), + sa.Column("railway", sa.Text(), nullable=True), + sa.Column("ref", sa.Text(), nullable=True), + sa.Column("religion", sa.Text(), nullable=True), + sa.Column("route", sa.Text(), nullable=True), + sa.Column("service", sa.Text(), nullable=True), + sa.Column("sidewalk", sa.Text(), nullable=True), + sa.Column("shop", sa.Text(), nullable=True), + sa.Column("sport", sa.Text(), nullable=True), + sa.Column("surface", sa.Text(), nullable=True), + sa.Column("toll", sa.Text(), nullable=True), + sa.Column("tourism", sa.Text(), nullable=True), + sa.Column("tower:type", sa.Text(), nullable=True), + sa.Column("tracktype", sa.Text(), nullable=True), + sa.Column("tunnel", sa.Text(), nullable=True), + sa.Column("water", sa.Text(), nullable=True), + sa.Column("waterway", sa.Text(), nullable=True), + sa.Column("wetland", sa.Text(), nullable=True), + sa.Column("width", sa.Text(), nullable=True), + sa.Column("wood", sa.Text(), nullable=True), + sa.Column("z_order", sa.Integer(), nullable=True), + sa.Column("way_area", sa.Float(), nullable=True), + sa.Column("abandoned:aeroway", sa.Text(), nullable=True), + sa.Column("abandoned:amenity", sa.Text(), nullable=True), + sa.Column("abandoned:building", sa.Text(), nullable=True), + sa.Column("abandoned:landuse", sa.Text(), nullable=True), + sa.Column("abandoned:power", sa.Text(), nullable=True), + sa.Column("area:highway", sa.Text(), nullable=True), + sa.Column( + "tags", postgresql.HSTORE(text_type=sa.Text()), nullable=True + ), + sa.Column( + "geom", + geoalchemy2.types.Geometry(geometry_type="LINESTRING", srid=3857), + nullable=True, + ), + sa.Column( + "gid", + sa.Integer(), + server_default=sa.text( + "nextval('openstreetmap.osm_roads_gid_seq'::regclass)" + ), + nullable=False, + ), + sa.PrimaryKeyConstraint("gid"), + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_roads_geom"), + "osm_roads", + ["geom"], + unique=False, + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_roads_tags"), + "osm_roads", + ["tags"], + unique=False, + schema="openstreetmap", + ) + op.create_table( + "osm_ways", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("nodes", sa.ARRAY(sa.BigInteger()), nullable=False), + sa.Column("tags", sa.ARRAY(sa.Text()), nullable=True), + sa.PrimaryKeyConstraint("id"), + schema="openstreetmap", + ) + op.create_index( + op.f("ix_openstreetmap_osm_ways_nodes"), + "osm_ways", + ["nodes"], + unique=False, + schema="openstreetmap", + ) + op.drop_table("spatial_ref_sys") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "spatial_ref_sys", + sa.Column("srid", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column( + "auth_name", + sa.VARCHAR(length=256), + autoincrement=False, + nullable=True, + ), + sa.Column( + "auth_srid", sa.INTEGER(), autoincrement=False, nullable=True + ), + sa.Column( + "srtext", + sa.VARCHAR(length=2048), + autoincrement=False, + nullable=True, + ), + sa.Column( + "proj4text", + sa.VARCHAR(length=2048), + autoincrement=False, + nullable=True, + ), + sa.CheckConstraint( + "(srid > 0) AND (srid <= 998999)", + name="spatial_ref_sys_srid_check", + ), + sa.PrimaryKeyConstraint("srid", name="spatial_ref_sys_pkey"), + ) + op.drop_index( + op.f("ix_openstreetmap_osm_ways_nodes"), + table_name="osm_ways", + schema="openstreetmap", + ) + op.drop_table("osm_ways", schema="openstreetmap") + op.drop_index( + op.f("ix_openstreetmap_osm_roads_tags"), + table_name="osm_roads", + schema="openstreetmap", + ) + op.drop_index( + op.f("ix_openstreetmap_osm_roads_geom"), + table_name="osm_roads", + schema="openstreetmap", + ) + op.drop_table("osm_roads", schema="openstreetmap") + op.drop_index( + op.f("ix_openstreetmap_osm_rels_parts"), + table_name="osm_rels", + schema="openstreetmap", + ) + op.drop_table("osm_rels", schema="openstreetmap") + op.drop_index( + op.f("ix_openstreetmap_osm_polygon_tags"), + table_name="osm_polygon", + schema="openstreetmap", + ) + op.drop_index( + op.f("ix_openstreetmap_osm_polygon_geom"), + table_name="osm_polygon", + schema="openstreetmap", + ) + op.drop_table("osm_polygon", schema="openstreetmap") + op.drop_index( + op.f("ix_openstreetmap_osm_point_tags"), + table_name="osm_point", + schema="openstreetmap", + ) + op.drop_index( + op.f("ix_openstreetmap_osm_point_geom"), + table_name="osm_point", + schema="openstreetmap", + ) + op.drop_table("osm_point", schema="openstreetmap") + op.drop_table("osm_nodes", schema="openstreetmap") + op.drop_index( + op.f("ix_openstreetmap_osm_line_tags"), + table_name="osm_line", + schema="openstreetmap", + ) + op.drop_index( + op.f("ix_openstreetmap_osm_line_geom"), + table_name="osm_line", + schema="openstreetmap", + ) + op.drop_table("osm_line", schema="openstreetmap") + # ### end Alembic commands ###