From 18172d6d85a190098bc114a1166ceeb7a13ba665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 4 Feb 2024 14:10:50 +0200 Subject: [PATCH 01/29] Adapted to Asphalt 5 --- docs/api.rst | 18 --- docs/conf.py | 4 +- docs/modules/component.rst | 5 + docs/modules/utils.rst | 5 + docs/py-modindex.rst | 2 + docs/usage.rst | 2 +- pyproject.toml | 6 +- src/asphalt/sqlalchemy/__init__.py | 13 ++ .../{component.py => _component.py} | 134 ++++++++++-------- .../sqlalchemy/{utils.py => _utils.py} | 0 tests/conftest.py | 20 +-- tests/test_component.py | 24 ++-- tests/test_testing_recipe.py | 8 +- tests/test_utils.py | 2 +- 14 files changed, 131 insertions(+), 112 deletions(-) delete mode 100644 docs/api.rst create mode 100644 docs/modules/component.rst create mode 100644 docs/modules/utils.rst create mode 100644 docs/py-modindex.rst rename src/asphalt/sqlalchemy/{component.py => _component.py} (69%) rename src/asphalt/sqlalchemy/{utils.py => _utils.py} (100%) diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index cc33c4e..0000000 --- a/docs/api.rst +++ /dev/null @@ -1,18 +0,0 @@ -API reference -============= - -Components ----------- - -.. py:currentmodule:: asphalt.sqlalchemy.component - -.. autoclass:: SQLAlchemyComponent - -Utilities ---------- - -.. py:currentmodule:: asphalt.sqlalchemy.utils - -.. autofunction:: clear_database -.. autofunction:: clear_async_database -.. autofunction:: apply_sqlite_hacks diff --git a/docs/conf.py b/docs/conf.py index e5c43f8..8a356fb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from importlib.metadata import version +import importlib.metadata from packaging.version import parse @@ -17,7 +17,7 @@ author = "Alex Grönholm" copyright = "2015, " + author -v = parse(version(project)) +v = parse(importlib.metadata.version(project)) version = v.base_version release = v.public diff --git a/docs/modules/component.rst b/docs/modules/component.rst new file mode 100644 index 0000000..33a4451 --- /dev/null +++ b/docs/modules/component.rst @@ -0,0 +1,5 @@ +:mod:`asphalt.sqlalchemy.component` +=================================== + +.. module:: asphalt.sqlalchemy.component +.. autoclass:: asphalt.sqlalchemy.component.SQLAlchemyComponent diff --git a/docs/modules/utils.rst b/docs/modules/utils.rst new file mode 100644 index 0000000..510ab61 --- /dev/null +++ b/docs/modules/utils.rst @@ -0,0 +1,5 @@ +:mod:`asphalt.sqlalchemy.utils` +=============================== + +.. automodule:: asphalt.sqlalchemy.utils + :members: diff --git a/docs/py-modindex.rst b/docs/py-modindex.rst new file mode 100644 index 0000000..3cdb8ca --- /dev/null +++ b/docs/py-modindex.rst @@ -0,0 +1,2 @@ +API reference +============= diff --git a/docs/usage.rst b/docs/usage.rst index 60d6013..0e2638c 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -46,7 +46,7 @@ you can skip this section. For those having to use engines for which no async counterpart is available, read on. Synchronous database connections in an asynchronous application pose a problem because -running operations against a database will block the event loop, and thus needs to be +running operations against a database will block the event loop, and thus need to be wrapped in worker threads. Another thing to watch out for is lazy loading of relationships and deferred columns which triggers implicit queries when those attributes are accessed. diff --git a/pyproject.toml b/pyproject.toml index 5f79d66..59f8afc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - "asphalt ~= 4.9", + "asphalt ~= 4.12", "SQLAlchemy[asyncio] >= 2.0", ] dynamic = ["version"] @@ -45,7 +45,7 @@ test = [ "pymysql", "psycopg >= 3.1; platform_python_implementation == 'CPython'", "pytest >= 7.4", - "pytest-lazy-fixture", + "pytest-lazy-fixtures", ] doc = [ "Sphinx >= 7.0", @@ -55,7 +55,7 @@ doc = [ ] [project.entry-points."asphalt.components"] -sqlalchemy = "asphalt.sqlalchemy.component:SQLAlchemyComponent" +sqlalchemy = "asphalt.sqlalchemy._component:SQLAlchemyComponent" [tool.setuptools_scm] version_scheme = "post-release" diff --git a/src/asphalt/sqlalchemy/__init__.py b/src/asphalt/sqlalchemy/__init__.py index e69de29..afa721f 100644 --- a/src/asphalt/sqlalchemy/__init__.py +++ b/src/asphalt/sqlalchemy/__init__.py @@ -0,0 +1,13 @@ +from typing import Any + +from ._component import SQLAlchemyComponent as SQLAlchemyComponent +from ._utils import apply_sqlite_hacks as apply_sqlite_hacks +from ._utils import clear_async_database as clear_async_database +from ._utils import clear_database as clear_database + +# Re-export imports, so they look like they live directly in this package +key: str +value: Any +for key, value in list(locals().items()): + if getattr(value, "__module__", "").startswith("asphalt.sqlalchemy."): + value.__module__ = __name__ diff --git a/src/asphalt/sqlalchemy/component.py b/src/asphalt/sqlalchemy/_component.py similarity index 69% rename from src/asphalt/sqlalchemy/component.py rename to src/asphalt/sqlalchemy/_component.py index 52d73e7..eee5554 100644 --- a/src/asphalt/sqlalchemy/component.py +++ b/src/asphalt/sqlalchemy/_component.py @@ -1,22 +1,21 @@ from __future__ import annotations import logging -from asyncio import get_running_loop -from collections.abc import AsyncGenerator, Callable -from concurrent.futures import ThreadPoolExecutor -from contextvars import copy_context +import sys +from collections.abc import Callable from inspect import isawaitable from typing import Any, cast +from anyio import CapacityLimiter, to_thread from asphalt.core import ( Component, Context, - context_teardown, + GeneratedResource, qualified_name, resolve_reference, ) -from asphalt.sqlalchemy.utils import apply_sqlite_hacks +from asphalt.sqlalchemy._utils import apply_sqlite_hacks from sqlalchemy.engine import Connection, Engine, create_engine from sqlalchemy.engine.url import URL, make_url from sqlalchemy.exc import InvalidRequestError @@ -82,12 +81,9 @@ class SQLAlchemyComponent(Component): :param resource_name: name space for the database resources """ - commit_executor: ThreadPoolExecutor - engine: Engine | AsyncEngine + _engine: Engine | AsyncEngine _bind: Connection | Engine - _sessionmaker: sessionmaker _async_bind: AsyncConnection | AsyncEngine - _async_sessionmaker: async_sessionmaker def __init__( self, @@ -97,13 +93,13 @@ def __init__( prefer_async: bool = True, engine_args: dict[str, Any] | None = None, session_args: dict[str, Any] | None = None, - commit_executor_workers: int = 5, + commit_executor_workers: int = 50, ready_callback: Callable[[Engine, sessionmaker], Any] | str | None = None, poolclass: str | type[Pool] | None = None, resource_name: str = "default", ): self.resource_name = resource_name - self.commit_executor_workers = commit_executor_workers + self.commit_thread_limiter = CapacityLimiter(commit_executor_workers) self.ready_callback = resolve_reference(ready_callback) engine_args = engine_args or {} session_args = session_args or {} @@ -112,14 +108,14 @@ def __init__( if bind: if isinstance(bind, Connection): self._bind = bind - self.engine = bind.engine + self._engine = bind.engine elif isinstance(bind, AsyncConnection): self._async_bind = bind - self.engine = bind.engine + self._engine = bind.engine elif isinstance(bind, Engine): - self.engine = self._bind = bind + self._engine = self._bind = bind elif isinstance(bind, AsyncEngine): - self.engine = self._async_bind = bind + self._engine = self._async_bind = bind else: raise TypeError(f"Incompatible bind argument: {qualified_name(bind)}") else: @@ -143,27 +139,27 @@ def __init__( pool_class = cast("type[Pool]", poolclass) if prefer_async: try: - self.engine = self._async_bind = create_async_engine( + self._engine = self._async_bind = create_async_engine( url, poolclass=pool_class, **engine_args ) except InvalidRequestError: - self.engine = self._bind = create_engine( + self._engine = self._bind = create_engine( url, poolclass=pool_class, **engine_args ) else: try: - self.engine = self._bind = create_engine( + self._engine = self._bind = create_engine( url, poolclass=pool_class, **engine_args ) except InvalidRequestError: - self.engine = self._async_bind = create_async_engine( + self._engine = self._async_bind = create_async_engine( url, poolclass=pool_class, **engine_args ) if url.get_dialect().name == "sqlite": - apply_sqlite_hacks(self.engine) + apply_sqlite_hacks(self._engine) - if isinstance(self.engine, AsyncEngine): + if isinstance(self._engine, AsyncEngine): # This is needed for listening to ORM events when async sessions are used self._sessionmaker = sessionmaker() self._async_sessionmaker = async_sessionmaker( @@ -174,31 +170,29 @@ def __init__( else: self._sessionmaker = sessionmaker(bind=self._bind, **session_args) - def create_session(self, ctx: Context) -> Session: - async def teardown_session(exception: BaseException | None) -> None: + def create_session(self, ctx: Context) -> GeneratedResource[Session]: + async def teardown_session() -> None: try: if session.in_transaction(): - context = copy_context() - if exception is None: - await get_running_loop().run_in_executor( - self.commit_executor, context.run, session.commit + if sys.exc_info()[1] is None: + await to_thread.run_sync( + session.commit, limiter=self.commit_thread_limiter ) else: - await get_running_loop().run_in_executor( - self.commit_executor, context.run, session.rollback + await to_thread.run_sync( + session.rollback, limiter=self.commit_thread_limiter ) finally: session.close() session = self._sessionmaker() - ctx.add_teardown_callback(teardown_session, pass_exception=True) - return session + return GeneratedResource(session, teardown_session) - def create_async_session(self, ctx: Context) -> AsyncSession: - async def teardown_session(exception: BaseException | None) -> None: + def create_async_session(self, ctx: Context) -> GeneratedResource[AsyncSession]: + async def teardown_session() -> None: try: if session.in_transaction(): - if exception is None: + if sys.exc_info()[1] is None: await session.commit() else: await session.rollback() @@ -206,26 +200,42 @@ async def teardown_session(exception: BaseException | None) -> None: await session.close() session: AsyncSession = self._async_sessionmaker() - ctx.add_teardown_callback(teardown_session, pass_exception=True) - return session + return GeneratedResource(session, teardown_session) - @context_teardown - async def start(self, ctx: Context) -> AsyncGenerator[None, Exception | None]: + async def start(self, ctx: Context) -> None: bind: Connection | Engine | AsyncConnection | AsyncEngine - if isinstance(self.engine, AsyncEngine): + if isinstance(self._engine, AsyncEngine): if self.ready_callback: retval = self.ready_callback(self._async_bind, self._sessionmaker) if isawaitable(retval): await retval bind = self._async_bind - ctx.add_resource(self.engine, self.resource_name) - ctx.add_resource(self._sessionmaker, self.resource_name) - ctx.add_resource(self._async_sessionmaker, self.resource_name) - ctx.add_resource_factory( + if isinstance(bind, AsyncEngine): + teardown_callback = self._engine.dispose + else: + teardown_callback = None + + await ctx.add_resource( + self._engine, + self.resource_name, + description="SQLAlchemy engine (asynchronous)", + teardown_callback=teardown_callback, + ) + await ctx.add_resource( + self._sessionmaker, + self.resource_name, + description="SQLAlchemy session factory (synchronous)", + ) + await ctx.add_resource( + self._async_sessionmaker, + self.resource_name, + description="SQLAlchemy session factory (asynchronous)", + ) + await ctx.add_resource_factory( self.create_async_session, - [AsyncSession], self.resource_name, + description="SQLAlchemy session (asynchronous)", ) else: if self.ready_callback: @@ -233,16 +243,27 @@ async def start(self, ctx: Context) -> AsyncGenerator[None, Exception | None]: if isawaitable(retval): await retval - self.commit_executor = ThreadPoolExecutor(self.commit_executor_workers) - ctx.add_teardown_callback(self.commit_executor.shutdown) - bind = self._bind - ctx.add_resource(self.engine, self.resource_name) - ctx.add_resource(self._sessionmaker, self.resource_name) - ctx.add_resource_factory( + if isinstance(bind, AsyncEngine): + teardown_callback = self._engine.dispose + else: + teardown_callback = None + + await ctx.add_resource( + self._engine, + self.resource_name, + description="SQLAlchemy engine (synchronous)", + teardown_callback=teardown_callback, + ) + await ctx.add_resource( + self._sessionmaker, + self.resource_name, + description="SQLAlchemy session factory (synchronous)", + ) + await ctx.add_resource_factory( self.create_session, - [Session], self.resource_name, + description="SQLAlchemy session (synchronous)", ) logger.info( @@ -251,12 +272,3 @@ async def start(self, ctx: Context) -> AsyncGenerator[None, Exception | None]: bind.dialect.name, bind.dialect.driver, ) - - yield - - if isinstance(bind, Engine): - bind.dispose() - elif isinstance(bind, AsyncEngine): - await bind.dispose() - - logger.info("SQLAlchemy resources (%s) shut down", self.resource_name) diff --git a/src/asphalt/sqlalchemy/utils.py b/src/asphalt/sqlalchemy/_utils.py similarity index 100% rename from src/asphalt/sqlalchemy/utils.py rename to src/asphalt/sqlalchemy/_utils.py diff --git a/tests/conftest.py b/tests/conftest.py index e644f42..9720460 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,11 +7,11 @@ import pytest from _pytest.fixtures import SubRequest from pytest import TempPathFactory -from pytest_lazyfixture import lazy_fixture +from pytest_lazy_fixtures import lf from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from sqlalchemy.future import Engine, create_engine -from asphalt.sqlalchemy.utils import apply_sqlite_hacks +from asphalt.sqlalchemy import apply_sqlite_hacks @pytest.fixture(scope="session") @@ -118,10 +118,10 @@ async def asyncmy_engine(asyncmy_url: str) -> AsyncGenerator[AsyncEngine, Any]: @pytest.fixture( params=[ - lazy_fixture("sqlite_memory_engine"), - lazy_fixture("sqlite_file_engine"), - lazy_fixture("pymysql_engine"), - lazy_fixture("psycopg_engine"), + lf("sqlite_memory_engine"), + lf("sqlite_file_engine"), + lf("pymysql_engine"), + lf("psycopg_engine"), ], scope="session", ) @@ -131,10 +131,10 @@ def sync_engine(request: SubRequest) -> Engine: @pytest.fixture( params=[ - lazy_fixture("aiosqlite_memory_engine"), - lazy_fixture("aiosqlite_file_engine"), - lazy_fixture("psycopg_async_engine"), - lazy_fixture("asyncmy_engine"), + lf("aiosqlite_memory_engine"), + lf("aiosqlite_file_engine"), + lf("psycopg_async_engine"), + lf("asyncmy_engine"), ], scope="session", ) diff --git a/tests/test_component.py b/tests/test_component.py index 05a5722..1ab167d 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -1,14 +1,13 @@ from __future__ import annotations import gc -from contextlib import ExitStack +from contextlib import AsyncExitStack from pathlib import Path from threading import Thread, current_thread from typing import Any import pytest -from asphalt.core import NoCurrentContext, current_context -from asphalt.core.context import Context, get_resource +from asphalt.core import Context, NoCurrentContext, current_context, get_resource from pytest import FixtureRequest from sqlalchemy.engine.url import URL from sqlalchemy.event import listen, remove @@ -23,8 +22,7 @@ from sqlalchemy.pool import AsyncAdaptedQueuePool, NullPool, QueuePool from sqlalchemy.sql import text -from asphalt.sqlalchemy.component import SQLAlchemyComponent -from asphalt.sqlalchemy.utils import clear_async_database, clear_database +from asphalt.sqlalchemy import SQLAlchemyComponent, clear_async_database, clear_database from .model import Person @@ -217,14 +215,14 @@ async def test_finish_commit(raise_exception: bool, tmp_path: Path) -> None: component = SQLAlchemyComponent( url={"drivername": "sqlite", "database": str(db_path)}, ) - with ExitStack() as stack: - async with Context() as ctx: - await component.start(ctx) - session = ctx.require_resource(Session) - session.execute(text("INSERT INTO foo (id) VALUES(3)")) - if raise_exception: - stack.enter_context(pytest.raises(Exception, match="dummy")) - raise Exception("dummy") + async with AsyncExitStack() as stack: + ctx = await stack.enter_async_context(Context()) + await component.start(ctx) + session = ctx.require_resource(Session) + session.execute(text("INSERT INTO foo (id) VALUES(3)")) + if raise_exception: + stack.enter_context(pytest.raises(Exception, match="dummy")) + raise Exception("dummy") rows = connection.execute(text("SELECT * FROM foo")).fetchall() assert len(rows) == (0 if raise_exception else 1) diff --git a/tests/test_testing_recipe.py b/tests/test_testing_recipe.py index 35c1a40..d6dbb3c 100644 --- a/tests/test_testing_recipe.py +++ b/tests/test_testing_recipe.py @@ -15,7 +15,7 @@ from sqlalchemy.orm import Session from sqlalchemy.sql.expression import delete, func, select -from asphalt.sqlalchemy.utils import clear_async_database, clear_database +from asphalt.sqlalchemy import clear_async_database, clear_database from .model import Base, Person @@ -62,8 +62,10 @@ async def test_rollback( ) -> None: # Simulate a rollback happening in a subcontext async with Context() as root_ctx: + print("Root context is", hex(id(root_ctx))) await root_component.start(root_ctx) async with Context() as ctx: + print("Subcontext is", hex(id(ctx))) session = ctx.require_resource(Session) try: # No value for a non-nullable column => IntegrityError! @@ -76,8 +78,8 @@ async def test_rollback( session.add(Person(name="Works now!")) session.flush() - # The context is gone, but the extra Person should still be around - assert dbsession.scalar(func.count(Person.id)) == 2 + # The context is gone, but the extra Person should still be around + assert dbsession.scalar(func.count(Person.id)) == 2 async def test_add_person( self, dbsession: Session, root_component: ContainerComponent diff --git a/tests/test_utils.py b/tests/test_utils.py index 47b9b90..f3c0d78 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,7 +9,7 @@ from sqlalchemy.sql.schema import Column, ForeignKey, MetaData, Table from sqlalchemy.sql.sqltypes import Integer -from asphalt.sqlalchemy.utils import clear_database +from asphalt.sqlalchemy._utils import clear_database @pytest.fixture From cf9b20478cbaa0d2f0edde73108be3a69639d8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 4 Feb 2024 15:02:38 +0200 Subject: [PATCH 02/29] Removed unused test dependency --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 59f8afc..598ad5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ test = [ "aiosqlite", "anyio >= 4.2", "asyncmy; platform_python_implementation == 'CPython'", - "asyncpg; platform_python_implementation == 'CPython'", "coverage >= 7", "pymysql", "psycopg >= 3.1; platform_python_implementation == 'CPython'", From 97fd3ecd3722d8179cea3b9d2c98fce04df16cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 4 Feb 2024 15:02:48 +0200 Subject: [PATCH 03/29] Fixed tests on PyPy3 --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9720460..ed0dd38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,7 @@ def anyio_backend() -> str: @pytest.fixture(scope="session") def psycopg_url() -> str: # type: ignore[return] - pytest.importorskip("asyncmy", reason="asyncmy is not available") + pytest.importorskip("psycopg", reason="psycopg is not available") try: return os.environ["POSTGRESQL_URL"] except KeyError: From 13d2ba388c04873f37cdf9cb1d30016e54394a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 4 Feb 2024 15:59:45 +0200 Subject: [PATCH 04/29] Enabled the tests to run on Trio where possible --- pyproject.toml | 2 +- tests/conftest.py | 68 ++++++++++++++++++++++++++++------------- tests/test_component.py | 18 ++++++----- 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 598ad5d..1b8b303 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ Homepage = "https://github.com/asphalt-framework/asphalt-sqlalchemy" [project.optional-dependencies] test = [ "aiosqlite", - "anyio >= 4.2", + "anyio[trio] >= 4.2", "asyncmy; platform_python_implementation == 'CPython'", "coverage >= 7", "pymysql", diff --git a/tests/conftest.py b/tests/conftest.py index ed0dd38..961936c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,12 +14,15 @@ from asphalt.sqlalchemy import apply_sqlite_hacks -@pytest.fixture(scope="session") -def anyio_backend() -> str: - return "asyncio" +@pytest.fixture +def aiosqlite_memory_url(anyio_backend_name: str) -> str: + if anyio_backend_name != "asyncio": + pytest.skip("Async SQLAlchemy only works with asyncio for now") + return "sqlite+aiosqlite:///:memory:" -@pytest.fixture(scope="session") + +@pytest.fixture def psycopg_url() -> str: # type: ignore[return] pytest.importorskip("psycopg", reason="psycopg is not available") try: @@ -28,7 +31,15 @@ def psycopg_url() -> str: # type: ignore[return] pytest.skip("POSTGRESQL_URL environment variable is not set") -@pytest.fixture(scope="session") +@pytest.fixture +def psycopg_url_async(psycopg_url: str, anyio_backend_name: str) -> str: + if anyio_backend_name != "asyncio": + pytest.skip("Async SQLAlchemy only works with asyncio for now") + + return psycopg_url + + +@pytest.fixture def mysql_url() -> str: # type: ignore[return] try: return os.environ["MYSQL_URL"] @@ -36,27 +47,27 @@ def mysql_url() -> str: # type: ignore[return] pytest.skip("MYSQL_URL environment variable is not set") -@pytest.fixture(scope="session") -def asyncmy_url(mysql_url: str) -> str: +@pytest.fixture +def asyncmy_url(mysql_url: str, anyio_backend_name: str) -> str: pytest.importorskip("asyncmy", reason="asyncmy is not available") return mysql_url.replace("pymysql", "asyncmy") -@pytest.fixture(scope="session") +@pytest.fixture def pymysql_engine(mysql_url: str) -> Generator[Engine, Any, None]: engine = create_engine(mysql_url) yield engine engine.dispose() -@pytest.fixture(scope="session") +@pytest.fixture def psycopg_engine(psycopg_url: str) -> Generator[Engine, Any, None]: engine = create_engine(psycopg_url, echo=True) yield engine engine.dispose() -@pytest.fixture(scope="session") +@pytest.fixture def sqlite_memory_engine() -> Generator[Engine, Any, None]: engine = create_engine( "sqlite:///:memory:", connect_args=dict(check_same_thread=False) @@ -66,7 +77,7 @@ def sqlite_memory_engine() -> Generator[Engine, Any, None]: engine.dispose() -@pytest.fixture(scope="session") +@pytest.fixture def sqlite_file_engine( tmp_path_factory: TempPathFactory, ) -> Generator[Engine, Any, None]: @@ -81,18 +92,23 @@ def sqlite_file_engine( db_path.unlink() -@pytest.fixture(scope="session") -async def aiosqlite_memory_engine() -> AsyncGenerator[AsyncEngine, Any]: - engine = create_async_engine("sqlite+aiosqlite:///:memory:") +@pytest.fixture +async def aiosqlite_memory_engine( + aiosqlite_memory_url: str +) -> AsyncGenerator[AsyncEngine, Any]: + engine = create_async_engine(aiosqlite_memory_url) apply_sqlite_hacks(engine) yield engine await engine.dispose() -@pytest.fixture(scope="session") +@pytest.fixture async def aiosqlite_file_engine( - tmp_path_factory: TempPathFactory, + tmp_path_factory: TempPathFactory, anyio_backend_name: str ) -> AsyncGenerator[AsyncEngine, Any]: + if anyio_backend_name != "asyncio": + pytest.skip("Async SQLAlchemy only works with asyncio for now") + db_path = tmp_path_factory.mktemp("asphalt-sqlalchemy") / "test.db" engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}") apply_sqlite_hacks(engine) @@ -102,15 +118,25 @@ async def aiosqlite_file_engine( db_path.unlink() -@pytest.fixture(scope="session") -async def psycopg_async_engine(psycopg_url: str) -> AsyncGenerator[AsyncEngine, Any]: +@pytest.fixture +async def psycopg_async_engine( + psycopg_url: str, anyio_backend_name: str +) -> AsyncGenerator[AsyncEngine, Any]: + if anyio_backend_name != "asyncio": + pytest.skip("Async SQLAlchemy only works with asyncio for now") + engine = create_async_engine(psycopg_url) yield engine await engine.dispose() -@pytest.fixture(scope="session") -async def asyncmy_engine(asyncmy_url: str) -> AsyncGenerator[AsyncEngine, Any]: +@pytest.fixture +async def asyncmy_engine( + asyncmy_url: str, anyio_backend_name: str +) -> AsyncGenerator[AsyncEngine, Any]: + if anyio_backend_name != "asyncio": + pytest.skip("Async SQLAlchemy only works with asyncio for now") + engine = create_async_engine(asyncmy_url) yield engine await engine.dispose() @@ -123,7 +149,6 @@ async def asyncmy_engine(asyncmy_url: str) -> AsyncGenerator[AsyncEngine, Any]: lf("pymysql_engine"), lf("psycopg_engine"), ], - scope="session", ) def sync_engine(request: SubRequest) -> Engine: return cast(Engine, request.param) @@ -136,7 +161,6 @@ def sync_engine(request: SubRequest) -> Engine: lf("psycopg_async_engine"), lf("asyncmy_engine"), ], - scope="session", ) async def async_engine(request: SubRequest) -> AsyncEngine: return cast(AsyncEngine, request.param) diff --git a/tests/test_component.py b/tests/test_component.py index 1ab167d..effd9ad 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -117,9 +117,9 @@ async def test_bind_sync_connection() -> None: connection.close() -async def test_bind_async_connection() -> None: +async def test_bind_async_connection(aiosqlite_memory_url: str) -> None: """Test that a Connection can be passed as "bind" in place of "url".""" - engine = create_async_engine("sqlite+aiosqlite:///:memory:") + engine = create_async_engine(aiosqlite_memory_url) connection = await engine.connect() component = SQLAlchemyComponent(bind=connection) async with Context() as ctx: @@ -176,9 +176,9 @@ async def test_close_twice_sync(psycopg_url: str) -> None: assert pool.checkedout() == 0 -async def test_close_twice_async(psycopg_url: str) -> None: +async def test_close_twice_async(psycopg_url_async: str) -> None: """Test that closing a session releases connection resources, but remains usable.""" - component = SQLAlchemyComponent(url=psycopg_url) + component = SQLAlchemyComponent(url=psycopg_url_async) async with Context() as ctx: await component.start(ctx) session = ctx.require_resource(AsyncSession) @@ -240,7 +240,7 @@ async def test_memory_leak() -> None: assert next((x for x in gc.get_objects() if isinstance(x, Context)), None) is None -async def test_session_event_sync(psycopg_url: str) -> None: +async def test_session_event_sync(psycopg_url_async: str) -> None: """Test that creating a session in a context does not leak memory.""" listener_session: Session | None = None listener_thread: Thread | None = None @@ -251,7 +251,7 @@ def listener(session: Session) -> None: listener_session = session listener_thread = current_thread() - component = SQLAlchemyComponent(url=psycopg_url, prefer_async=False) + component = SQLAlchemyComponent(url=psycopg_url_async, prefer_async=False) engine: Engine | None = None try: async with Context() as ctx: @@ -272,7 +272,9 @@ def listener(session: Session) -> None: clear_database(conn) -async def test_session_event_async(request: FixtureRequest, psycopg_url: str) -> None: +async def test_session_event_async( + request: FixtureRequest, psycopg_url_async: str +) -> None: """Test that creating a session in a context does not leak memory.""" listener_session: Session | None = None listener_thread: Thread | None = None @@ -290,7 +292,7 @@ def listener(session: Session) -> None: listen(Session, "before_commit", listener) request.addfinalizer(lambda: remove(Session, "before_commit", listener)) - component = SQLAlchemyComponent(url=psycopg_url) + component = SQLAlchemyComponent(url=psycopg_url_async) engine: AsyncEngine | None = None try: async with Context() as ctx: From 90afc1f8eb4ae9f05c1de4d2eac69ea81b8e43d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 5 Feb 2024 21:55:40 +0200 Subject: [PATCH 05/29] Adapted code to the latest 5.0 core changes --- src/asphalt/sqlalchemy/_component.py | 23 +++--- tests/test_component.py | 113 ++++++++++++++------------- tests/test_testing_recipe.py | 52 ++++++------ 3 files changed, 96 insertions(+), 92 deletions(-) diff --git a/src/asphalt/sqlalchemy/_component.py b/src/asphalt/sqlalchemy/_component.py index eee5554..60a5faf 100644 --- a/src/asphalt/sqlalchemy/_component.py +++ b/src/asphalt/sqlalchemy/_component.py @@ -9,8 +9,9 @@ from anyio import CapacityLimiter, to_thread from asphalt.core import ( Component, - Context, GeneratedResource, + add_resource, + add_resource_factory, qualified_name, resolve_reference, ) @@ -170,7 +171,7 @@ def __init__( else: self._sessionmaker = sessionmaker(bind=self._bind, **session_args) - def create_session(self, ctx: Context) -> GeneratedResource[Session]: + def create_session(self) -> GeneratedResource[Session]: async def teardown_session() -> None: try: if session.in_transaction(): @@ -188,7 +189,7 @@ async def teardown_session() -> None: session = self._sessionmaker() return GeneratedResource(session, teardown_session) - def create_async_session(self, ctx: Context) -> GeneratedResource[AsyncSession]: + def create_async_session(self) -> GeneratedResource[AsyncSession]: async def teardown_session() -> None: try: if session.in_transaction(): @@ -202,7 +203,7 @@ async def teardown_session() -> None: session: AsyncSession = self._async_sessionmaker() return GeneratedResource(session, teardown_session) - async def start(self, ctx: Context) -> None: + async def start(self) -> None: bind: Connection | Engine | AsyncConnection | AsyncEngine if isinstance(self._engine, AsyncEngine): if self.ready_callback: @@ -216,23 +217,23 @@ async def start(self, ctx: Context) -> None: else: teardown_callback = None - await ctx.add_resource( + await add_resource( self._engine, self.resource_name, description="SQLAlchemy engine (asynchronous)", teardown_callback=teardown_callback, ) - await ctx.add_resource( + await add_resource( self._sessionmaker, self.resource_name, description="SQLAlchemy session factory (synchronous)", ) - await ctx.add_resource( + await add_resource( self._async_sessionmaker, self.resource_name, description="SQLAlchemy session factory (asynchronous)", ) - await ctx.add_resource_factory( + await add_resource_factory( self.create_async_session, self.resource_name, description="SQLAlchemy session (asynchronous)", @@ -249,18 +250,18 @@ async def start(self, ctx: Context) -> None: else: teardown_callback = None - await ctx.add_resource( + await add_resource( self._engine, self.resource_name, description="SQLAlchemy engine (synchronous)", teardown_callback=teardown_callback, ) - await ctx.add_resource( + await add_resource( self._sessionmaker, self.resource_name, description="SQLAlchemy session factory (synchronous)", ) - await ctx.add_resource_factory( + await add_resource_factory( self.create_session, self.resource_name, description="SQLAlchemy session (synchronous)", diff --git a/tests/test_component.py b/tests/test_component.py index effd9ad..dd08a04 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -7,7 +7,13 @@ from typing import Any import pytest -from asphalt.core import Context, NoCurrentContext, current_context, get_resource +from asphalt.core import ( + Context, + NoCurrentContext, + current_context, + get_resource, + require_resource, +) from pytest import FixtureRequest from sqlalchemy.engine.url import URL from sqlalchemy.event import listen, remove @@ -42,12 +48,12 @@ async def test_component_start_sync( """Test that the component creates all the expected (synchronous) resources.""" url = URL.create("sqlite", database=":memory:") component = SQLAlchemyComponent(url=url, **component_opts) - async with Context() as ctx: - await component.start(ctx) + async with Context(): + await component.start() - ctx.require_resource(Engine, *args) - ctx.require_resource(sessionmaker, *args) - ctx.require_resource(Session, *args) + require_resource(Engine, *args) + require_resource(sessionmaker, *args) + require_resource(Session, *args) @pytest.mark.parametrize( @@ -63,13 +69,13 @@ async def test_component_start_async( """Test that the component creates all the expected (asynchronous) resources.""" url = URL.create("sqlite+aiosqlite", database=":memory:") component = SQLAlchemyComponent(url=url, **component_opts) - async with Context() as ctx: - await component.start(ctx) + async with Context(): + await component.start() - ctx.require_resource(AsyncEngine, *args) - async_session_class = ctx.require_resource(async_sessionmaker, *args) - ctx.require_resource(AsyncSession, *args) - sync_session_class = ctx.require_resource(sessionmaker, *args) + require_resource(AsyncEngine, *args) + async_session_class = require_resource(async_sessionmaker, *args) + require_resource(AsyncSession, *args) + sync_session_class = require_resource(sessionmaker, *args) assert async_session_class.kw["sync_session_class"] is sync_session_class @@ -94,11 +100,11 @@ async def ready_callback_async( callback = ready_callback_async if asynchronous else ready_callback component = SQLAlchemyComponent(url="sqlite:///:memory:", ready_callback=callback) - async with Context() as ctx: - await component.start(ctx) + async with Context(): + await component.start() - engine = ctx.require_resource(Engine) - factory = ctx.require_resource(sessionmaker) + engine = require_resource(Engine) + factory = require_resource(sessionmaker) assert engine is engine2 assert factory is factory2 @@ -108,11 +114,11 @@ async def test_bind_sync_connection() -> None: engine = create_engine("sqlite:///:memory:") connection = engine.connect() component = SQLAlchemyComponent(bind=connection) - async with Context() as ctx: - await component.start(ctx) + async with Context(): + await component.start() - assert ctx.require_resource(Engine) is engine - assert ctx.require_resource(Session).bind is connection + assert require_resource(Engine) is engine + assert require_resource(Session).bind is connection connection.close() @@ -122,11 +128,11 @@ async def test_bind_async_connection(aiosqlite_memory_url: str) -> None: engine = create_async_engine(aiosqlite_memory_url) connection = await engine.connect() component = SQLAlchemyComponent(bind=connection) - async with Context() as ctx: - await component.start(ctx) + async with Context(): + await component.start() - assert ctx.require_resource(AsyncEngine) is engine - assert ctx.require_resource(AsyncSession).bind is connection + assert require_resource(AsyncEngine) is engine + assert require_resource(AsyncSession).bind is connection await connection.close() @@ -135,11 +141,11 @@ async def test_bind_sync_engine() -> None: """Test that a Connection can be passed as "bind" in place of "url".""" engine = create_engine("sqlite:///:memory:") component = SQLAlchemyComponent(bind=engine) - async with Context() as ctx: - await component.start(ctx) + async with Context(): + await component.start() - assert ctx.require_resource(Engine) is engine - assert ctx.require_resource(Session).bind is engine + assert require_resource(Engine) is engine + assert require_resource(Session).bind is engine engine.dispose() @@ -148,11 +154,11 @@ async def test_bind_async_engine() -> None: """Test that a Connection can be passed as "bind" in place of "url".""" engine = create_async_engine("sqlite+aiosqlite:///:memory:") component = SQLAlchemyComponent(bind=engine) - async with Context() as ctx: - await component.start(ctx) + async with Context(): + await component.start() - assert ctx.require_resource(AsyncEngine) is engine - assert ctx.require_resource(AsyncSession).bind is engine + assert require_resource(AsyncEngine) is engine + assert require_resource(AsyncSession).bind is engine await engine.dispose() @@ -160,9 +166,9 @@ async def test_bind_async_engine() -> None: async def test_close_twice_sync(psycopg_url: str) -> None: """Test that closing a session releases connection resources, but remains usable.""" component = SQLAlchemyComponent(url=psycopg_url, prefer_async=False) - async with Context() as ctx: - await component.start(ctx) - session = ctx.require_resource(Session) + async with Context(): + await component.start() + session = require_resource(Session) assert isinstance(session.bind, Engine) pool = session.bind.pool assert isinstance(pool, QueuePool) @@ -179,9 +185,9 @@ async def test_close_twice_sync(psycopg_url: str) -> None: async def test_close_twice_async(psycopg_url_async: str) -> None: """Test that closing a session releases connection resources, but remains usable.""" component = SQLAlchemyComponent(url=psycopg_url_async) - async with Context() as ctx: - await component.start(ctx) - session = ctx.require_resource(AsyncSession) + async with Context(): + await component.start() + session = require_resource(AsyncSession) assert isinstance(session.bind, AsyncEngine) pool = session.bind.pool assert isinstance(pool, AsyncAdaptedQueuePool) @@ -216,9 +222,9 @@ async def test_finish_commit(raise_exception: bool, tmp_path: Path) -> None: url={"drivername": "sqlite", "database": str(db_path)}, ) async with AsyncExitStack() as stack: - ctx = await stack.enter_async_context(Context()) - await component.start(ctx) - session = ctx.require_resource(Session) + await stack.enter_async_context(Context()) + await component.start() + session = require_resource(Session) session.execute(text("INSERT INTO foo (id) VALUES(3)")) if raise_exception: stack.enter_context(pytest.raises(Exception, match="dummy")) @@ -231,11 +237,10 @@ async def test_finish_commit(raise_exception: bool, tmp_path: Path) -> None: async def test_memory_leak() -> None: """Test that creating a session in a context does not leak memory.""" component = SQLAlchemyComponent(url="sqlite:///:memory:") - async with Context() as ctx: - await component.start(ctx) - ctx.require_resource(Session) + async with Context(): + await component.start() + require_resource(Session) - del ctx gc.collect() # needed on PyPy assert next((x for x in gc.get_objects() if isinstance(x, Context)), None) is None @@ -254,14 +259,14 @@ def listener(session: Session) -> None: component = SQLAlchemyComponent(url=psycopg_url_async, prefer_async=False) engine: Engine | None = None try: - async with Context() as ctx: - await component.start(ctx) - engine = ctx.require_resource(Engine) + async with Context(): + await component.start() + engine = require_resource(Engine) Person.metadata.create_all(engine) - session_factory = ctx.require_resource(sessionmaker) + session_factory = require_resource(sessionmaker) listen(session_factory, "before_commit", listener) - dbsession = ctx.require_resource(Session) + dbsession = require_resource(Session) dbsession.add(Person(name="Test person")) assert listener_session is dbsession @@ -295,10 +300,10 @@ def listener(session: Session) -> None: component = SQLAlchemyComponent(url=psycopg_url_async) engine: AsyncEngine | None = None try: - async with Context() as ctx: - await component.start(ctx) - engine = ctx.require_resource(AsyncEngine) - dbsession = ctx.require_resource(AsyncSession) + async with Context(): + await component.start() + engine = require_resource(AsyncEngine) + dbsession = require_resource(AsyncSession) await dbsession.run_sync( lambda session: Person.metadata.create_all(session.bind) ) diff --git a/tests/test_testing_recipe.py b/tests/test_testing_recipe.py index d6dbb3c..c010593 100644 --- a/tests/test_testing_recipe.py +++ b/tests/test_testing_recipe.py @@ -8,7 +8,7 @@ from typing import Any import pytest -from asphalt.core import ContainerComponent, Context +from asphalt.core import ContainerComponent, Context, require_resource from sqlalchemy.engine import Connection, Engine from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession @@ -61,12 +61,10 @@ async def test_rollback( self, dbsession: Session, root_component: ContainerComponent ) -> None: # Simulate a rollback happening in a subcontext - async with Context() as root_ctx: - print("Root context is", hex(id(root_ctx))) - await root_component.start(root_ctx) - async with Context() as ctx: - print("Subcontext is", hex(id(ctx))) - session = ctx.require_resource(Session) + async with Context(): + await root_component.start() + async with Context(): + session = require_resource(Session) try: # No value for a non-nullable column => IntegrityError! session.add(Person()) @@ -85,10 +83,10 @@ async def test_add_person( self, dbsession: Session, root_component: ContainerComponent ) -> None: # Simulate adding a row to the "people" table in the application - async with Context() as root_ctx: - await root_component.start(root_ctx) - async with Context() as ctx: - session = ctx.require_resource(Session) + async with Context(): + await root_component.start() + async with Context(): + session = require_resource(Session) session.add(Person(name="Another person")) # The testing code should see both rows now @@ -98,10 +96,10 @@ async def test_delete_person( self, dbsession: Session, root_component: ContainerComponent ) -> None: # Simulate removing the test person in the application - async with Context() as root_ctx: - await root_component.start(root_ctx) - async with Context() as ctx: - session = ctx.require_resource(Session) + async with Context(): + await root_component.start() + async with Context(): + session = require_resource(Session) session.execute(delete(Person)) # The testing code should not see any rows now @@ -151,10 +149,10 @@ async def test_rollback( self, dbsession: AsyncSession, root_component: ContainerComponent ) -> None: # Simulate a rollback happening in a subcontext - async with Context() as root_ctx: - await root_component.start(root_ctx) - async with Context() as ctx: - session = ctx.require_resource(AsyncSession) + async with Context(): + await root_component.start() + async with Context(): + session = require_resource(AsyncSession) try: # No value for a non-nullable column => IntegrityError! session.add(Person()) @@ -173,10 +171,10 @@ async def test_add_person( self, dbsession: AsyncSession, root_component: ContainerComponent ) -> None: # Simulate adding a row to the "people" table in the application - async with Context() as root_ctx: - await root_component.start(root_ctx) - async with Context() as ctx: - session = ctx.require_resource(AsyncSession) + async with Context(): + await root_component.start() + async with Context(): + session = require_resource(AsyncSession) session.add(Person(name="Another person")) # The testing code should see both rows now @@ -186,10 +184,10 @@ async def test_delete_person( self, dbsession: AsyncSession, root_component: ContainerComponent ) -> None: # Simulate removing the test person in the application - async with Context() as root_ctx: - await root_component.start(root_ctx) - async with Context() as ctx: - session = ctx.require_resource(AsyncSession) + async with Context(): + await root_component.start() + async with Context(): + session = require_resource(AsyncSession) await session.execute(delete(Person)) # The testing code should not see any rows now From aa56e76cd68f0437b334f30caf4ce719da8783df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 5 Feb 2024 22:29:31 +0200 Subject: [PATCH 06/29] Adapted examples to Asphalt 5.0 --- examples/csvimport.py | 15 +++++++++------ examples/csvimport_orm.py | 11 +++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/examples/csvimport.py b/examples/csvimport.py index a0c10fb..63b1dce 100644 --- a/examples/csvimport.py +++ b/examples/csvimport.py @@ -9,9 +9,9 @@ import logging from pathlib import Path +from anyio import to_thread from asphalt.core import ( CLIApplicationComponent, - Context, inject, resource, run_application, @@ -38,7 +38,7 @@ def __init__(self) -> None: super().__init__() self.csv_path = Path(__file__).with_name("people.csv") - async def start(self, ctx: Context) -> None: + async def start(self) -> None: # Remove the db file if it exists db_path = self.csv_path.with_name("people.db") if db_path.exists(): @@ -49,11 +49,11 @@ async def start(self, ctx: Context) -> None: url=f"sqlite:///{db_path}", ready_callback=lambda bind, factory: metadata.create_all(bind), ) - await super().start(ctx) + await super().start() @inject - async def run(self, ctx: Context, *, dbsession: Session = resource()) -> None: - async with ctx.threadpool(): + async def run(self, *, dbsession: Session = resource()) -> None: + def insert_rows_in_thread() -> int: num_rows = 0 with self.csv_path.open() as csvfile: reader = csv.reader(csvfile, delimiter="|") @@ -65,7 +65,10 @@ async def run(self, ctx: Context, *, dbsession: Session = resource()) -> None: ) ) - logger.info("Imported %d rows of data", num_rows) + return num_rows + + inserted_rows = await to_thread.run_sync(insert_rows_in_thread) + logger.info("Imported %d rows of data", inserted_rows) run_application(CSVImporterComponent(), logging=logging.DEBUG) diff --git a/examples/csvimport_orm.py b/examples/csvimport_orm.py index cf0f41c..e14d6c3 100644 --- a/examples/csvimport_orm.py +++ b/examples/csvimport_orm.py @@ -9,6 +9,7 @@ import logging from pathlib import Path +from anyio import to_thread from asphalt.core import ( CLIApplicationComponent, Context, @@ -40,7 +41,7 @@ def __init__(self) -> None: super().__init__() self.csv_path = Path(__file__).with_name("people.csv") - async def start(self, ctx: Context) -> None: + async def start(self) -> None: # Remove the db file if it exists db_path = self.csv_path.with_name("people.db") if db_path.exists(): @@ -53,11 +54,11 @@ async def start(self, ctx: Context) -> None: bind ), ) - await super().start(ctx) + await super().start() @inject async def run(self, ctx: Context, *, dbsession: Session = resource()) -> None: - async with ctx.threadpool(): + def insert_rows_in_thread() -> int: num_rows = 0 with self.csv_path.open() as csvfile: reader = csv.reader(csvfile, delimiter="|") @@ -70,8 +71,10 @@ async def run(self, ctx: Context, *, dbsession: Session = resource()) -> None: # Emit pending INSERTs (though this would happen when the context ends # anyway) dbsession.flush() + return num_rows - logger.info("Imported %d rows of data", num_rows) + inserted_rows = await to_thread.run_sync(insert_rows_in_thread) + logger.info("Imported %d rows of data", inserted_rows) run_application(CSVImporterComponent(), logging=logging.DEBUG) From c88bdc1fb3c382798a94d7c5b236e94eff1d685b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 7 Feb 2024 15:34:11 +0200 Subject: [PATCH 07/29] Use a component reference consistent with the other projects --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1b8b303..0b5f51c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ doc = [ ] [project.entry-points."asphalt.components"] -sqlalchemy = "asphalt.sqlalchemy._component:SQLAlchemyComponent" +sqlalchemy = "asphalt.sqlalchemy:SQLAlchemyComponent" [tool.setuptools_scm] version_scheme = "post-release" From 78ea3d1df0d4dbc48e45656fc52ad50199356adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Thu, 8 Feb 2024 12:39:38 +0200 Subject: [PATCH 08/29] Updated docstrings and MyPy settings --- .pre-commit-config.yaml | 5 ++++- pyproject.toml | 2 ++ src/asphalt/sqlalchemy/_component.py | 9 ++++----- src/asphalt/sqlalchemy/_utils.py | 12 ++++++------ tests/test_component.py | 7 +++---- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7211c7c..e4bd4bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,10 @@ repos: rev: v1.7.1 hooks: - id: mypy - additional_dependencies: ["pytest"] + additional_dependencies: +# - asphalt + - pytest + - sqlalchemy - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 diff --git a/pyproject.toml b/pyproject.toml index 0b5f51c..d9d1c62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,8 @@ testpaths = ["tests"] [tool.mypy] python_version = "3.8" +#strict = true +explicit_package_bases = true [tool.coverage.run] source = ["asphalt.sqlalchemy"] diff --git a/src/asphalt/sqlalchemy/_component.py b/src/asphalt/sqlalchemy/_component.py index 60a5faf..765f186 100644 --- a/src/asphalt/sqlalchemy/_component.py +++ b/src/asphalt/sqlalchemy/_component.py @@ -43,7 +43,7 @@ class SQLAlchemyComponent(Component): For synchronous engines, the following resources are provided: - * :class:`~sqlalchemy.future.engine.Engine` + * :class:`~sqlalchemy.engine.Engine` * :class:`~sqlalchemy.orm.session.sessionmaker` * :class:`~sqlalchemy.orm.session.Session` @@ -57,17 +57,16 @@ class SQLAlchemyComponent(Component): .. note:: The following options will always be set to fixed values in sessions: * ``expire_on_commit``: ``False`` - * ``future``: ``True`` :param url: the connection url passed to - :func:`~sqlalchemy.future.engine.create_engine` + :func:`~sqlalchemy.engine.create_engine` (can also be a dictionary of :class:`~sqlalchemy.engine.url.URL` keyword arguments) :param bind: a connection or engine to use instead of creating a new engine :param prefer_async: if ``True``, try to create an async engine rather than a synchronous one, in cases like ``psycopg`` where the driver supports both :param engine_args: extra keyword arguments passed to - :func:`sqlalchemy.future.engine.create_engine` or + :func:`sqlalchemy.engine.create_engine` or :func:`sqlalchemy.ext.asyncio.create_engine` :param session_args: extra keyword arguments passed to :class:`~sqlalchemy.orm.session.Session` or @@ -77,7 +76,7 @@ class SQLAlchemyComponent(Component): :param ready_callback: a callable that is called right before the resources are added to the context (can be a coroutine function too) :param poolclass: the SQLAlchemy pool class (or a textual reference to one) to use; - passed to :func:`sqlalchemy.future.engine.create_engine` or + passed to :func:`sqlalchemy.engine.create_engine` or :func:`sqlalchemy.ext.asyncio.create_engine` :param resource_name: name space for the database resources """ diff --git a/src/asphalt/sqlalchemy/_utils.py b/src/asphalt/sqlalchemy/_utils.py index 3ca5bba..8e18b9a 100644 --- a/src/asphalt/sqlalchemy/_utils.py +++ b/src/asphalt/sqlalchemy/_utils.py @@ -3,9 +3,9 @@ import sqlite3 from collections.abc import Iterable -from sqlalchemy import event +from sqlalchemy import Connection, Engine +from sqlalchemy.event import listen from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine -from sqlalchemy.future import Connection, Engine from sqlalchemy.pool import ConnectionPoolEntry from sqlalchemy.sql.schema import MetaData @@ -63,11 +63,11 @@ def apply_sqlite_hacks(engine: Engine | AsyncEngine) -> None: integration tests (the connection is passed to the component as the ``bind`` option). + :param engine: an engine using the sqlite dialect + .. seealso:: https://docs.sqlalchemy.org/en/14/dialects/sqlite.html\ #pysqlite-serializable - :param engine: an engine using the sqlite dialect - """ def do_connect( @@ -88,5 +88,5 @@ def do_begin(conn: Connection) -> None: ) sync_engine = engine.sync_engine if isinstance(engine, AsyncEngine) else engine - event.listen(sync_engine, "connect", do_connect) - event.listen(sync_engine, "begin", do_begin) + listen(sync_engine, "connect", do_connect) + listen(sync_engine, "begin", do_begin) diff --git a/tests/test_component.py b/tests/test_component.py index dd08a04..f03d15b 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -257,7 +257,7 @@ def listener(session: Session) -> None: listener_thread = current_thread() component = SQLAlchemyComponent(url=psycopg_url_async, prefer_async=False) - engine: Engine | None = None + engine: Engine try: async with Context(): await component.start() @@ -272,9 +272,8 @@ def listener(session: Session) -> None: assert listener_session is dbsession assert listener_thread != current_thread() finally: - if engine: - with engine.connect() as conn: - clear_database(conn) + with engine.connect() as conn: + clear_database(conn) async def test_session_event_async( From 893f3c67421485817030dee6c139735fa74b1dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Thu, 8 Feb 2024 12:40:17 +0200 Subject: [PATCH 09/29] Updated documentation --- docs/api.rst | 16 ++++++++++++++++ docs/conf.py | 1 + docs/modules/component.rst | 5 ----- docs/modules/utils.rst | 5 ----- docs/py-modindex.rst | 2 -- 5 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 docs/api.rst delete mode 100644 docs/modules/component.rst delete mode 100644 docs/modules/utils.rst delete mode 100644 docs/py-modindex.rst diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..65b5a0e --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,16 @@ +API reference +============= + +.. py:currentmodule:: asphalt.sqlalchemy + +Component +--------- + +.. autoclass:: SQLAlchemyComponent + +Utility functions +----------------- + +.. autofunction:: clear_database +.. autofunction:: clear_async_database +.. autofunction:: apply_sqlite_hacks diff --git a/docs/conf.py b/docs/conf.py index 8a356fb..838e2e3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,6 +25,7 @@ exclude_patterns = ["_build"] pygments_style = "sphinx" +autodoc_default_options = {"members": True, "show-inheritance": True} highlight_language = "python3" todo_include_todos = False diff --git a/docs/modules/component.rst b/docs/modules/component.rst deleted file mode 100644 index 33a4451..0000000 --- a/docs/modules/component.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`asphalt.sqlalchemy.component` -=================================== - -.. module:: asphalt.sqlalchemy.component -.. autoclass:: asphalt.sqlalchemy.component.SQLAlchemyComponent diff --git a/docs/modules/utils.rst b/docs/modules/utils.rst deleted file mode 100644 index 510ab61..0000000 --- a/docs/modules/utils.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`asphalt.sqlalchemy.utils` -=============================== - -.. automodule:: asphalt.sqlalchemy.utils - :members: diff --git a/docs/py-modindex.rst b/docs/py-modindex.rst deleted file mode 100644 index 3cdb8ca..0000000 --- a/docs/py-modindex.rst +++ /dev/null @@ -1,2 +0,0 @@ -API reference -============= From 1711c7e9b8e91d04b6db97e43d1f652954620513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 5 Mar 2024 15:09:20 +0200 Subject: [PATCH 10/29] Updated for the latest Asphalt 5 changes --- src/asphalt/sqlalchemy/_component.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/asphalt/sqlalchemy/_component.py b/src/asphalt/sqlalchemy/_component.py index 765f186..4f3d76d 100644 --- a/src/asphalt/sqlalchemy/_component.py +++ b/src/asphalt/sqlalchemy/_component.py @@ -216,23 +216,23 @@ async def start(self) -> None: else: teardown_callback = None - await add_resource( + add_resource( self._engine, self.resource_name, description="SQLAlchemy engine (asynchronous)", teardown_callback=teardown_callback, ) - await add_resource( + add_resource( self._sessionmaker, self.resource_name, description="SQLAlchemy session factory (synchronous)", ) - await add_resource( + add_resource( self._async_sessionmaker, self.resource_name, description="SQLAlchemy session factory (asynchronous)", ) - await add_resource_factory( + add_resource_factory( self.create_async_session, self.resource_name, description="SQLAlchemy session (asynchronous)", @@ -249,18 +249,18 @@ async def start(self) -> None: else: teardown_callback = None - await add_resource( + add_resource( self._engine, self.resource_name, description="SQLAlchemy engine (synchronous)", teardown_callback=teardown_callback, ) - await add_resource( + add_resource( self._sessionmaker, self.resource_name, description="SQLAlchemy session factory (synchronous)", ) - await add_resource_factory( + add_resource_factory( self.create_session, self.resource_name, description="SQLAlchemy session (synchronous)", From 38c8b63a8db512b729085a80ebf2e15ce88257db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 5 Mar 2024 15:10:40 +0200 Subject: [PATCH 11/29] Updated pre-commit modules and ruff configuration --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 4 ++-- tests/conftest.py | 2 +- tests/test_testing_recipe.py | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e4bd4bd..83b20b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,14 +16,14 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.6 + rev: v0.3.0 hooks: - id: ruff args: [--fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.1 + rev: v1.8.0 hooks: - id: mypy additional_dependencies: diff --git a/pyproject.toml b/pyproject.toml index d9d1c62..d45136f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ sqlalchemy = "asphalt.sqlalchemy:SQLAlchemyComponent" version_scheme = "post-release" local_scheme = "dirty-tag" -[tool.ruff] +[tool.ruff.lint] select = [ "ASYNC", # flake8-async "E", "F", "W", # default Flake8 @@ -72,7 +72,7 @@ select = [ "UP", # pyupgrade ] -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ["asphalt.sqlalchemy"] [tool.pytest.ini_options] diff --git a/tests/conftest.py b/tests/conftest.py index 961936c..65b5bfb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -94,7 +94,7 @@ def sqlite_file_engine( @pytest.fixture async def aiosqlite_memory_engine( - aiosqlite_memory_url: str + aiosqlite_memory_url: str, ) -> AsyncGenerator[AsyncEngine, Any]: engine = create_async_engine(aiosqlite_memory_url) apply_sqlite_hacks(engine) diff --git a/tests/test_testing_recipe.py b/tests/test_testing_recipe.py index c010593..70aa401 100644 --- a/tests/test_testing_recipe.py +++ b/tests/test_testing_recipe.py @@ -2,6 +2,7 @@ This module exists to make sure that the testing recipe in the documentation is and stays valid, and works with different backends. """ + from __future__ import annotations from collections.abc import AsyncGenerator, Generator From 6697bd368c0cfd5450f648b72bad81c73d13d0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 5 Mar 2024 18:08:53 +0200 Subject: [PATCH 12/29] Added some finishing touches/updates --- .pre-commit-config.yaml | 2 +- examples/csvimport.py | 3 ++- examples/csvimport_orm.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83b20b3..945e4d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: hooks: - id: mypy additional_dependencies: -# - asphalt + - asphalt@git+https://github.com/asphalt-framework/asphalt@5.0 - pytest - sqlalchemy diff --git a/examples/csvimport.py b/examples/csvimport.py index 63b1dce..4d51e7c 100644 --- a/examples/csvimport.py +++ b/examples/csvimport.py @@ -9,6 +9,7 @@ import logging from pathlib import Path +import anyio from anyio import to_thread from asphalt.core import ( CLIApplicationComponent, @@ -71,4 +72,4 @@ def insert_rows_in_thread() -> int: logger.info("Imported %d rows of data", inserted_rows) -run_application(CSVImporterComponent(), logging=logging.DEBUG) +anyio.run(run_application, CSVImporterComponent()) diff --git a/examples/csvimport_orm.py b/examples/csvimport_orm.py index e14d6c3..c76c3f2 100644 --- a/examples/csvimport_orm.py +++ b/examples/csvimport_orm.py @@ -9,10 +9,10 @@ import logging from pathlib import Path +import anyio from anyio import to_thread from asphalt.core import ( CLIApplicationComponent, - Context, inject, resource, run_application, @@ -57,7 +57,7 @@ async def start(self) -> None: await super().start() @inject - async def run(self, ctx: Context, *, dbsession: Session = resource()) -> None: + async def run(self, *, dbsession: Session = resource()) -> None: def insert_rows_in_thread() -> int: num_rows = 0 with self.csv_path.open() as csvfile: @@ -77,4 +77,4 @@ def insert_rows_in_thread() -> int: logger.info("Imported %d rows of data", inserted_rows) -run_application(CSVImporterComponent(), logging=logging.DEBUG) +anyio.run(run_application, CSVImporterComponent()) From 7955450d04eecfb2bd7dccd7fe324663a65959f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Thu, 7 Mar 2024 17:12:56 +0200 Subject: [PATCH 13/29] Updated GitHub actions and fixed Mypy errors --- .github/workflows/publish.yml | 4 ++-- .github/workflows/test.yml | 2 +- pyproject.toml | 2 +- src/asphalt/sqlalchemy/__init__.py | 2 +- src/asphalt/sqlalchemy/_component.py | 2 +- tests/conftest.py | 4 ++-- tests/test_component.py | 6 +++--- tests/test_utils.py | 6 +++--- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d0fb4c2..92bbd0d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,7 +23,7 @@ jobs: - name: Create packages run: python -m build - name: Archive packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist @@ -36,7 +36,7 @@ jobs: id-token: write steps: - name: Retrieve packages - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Upload packages uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab10332..a85fb66 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: - name: Start external services run: docker-compose up -d - name: Install dependencies - run: pip install .[test] + run: pip install -e .[test] - name: Test with pytest run: coverage run -m pytest -v - name: Generate coverage report diff --git a/pyproject.toml b/pyproject.toml index d45136f..c480289 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ testpaths = ["tests"] [tool.mypy] python_version = "3.8" -#strict = true +strict = true explicit_package_bases = true [tool.coverage.run] diff --git a/src/asphalt/sqlalchemy/__init__.py b/src/asphalt/sqlalchemy/__init__.py index afa721f..9dbb27a 100644 --- a/src/asphalt/sqlalchemy/__init__.py +++ b/src/asphalt/sqlalchemy/__init__.py @@ -9,5 +9,5 @@ key: str value: Any for key, value in list(locals().items()): - if getattr(value, "__module__", "").startswith("asphalt.sqlalchemy."): + if getattr(value, "__module__", "").startswith(f"{__name__}."): value.__module__ = __name__ diff --git a/src/asphalt/sqlalchemy/_component.py b/src/asphalt/sqlalchemy/_component.py index 4f3d76d..deca35c 100644 --- a/src/asphalt/sqlalchemy/_component.py +++ b/src/asphalt/sqlalchemy/_component.py @@ -94,7 +94,7 @@ def __init__( engine_args: dict[str, Any] | None = None, session_args: dict[str, Any] | None = None, commit_executor_workers: int = 50, - ready_callback: Callable[[Engine, sessionmaker], Any] | str | None = None, + ready_callback: Callable[[Engine, sessionmaker[Any]], Any] | str | None = None, poolclass: str | type[Pool] | None = None, resource_name: str = "default", ): diff --git a/tests/conftest.py b/tests/conftest.py index 65b5bfb..54dffa2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,7 @@ def aiosqlite_memory_url(anyio_backend_name: str) -> str: @pytest.fixture -def psycopg_url() -> str: # type: ignore[return] +def psycopg_url() -> str: pytest.importorskip("psycopg", reason="psycopg is not available") try: return os.environ["POSTGRESQL_URL"] @@ -40,7 +40,7 @@ def psycopg_url_async(psycopg_url: str, anyio_backend_name: str) -> str: @pytest.fixture -def mysql_url() -> str: # type: ignore[return] +def mysql_url() -> str: try: return os.environ["MYSQL_URL"] except KeyError: diff --git a/tests/test_component.py b/tests/test_component.py index f03d15b..f031980 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -84,15 +84,15 @@ async def test_component_start_async( ) async def test_ready_callback(asynchronous: bool) -> None: engine2: Engine | AsyncEngine | None = None - factory2: sessionmaker | async_sessionmaker | None = None + factory2: sessionmaker[Any] | async_sessionmaker[Any] | None = None - def ready_callback(engine: Engine, factory: sessionmaker) -> None: + def ready_callback(engine: Engine, factory: sessionmaker[Any]) -> None: nonlocal engine2, factory2 engine2 = engine factory2 = factory async def ready_callback_async( - engine: AsyncEngine, factory: async_sessionmaker + engine: AsyncEngine, factory: async_sessionmaker[Any] ) -> None: nonlocal engine2, factory2 engine2 = engine diff --git a/tests/test_utils.py b/tests/test_utils.py index f3c0d78..5e21c70 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,7 +9,7 @@ from sqlalchemy.sql.schema import Column, ForeignKey, MetaData, Table from sqlalchemy.sql.sqltypes import Integer -from asphalt.sqlalchemy._utils import clear_database +from asphalt.sqlalchemy import clear_database @pytest.fixture @@ -19,7 +19,7 @@ def connection(sync_engine: Engine) -> Generator[Connection, Any, None]: Table("table", metadata, Column("column1", Integer, primary_key=True)) Table("table2", metadata, Column("fk_column", ForeignKey("table.column1"))) if conn.dialect.name != "sqlite": - conn.execute(CreateSchema("altschema")) + conn.execute(CreateSchema("altschema")) # type: ignore[no-untyped-call] Table("table3", metadata, Column("fk_column", Integer), schema="altschema") metadata.create_all(conn) @@ -28,7 +28,7 @@ def connection(sync_engine: Engine) -> Generator[Connection, Any, None]: if conn.dialect.name != "sqlite": metadata.drop_all(conn) - conn.execute(DropSchema("altschema")) + conn.execute(DropSchema("altschema")) # type: ignore[no-untyped-call] def test_clear_database(connection: Connection) -> None: From 47e9509b7a9f63f1e8d5e83b13513a6f93cceaaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Thu, 7 Mar 2024 23:29:00 +0200 Subject: [PATCH 14/29] Downgraded the minimum Asphalt requirement for testing --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c480289..d886966 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - "asphalt ~= 4.12", + "asphalt ~= 4.11", "SQLAlchemy[asyncio] >= 2.0", ] dynamic = ["version"] From 6cd4a241d1eb8727ba78705e6f4a41a28540410b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 26 Mar 2024 19:08:23 +0200 Subject: [PATCH 15/29] Updated code, docs and tests to match the new resource API --- docs/events.rst | 4 +-- src/asphalt/sqlalchemy/_utils.py | 9 +++-- tests/test_component.py | 61 ++++++++++++++++---------------- tests/test_testing_recipe.py | 14 ++++---- 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/docs/events.rst b/docs/events.rst index 35e79ca..8223534 100644 --- a/docs/events.rst +++ b/docs/events.rst @@ -33,7 +33,7 @@ a synchronous ``sessionmaker`` resource is provided by this component even for a engines. To add listeners, simply use this session maker as the target for the listeners:: - from asphalt.core import NoCurrentContext, get_resource, inject, resource + from asphalt.core import NoCurrentContext, get_resource_nowait, inject, resource from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import sessionmaker from sqlalchemy import event @@ -41,7 +41,7 @@ listeners:: def precommit_hook(session): try: - async_session = get_resource(AsyncSession) + async_session = get_resource_nowait(AsyncSession) except NoCurrentContext: return diff --git a/src/asphalt/sqlalchemy/_utils.py b/src/asphalt/sqlalchemy/_utils.py index 8e18b9a..ff5cd1f 100644 --- a/src/asphalt/sqlalchemy/_utils.py +++ b/src/asphalt/sqlalchemy/_utils.py @@ -3,11 +3,10 @@ import sqlite3 from collections.abc import Iterable -from sqlalchemy import Connection, Engine +from sqlalchemy import Connection, Engine, MetaData from sqlalchemy.event import listen from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine from sqlalchemy.pool import ConnectionPoolEntry -from sqlalchemy.sql.schema import MetaData def clear_database(engine: Engine | Connection, schemas: Iterable[str] = ()) -> None: @@ -47,7 +46,11 @@ async def clear_async_database( for schema in all_schemas: # Reflect the schema to get the list of the tables, views and constraints metadata = MetaData() - await connection.run_sync(metadata.reflect, schema=schema, views=True) + await connection.run_sync( + metadata.reflect, # type: ignore[arg-type] + schema=schema, + views=True, + ) metadatas.append(metadata) for metadata in metadatas: diff --git a/tests/test_component.py b/tests/test_component.py index f031980..33c4eac 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -11,8 +11,7 @@ Context, NoCurrentContext, current_context, - get_resource, - require_resource, + get_resource_nowait, ) from pytest import FixtureRequest from sqlalchemy.engine.url import URL @@ -51,9 +50,9 @@ async def test_component_start_sync( async with Context(): await component.start() - require_resource(Engine, *args) - require_resource(sessionmaker, *args) - require_resource(Session, *args) + get_resource_nowait(Engine, *args) + get_resource_nowait(sessionmaker, *args) + get_resource_nowait(Session, *args) @pytest.mark.parametrize( @@ -72,10 +71,10 @@ async def test_component_start_async( async with Context(): await component.start() - require_resource(AsyncEngine, *args) - async_session_class = require_resource(async_sessionmaker, *args) - require_resource(AsyncSession, *args) - sync_session_class = require_resource(sessionmaker, *args) + get_resource_nowait(AsyncEngine, *args) + async_session_class = get_resource_nowait(async_sessionmaker, *args) + get_resource_nowait(AsyncSession, *args) + sync_session_class = get_resource_nowait(sessionmaker, *args) assert async_session_class.kw["sync_session_class"] is sync_session_class @@ -103,8 +102,8 @@ async def ready_callback_async( async with Context(): await component.start() - engine = require_resource(Engine) - factory = require_resource(sessionmaker) + engine = get_resource_nowait(Engine) + factory = get_resource_nowait(sessionmaker) assert engine is engine2 assert factory is factory2 @@ -117,8 +116,8 @@ async def test_bind_sync_connection() -> None: async with Context(): await component.start() - assert require_resource(Engine) is engine - assert require_resource(Session).bind is connection + assert get_resource_nowait(Engine) is engine + assert get_resource_nowait(Session).bind is connection connection.close() @@ -131,8 +130,8 @@ async def test_bind_async_connection(aiosqlite_memory_url: str) -> None: async with Context(): await component.start() - assert require_resource(AsyncEngine) is engine - assert require_resource(AsyncSession).bind is connection + assert get_resource_nowait(AsyncEngine) is engine + assert get_resource_nowait(AsyncSession).bind is connection await connection.close() @@ -144,8 +143,8 @@ async def test_bind_sync_engine() -> None: async with Context(): await component.start() - assert require_resource(Engine) is engine - assert require_resource(Session).bind is engine + assert get_resource_nowait(Engine) is engine + assert get_resource_nowait(Session).bind is engine engine.dispose() @@ -157,8 +156,8 @@ async def test_bind_async_engine() -> None: async with Context(): await component.start() - assert require_resource(AsyncEngine) is engine - assert require_resource(AsyncSession).bind is engine + assert get_resource_nowait(AsyncEngine) is engine + assert get_resource_nowait(AsyncSession).bind is engine await engine.dispose() @@ -168,7 +167,7 @@ async def test_close_twice_sync(psycopg_url: str) -> None: component = SQLAlchemyComponent(url=psycopg_url, prefer_async=False) async with Context(): await component.start() - session = require_resource(Session) + session = get_resource_nowait(Session) assert isinstance(session.bind, Engine) pool = session.bind.pool assert isinstance(pool, QueuePool) @@ -187,7 +186,7 @@ async def test_close_twice_async(psycopg_url_async: str) -> None: component = SQLAlchemyComponent(url=psycopg_url_async) async with Context(): await component.start() - session = require_resource(AsyncSession) + session = get_resource_nowait(AsyncSession) assert isinstance(session.bind, AsyncEngine) pool = session.bind.pool assert isinstance(pool, AsyncAdaptedQueuePool) @@ -224,7 +223,7 @@ async def test_finish_commit(raise_exception: bool, tmp_path: Path) -> None: async with AsyncExitStack() as stack: await stack.enter_async_context(Context()) await component.start() - session = require_resource(Session) + session = get_resource_nowait(Session) session.execute(text("INSERT INTO foo (id) VALUES(3)")) if raise_exception: stack.enter_context(pytest.raises(Exception, match="dummy")) @@ -239,7 +238,7 @@ async def test_memory_leak() -> None: component = SQLAlchemyComponent(url="sqlite:///:memory:") async with Context(): await component.start() - require_resource(Session) + get_resource_nowait(Session) gc.collect() # needed on PyPy assert next((x for x in gc.get_objects() if isinstance(x, Context)), None) is None @@ -261,12 +260,12 @@ def listener(session: Session) -> None: try: async with Context(): await component.start() - engine = require_resource(Engine) + engine = get_resource_nowait(Engine) Person.metadata.create_all(engine) - session_factory = require_resource(sessionmaker) + session_factory = get_resource_nowait(sessionmaker) listen(session_factory, "before_commit", listener) - dbsession = require_resource(Session) + dbsession = get_resource_nowait(Session) dbsession.add(Person(name="Test person")) assert listener_session is dbsession @@ -286,7 +285,7 @@ async def test_session_event_async( def listener(session: Session) -> None: nonlocal listener_session, listener_thread try: - async_session = get_resource(AsyncSession) + async_session = get_resource_nowait(AsyncSession) except NoCurrentContext: return @@ -301,10 +300,12 @@ def listener(session: Session) -> None: try: async with Context(): await component.start() - engine = require_resource(AsyncEngine) - dbsession = require_resource(AsyncSession) + engine = get_resource_nowait(AsyncEngine) + dbsession = get_resource_nowait(AsyncSession) await dbsession.run_sync( - lambda session: Person.metadata.create_all(session.bind) + lambda session: Person.metadata.create_all( + session.bind # type: ignore[arg-type] + ) ) dbsession.add(Person(name="Test person")) diff --git a/tests/test_testing_recipe.py b/tests/test_testing_recipe.py index 70aa401..1dbb912 100644 --- a/tests/test_testing_recipe.py +++ b/tests/test_testing_recipe.py @@ -9,7 +9,7 @@ from typing import Any import pytest -from asphalt.core import ContainerComponent, Context, require_resource +from asphalt.core import ContainerComponent, Context, get_resource_nowait from sqlalchemy.engine import Connection, Engine from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession @@ -65,7 +65,7 @@ async def test_rollback( async with Context(): await root_component.start() async with Context(): - session = require_resource(Session) + session = get_resource_nowait(Session) try: # No value for a non-nullable column => IntegrityError! session.add(Person()) @@ -87,7 +87,7 @@ async def test_add_person( async with Context(): await root_component.start() async with Context(): - session = require_resource(Session) + session = get_resource_nowait(Session) session.add(Person(name="Another person")) # The testing code should see both rows now @@ -100,7 +100,7 @@ async def test_delete_person( async with Context(): await root_component.start() async with Context(): - session = require_resource(Session) + session = get_resource_nowait(Session) session.execute(delete(Person)) # The testing code should not see any rows now @@ -153,7 +153,7 @@ async def test_rollback( async with Context(): await root_component.start() async with Context(): - session = require_resource(AsyncSession) + session = get_resource_nowait(AsyncSession) try: # No value for a non-nullable column => IntegrityError! session.add(Person()) @@ -175,7 +175,7 @@ async def test_add_person( async with Context(): await root_component.start() async with Context(): - session = require_resource(AsyncSession) + session = get_resource_nowait(AsyncSession) session.add(Person(name="Another person")) # The testing code should see both rows now @@ -188,7 +188,7 @@ async def test_delete_person( async with Context(): await root_component.start() async with Context(): - session = require_resource(AsyncSession) + session = get_resource_nowait(AsyncSession) await session.execute(delete(Person)) # The testing code should not see any rows now From 8b6ca10f3437588cb5afcea2da82b5734ae58032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 26 Mar 2024 19:41:00 +0200 Subject: [PATCH 16/29] Updated pre-commit modules --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 945e4d7..337227b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,14 +16,14 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.0 + rev: v0.3.4 hooks: - id: ruff args: [--fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.9.0 hooks: - id: mypy additional_dependencies: From c2d8b9c79ebbeb16c4fd1b9197ec09590d018fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Fri, 19 Apr 2024 01:32:16 +0300 Subject: [PATCH 17/29] Updated use of run_application() in examples to match the latest changes --- examples/csvimport.py | 3 +-- examples/csvimport_orm.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/csvimport.py b/examples/csvimport.py index 4d51e7c..cd2bc08 100644 --- a/examples/csvimport.py +++ b/examples/csvimport.py @@ -9,7 +9,6 @@ import logging from pathlib import Path -import anyio from anyio import to_thread from asphalt.core import ( CLIApplicationComponent, @@ -72,4 +71,4 @@ def insert_rows_in_thread() -> int: logger.info("Imported %d rows of data", inserted_rows) -anyio.run(run_application, CSVImporterComponent()) +run_application(CSVImporterComponent()) diff --git a/examples/csvimport_orm.py b/examples/csvimport_orm.py index c76c3f2..7d8216b 100644 --- a/examples/csvimport_orm.py +++ b/examples/csvimport_orm.py @@ -9,7 +9,6 @@ import logging from pathlib import Path -import anyio from anyio import to_thread from asphalt.core import ( CLIApplicationComponent, @@ -77,4 +76,4 @@ def insert_rows_in_thread() -> int: logger.info("Imported %d rows of data", inserted_rows) -anyio.run(run_application, CSVImporterComponent()) +run_application(CSVImporterComponent()) From f0a4b265530ba68d2387f4179d0357fc6e31256d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 6 May 2024 01:46:39 +0300 Subject: [PATCH 18/29] Removed uses of GeneratedResource --- src/asphalt/sqlalchemy/_component.py | 12 +++++++----- src/asphalt/sqlalchemy/_utils.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/asphalt/sqlalchemy/_component.py b/src/asphalt/sqlalchemy/_component.py index deca35c..6bc6c35 100644 --- a/src/asphalt/sqlalchemy/_component.py +++ b/src/asphalt/sqlalchemy/_component.py @@ -9,9 +9,9 @@ from anyio import CapacityLimiter, to_thread from asphalt.core import ( Component, - GeneratedResource, add_resource, add_resource_factory, + add_teardown_callback, qualified_name, resolve_reference, ) @@ -170,7 +170,7 @@ def __init__( else: self._sessionmaker = sessionmaker(bind=self._bind, **session_args) - def create_session(self) -> GeneratedResource[Session]: + def create_session(self) -> Session: async def teardown_session() -> None: try: if session.in_transaction(): @@ -186,9 +186,10 @@ async def teardown_session() -> None: session.close() session = self._sessionmaker() - return GeneratedResource(session, teardown_session) + add_teardown_callback(teardown_session) + return session - def create_async_session(self) -> GeneratedResource[AsyncSession]: + def create_async_session(self) -> AsyncSession: async def teardown_session() -> None: try: if session.in_transaction(): @@ -200,7 +201,8 @@ async def teardown_session() -> None: await session.close() session: AsyncSession = self._async_sessionmaker() - return GeneratedResource(session, teardown_session) + add_teardown_callback(teardown_session) + return session async def start(self) -> None: bind: Connection | Engine | AsyncConnection | AsyncEngine diff --git a/src/asphalt/sqlalchemy/_utils.py b/src/asphalt/sqlalchemy/_utils.py index ff5cd1f..a2a6b79 100644 --- a/src/asphalt/sqlalchemy/_utils.py +++ b/src/asphalt/sqlalchemy/_utils.py @@ -47,7 +47,7 @@ async def clear_async_database( # Reflect the schema to get the list of the tables, views and constraints metadata = MetaData() await connection.run_sync( - metadata.reflect, # type: ignore[arg-type] + metadata.reflect, schema=schema, views=True, ) From af9b4ef6c76c5b2ca2d8f91e88e5c34523172b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 11 Jun 2024 01:40:54 +0300 Subject: [PATCH 19/29] Updated code to work with the latest changes in the core --- .pre-commit-config.yaml | 6 +++--- docs/configuration.rst | 3 +-- docs/usage.rst | 25 ++++--------------------- examples/csvimport.py | 2 +- examples/csvimport_orm.py | 2 +- tests/test_testing_recipe.py | 34 +++++++++++++++++----------------- 6 files changed, 27 insertions(+), 45 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 337227b..15c6963 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ # * Run "pre-commit install". repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-toml - id: check-yaml @@ -16,14 +16,14 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.4 + rev: v0.4.8 hooks: - id: ruff args: [--fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.10.0 hooks: - id: mypy additional_dependencies: diff --git a/docs/configuration.rst b/docs/configuration.rst index 3773a02..181079a 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -63,8 +63,7 @@ of the ``sqlalchemy`` component:: sqlalchemy: resource_name: db1 url: postgresql+asyncpg:///mydatabase - sqlalchemy2: - type: sqlalchemy + sqlalchemy/db2: resource_name: db2 url: sqlite+aiosqlite:///mydb.sqlite diff --git a/docs/usage.rst b/docs/usage.rst index 0e2638c..a28a304 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -51,26 +51,10 @@ wrapped in worker threads. Another thing to watch out for is lazy loading of relationships and deferred columns which triggers implicit queries when those attributes are accessed. -On Python 3.9 and above, you would do:: - - from asyncio import to_thread - - from asphalt.core import inject, resource - from sqlalchemy.orm import Session - from sqlalchemy.future import select - - from yourapp.model import Person - - - @inject - async def handler(*, dbsession: Session = resource()) -> None: - people = await to_thread(dbsession.scalars, select(Person)) - - -On earlier Python versions:: - - from asyncio import get_running_loop +Here's an example of how you could use worker threads to run a blocking database +operation:: + from anyio import to_thread from asphalt.core import inject, resource from sqlalchemy.orm import Session from sqlalchemy.future import select @@ -80,8 +64,7 @@ On earlier Python versions:: @inject async def handler(*, dbsession: Session = resource()) -> None: - loop = get_running_loop() - people = await loop.run_in_executor(None, dbsession.scalars, select(Person)) + people = await to_thread.run_sync(dbsession.scalars, select(Person)) Releasing database resources during a long operation ---------------------------------------------------- diff --git a/examples/csvimport.py b/examples/csvimport.py index cd2bc08..e3c15f6 100644 --- a/examples/csvimport.py +++ b/examples/csvimport.py @@ -71,4 +71,4 @@ def insert_rows_in_thread() -> int: logger.info("Imported %d rows of data", inserted_rows) -run_application(CSVImporterComponent()) +run_application(CSVImporterComponent) diff --git a/examples/csvimport_orm.py b/examples/csvimport_orm.py index 7d8216b..79e753e 100644 --- a/examples/csvimport_orm.py +++ b/examples/csvimport_orm.py @@ -76,4 +76,4 @@ def insert_rows_in_thread() -> int: logger.info("Imported %d rows of data", inserted_rows) -run_application(CSVImporterComponent()) +run_application(CSVImporterComponent) diff --git a/tests/test_testing_recipe.py b/tests/test_testing_recipe.py index 1dbb912..8acf1e1 100644 --- a/tests/test_testing_recipe.py +++ b/tests/test_testing_recipe.py @@ -9,7 +9,7 @@ from typing import Any import pytest -from asphalt.core import ContainerComponent, Context, get_resource_nowait +from asphalt.core import Component, Context, get_resource_nowait, start_component from sqlalchemy.engine import Connection, Engine from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession @@ -49,8 +49,8 @@ def person(self, connection: Connection) -> Person: return person @pytest.fixture - def root_component(self, connection: Connection) -> ContainerComponent: - return ContainerComponent({"sqlalchemy": {"bind": connection}}) + def component_config(self, connection: Connection) -> dict[str, Any]: + return {"components": {"sqlalchemy": {"bind": connection}}} @pytest.fixture def dbsession(self, connection: Connection) -> Generator[Session, Any, None]: @@ -59,11 +59,11 @@ def dbsession(self, connection: Connection) -> Generator[Session, Any, None]: yield session async def test_rollback( - self, dbsession: Session, root_component: ContainerComponent + self, dbsession: Session, component_config: dict[str, Any] ) -> None: # Simulate a rollback happening in a subcontext async with Context(): - await root_component.start() + await start_component(Component, component_config) async with Context(): session = get_resource_nowait(Session) try: @@ -81,11 +81,11 @@ async def test_rollback( assert dbsession.scalar(func.count(Person.id)) == 2 async def test_add_person( - self, dbsession: Session, root_component: ContainerComponent + self, dbsession: Session, component_config: dict[str, Any] ) -> None: # Simulate adding a row to the "people" table in the application async with Context(): - await root_component.start() + await start_component(Component, component_config) async with Context(): session = get_resource_nowait(Session) session.add(Person(name="Another person")) @@ -94,11 +94,11 @@ async def test_add_person( assert dbsession.scalar(func.count(Person.id)) == 2 async def test_delete_person( - self, dbsession: Session, root_component: ContainerComponent + self, dbsession: Session, component_config: dict[str, Any] ) -> None: # Simulate removing the test person in the application async with Context(): - await root_component.start() + await start_component(Component, component_config) async with Context(): session = get_resource_nowait(Session) session.execute(delete(Person)) @@ -135,8 +135,8 @@ async def person(self, connection: AsyncConnection) -> Person: return person @pytest.fixture - def root_component(self, connection: AsyncConnection) -> ContainerComponent: - return ContainerComponent({"sqlalchemy": {"bind": connection}}) + def component_config(self, connection: Connection) -> dict[str, Any]: + return {"components": {"sqlalchemy": {"bind": connection}}} @pytest.fixture async def dbsession( @@ -147,11 +147,11 @@ async def dbsession( yield session async def test_rollback( - self, dbsession: AsyncSession, root_component: ContainerComponent + self, dbsession: AsyncSession, component_config: dict[str, Any] ) -> None: # Simulate a rollback happening in a subcontext async with Context(): - await root_component.start() + await start_component(Component, component_config) async with Context(): session = get_resource_nowait(AsyncSession) try: @@ -169,11 +169,11 @@ async def test_rollback( assert await dbsession.scalar(func.count(Person.id)) == 2 async def test_add_person( - self, dbsession: AsyncSession, root_component: ContainerComponent + self, dbsession: AsyncSession, component_config: dict[str, Any] ) -> None: # Simulate adding a row to the "people" table in the application async with Context(): - await root_component.start() + await start_component(Component, component_config) async with Context(): session = get_resource_nowait(AsyncSession) session.add(Person(name="Another person")) @@ -182,11 +182,11 @@ async def test_add_person( assert await dbsession.scalar(select(func.count(Person.id))) == 2 async def test_delete_person( - self, dbsession: AsyncSession, root_component: ContainerComponent + self, dbsession: AsyncSession, component_config: dict[str, Any] ) -> None: # Simulate removing the test person in the application async with Context(): - await root_component.start() + await start_component(Component, component_config) async with Context(): session = get_resource_nowait(AsyncSession) await session.execute(delete(Person)) From 3cb8d0aaa88934634f8f0773ec0e3c42d3f7d853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 14 Dec 2024 12:56:56 +0200 Subject: [PATCH 20/29] Asphalt5 update --- .github/workflows/test.yml | 10 ++++++---- .pre-commit-config.yaml | 6 +++--- .readthedocs.yml | 2 +- pyproject.toml | 9 ++++----- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a85fb66..f00bd2a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test suite on: push: - branches: [master] + branches: [master, "asphalt5"] pull_request: jobs: @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["3.8", "3.10", "3.11", "3.12", "pypy-3.9"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "pypy-3.10"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -22,9 +22,11 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - name: Start external services - run: docker-compose up -d + run: docker compose up -d - name: Install dependencies - run: pip install -e .[test] + run: | + pip install git+https://github.com/asphalt-framework/asphalt.git@5.0 + pip install -e .[test] coverage - name: Test with pytest run: coverage run -m pytest -v - name: Generate coverage report diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15c6963..2da49ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ # * Run "pre-commit install". repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-toml - id: check-yaml @@ -16,14 +16,14 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.8 + rev: v0.8.3 hooks: - id: ruff args: [--fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.13.0 hooks: - id: mypy additional_dependencies: diff --git a/.readthedocs.yml b/.readthedocs.yml index 026b967..ac1099a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.8" + python: "3.9" sphinx: configuration: docs/conf.py diff --git a/pyproject.toml b/pyproject.toml index d886966..23ac855 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,13 @@ classifiers = [ "Topic :: Database", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "asphalt ~= 4.11", "SQLAlchemy[asyncio] >= 2.0", @@ -40,7 +40,6 @@ test = [ "aiosqlite", "anyio[trio] >= 4.2", "asyncmy; platform_python_implementation == 'CPython'", - "coverage >= 7", "pymysql", "psycopg >= 3.1; platform_python_implementation == 'CPython'", "pytest >= 7.4", @@ -80,7 +79,7 @@ addopts = "-rsx --tb=short" testpaths = ["tests"] [tool.mypy] -python_version = "3.8" +python_version = "3.9" strict = true explicit_package_bases = true @@ -95,7 +94,7 @@ show_missing = true [tool.tox] legacy_tox_ini = """ [tox] -envlist = py38, py39, py310, py311, py312, pypy3 +envlist = py39, py310, py311, py312, py313, pypy3 skip_missing_interpreters = true minversion = 4.0 From a9910d5716162d08d525d23f8477faee9a5390a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 14 Dec 2024 13:06:57 +0200 Subject: [PATCH 21/29] Don't test on pypy --- .github/workflows/test.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f00bd2a..7f95541 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "pypy-3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index 23ac855..deaf6c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,13 +62,13 @@ local_scheme = "dirty-tag" [tool.ruff.lint] select = [ "ASYNC", # flake8-async - "E", "F", "W", # default Flake8 "G", # flake8-logging-format "I", # isort "ISC", # flake8-implicit-str-concat "PGH", # pygrep-hooks "RUF100", # unused noqa (yesqa) "UP", # pyupgrade + "W", # pycodestyle warnings ] [tool.ruff.lint.isort] From c8b0275265f416b9c89307486bfe8bcea5fcce50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 30 Dec 2024 17:29:33 +0200 Subject: [PATCH 22/29] Updated configs, docs and some code for Asphalt 5 --- .github/workflows/test.yml | 4 +-- docs/conf.py | 2 ++ docs/configuration.rst | 34 ++++++----------------- pyproject.toml | 32 +++++++++++----------- src/asphalt/sqlalchemy/_component.py | 29 ++++---------------- tests/test_component.py | 40 ++++++++-------------------- 6 files changed, 43 insertions(+), 98 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f95541..85e99af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,9 +24,7 @@ jobs: - name: Start external services run: docker compose up -d - name: Install dependencies - run: | - pip install git+https://github.com/asphalt-framework/asphalt.git@5.0 - pip install -e .[test] coverage + run: pip install -e .[test] - name: Test with pytest run: coverage run -m pytest -v - name: Generate coverage report diff --git a/docs/conf.py b/docs/conf.py index 838e2e3..22b23f5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,6 +7,7 @@ "sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx_autodoc_typehints", + "sphinx_rtd_theme", "sphinx_tabs.tabs", ] @@ -26,6 +27,7 @@ exclude_patterns = ["_build"] pygments_style = "sphinx" autodoc_default_options = {"members": True, "show-inheritance": True} +autodoc_inherit_docstrings = False highlight_language = "python3" todo_include_todos = False diff --git a/docs/configuration.rst b/docs/configuration.rst index 181079a..a0da32f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -5,16 +5,16 @@ Configuration A typical SQLAlchemy configuration consists of a single database. At minimum, you only need a connection URL (see the -`SQLAlchemy documentation`_ for how to construct one). Such a configuration would look -something like this:: +:doc:`SQLAlchemy documentation ` on how to construct one). +Such a configuration would look something like this:: components: sqlalchemy: - url: postgresql+asyncpg://user:password@10.0.0.8/mydatabase + url: postgresql+psycopg://user:password@10.0.0.8/mydatabase This will add two static resources and one resource factory, as follows: -* engine (type: :class:`sqlalchemy.future.engine.Engine` or +* engine (type: :class:`sqlalchemy.engine.Engine` or :class:`sqlalchemy.ext.asyncio.AsyncEngine`) * sessionmaker (type: :class:`sqlalchemy.orm.sessionmaker`) * session factory (generates :class:`~sqlalchemy.orm.Session` or @@ -26,7 +26,7 @@ sharing features):: components: sqlalchemy: url: - drivername: postgresql+asyncpg + drivername: postgresql+psycopg username: user password: password host: 10.0.0.8 @@ -34,15 +34,14 @@ sharing features):: .. seealso:: * :class:`sqlalchemy.engine.URL` - * :class:`asphalt.sqlalchemy.component.SQLAlchemyComponent` - -.. _SQLAlchemy documentation: https://docs.sqlalchemy.org/en/14/core/engines.html + * :class:`asphalt.sqlalchemy.SQLAlchemyComponent` Setting engine or session options --------------------------------- If you need to adjust the options used for creating new sessions, or pass extra -arguments to the engine, you can do so by specifying them in the ``session`` option:: +arguments to the engine, you can do so by specifying them in the ``session_args`` and +``engine_args``, respectively:: components: sqlalchemy: @@ -52,20 +51,3 @@ arguments to the engine, you can do so by specifying them in the ``session`` opt session_args: info: hello: world - -Multiple databases ------------------- - -If you need to work with multiple databases, you will need to use multiple instances -of the ``sqlalchemy`` component:: - - components: - sqlalchemy: - resource_name: db1 - url: postgresql+asyncpg:///mydatabase - sqlalchemy/db2: - resource_name: db2 - url: sqlite+aiosqlite:///mydb.sqlite - -This will make the appropriate resources available using their respective namespaces -(``db1`` or ``db2`` instead of ``default``). diff --git a/pyproject.toml b/pyproject.toml index deaf6c7..c5a4d6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ - "asphalt ~= 4.11", + "asphalt @ git+https://github.com/asphalt-framework/asphalt", "SQLAlchemy[asyncio] >= 2.0", ] dynamic = ["version"] @@ -40,6 +40,7 @@ test = [ "aiosqlite", "anyio[trio] >= 4.2", "asyncmy; platform_python_implementation == 'CPython'", + "coverage >= 7", "pymysql", "psycopg >= 3.1; platform_python_implementation == 'CPython'", "pytest >= 7.4", @@ -73,9 +74,10 @@ select = [ [tool.ruff.lint.isort] known-first-party = ["asphalt.sqlalchemy"] +known-third-party = ["asphalt.core"] [tool.pytest.ini_options] -addopts = "-rsx --tb=short" +addopts = ["-rsfE", "--tb=short"] testpaths = ["tests"] [tool.mypy] @@ -92,20 +94,18 @@ branch = true show_missing = true [tool.tox] -legacy_tox_ini = """ -[tox] -envlist = py39, py310, py311, py312, py313, pypy3 +env_list = ["py39", "py310", "py311", "py312", "py313"] skip_missing_interpreters = true -minversion = 4.0 -[testenv] -extras = test -commands = python -m pytest {posargs} -setenv = - MYSQL_URL = mysql+pymysql://root@localhost:33060/asphalttest - POSTGRESQL_URL = postgresql+psycopg://postgres:secret@localhost:54320/asphalttest - -[testenv:docs] -extras = doc -commands = sphinx-build -W docs build/sphinx +[tool.tox.env_run_base] +commands = [["python", "-m", "pytest", { replace = "posargs", extend = true }]] +package = "editable" +extras = ["test"] +setenv = """ + MYSQL_URL = mysql+pymysql://root@localhost:33060/asphalttest + POSTGRESQL_URL = postgresql+psycopg://postgres:secret@localhost:54320/asphalttest """ + +[tool.tox.env.docs] +commands = [["sphinx-build", "-W", "-n", "docs", "build/sphinx", { replace = "posargs", extend = true }]] +extras = ["doc"] diff --git a/src/asphalt/sqlalchemy/_component.py b/src/asphalt/sqlalchemy/_component.py index 6bc6c35..4813efa 100644 --- a/src/asphalt/sqlalchemy/_component.py +++ b/src/asphalt/sqlalchemy/_component.py @@ -30,8 +30,6 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import Pool -logger = logging.getLogger(__name__) - class SQLAlchemyComponent(Component): """ @@ -59,15 +57,15 @@ class SQLAlchemyComponent(Component): * ``expire_on_commit``: ``False`` :param url: the connection url passed to - :func:`~sqlalchemy.engine.create_engine` + :func:`~sqlalchemy.create_engine` (can also be a dictionary of :class:`~sqlalchemy.engine.url.URL` keyword arguments) :param bind: a connection or engine to use instead of creating a new engine :param prefer_async: if ``True``, try to create an async engine rather than a synchronous one, in cases like ``psycopg`` where the driver supports both :param engine_args: extra keyword arguments passed to - :func:`sqlalchemy.engine.create_engine` or - :func:`sqlalchemy.ext.asyncio.create_engine` + :func:`sqlalchemy.create_engine` or + :func:`sqlalchemy.ext.asyncio.create_async_engine` :param session_args: extra keyword arguments passed to :class:`~sqlalchemy.orm.session.Session` or :class:`~sqlalchemy.ext.asyncio.AsyncSession` @@ -76,9 +74,8 @@ class SQLAlchemyComponent(Component): :param ready_callback: a callable that is called right before the resources are added to the context (can be a coroutine function too) :param poolclass: the SQLAlchemy pool class (or a textual reference to one) to use; - passed to :func:`sqlalchemy.engine.create_engine` or - :func:`sqlalchemy.ext.asyncio.create_engine` - :param resource_name: name space for the database resources + passed to :func:`sqlalchemy.create_engine` or + :func:`sqlalchemy.ext.asyncio.create_async_engine` """ _engine: Engine | AsyncEngine @@ -96,9 +93,7 @@ def __init__( commit_executor_workers: int = 50, ready_callback: Callable[[Engine, sessionmaker[Any]], Any] | str | None = None, poolclass: str | type[Pool] | None = None, - resource_name: str = "default", ): - self.resource_name = resource_name self.commit_thread_limiter = CapacityLimiter(commit_executor_workers) self.ready_callback = resolve_reference(ready_callback) engine_args = engine_args or {} @@ -220,23 +215,19 @@ async def start(self) -> None: add_resource( self._engine, - self.resource_name, description="SQLAlchemy engine (asynchronous)", teardown_callback=teardown_callback, ) add_resource( self._sessionmaker, - self.resource_name, description="SQLAlchemy session factory (synchronous)", ) add_resource( self._async_sessionmaker, - self.resource_name, description="SQLAlchemy session factory (asynchronous)", ) add_resource_factory( self.create_async_session, - self.resource_name, description="SQLAlchemy session (asynchronous)", ) else: @@ -253,24 +244,14 @@ async def start(self) -> None: add_resource( self._engine, - self.resource_name, description="SQLAlchemy engine (synchronous)", teardown_callback=teardown_callback, ) add_resource( self._sessionmaker, - self.resource_name, description="SQLAlchemy session factory (synchronous)", ) add_resource_factory( self.create_session, - self.resource_name, description="SQLAlchemy session (synchronous)", ) - - logger.info( - "Configured SQLAlchemy resources (%s; dialect=%s, driver=%s)", - self.resource_name, - bind.dialect.name, - bind.dialect.driver, - ) diff --git a/tests/test_component.py b/tests/test_component.py index 33c4eac..bc714ea 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -34,47 +34,29 @@ pytestmark = pytest.mark.anyio -@pytest.mark.parametrize( - "component_opts, args", - [ - pytest.param({}, ()), - pytest.param({"resource_name": "alternate"}, ("alternate",)), - ], -) -async def test_component_start_sync( - component_opts: dict[str, Any], args: tuple[Any] -) -> None: +async def test_component_start_sync() -> None: """Test that the component creates all the expected (synchronous) resources.""" url = URL.create("sqlite", database=":memory:") - component = SQLAlchemyComponent(url=url, **component_opts) + component = SQLAlchemyComponent(url=url) async with Context(): await component.start() - get_resource_nowait(Engine, *args) - get_resource_nowait(sessionmaker, *args) - get_resource_nowait(Session, *args) + get_resource_nowait(Engine) + get_resource_nowait(sessionmaker) + get_resource_nowait(Session) -@pytest.mark.parametrize( - "component_opts, args", - [ - pytest.param({}, ()), - pytest.param({"resource_name": "alternate"}, ("alternate",)), - ], -) -async def test_component_start_async( - component_opts: dict[str, Any], args: tuple[Any] -) -> None: +async def test_component_start_async() -> None: """Test that the component creates all the expected (asynchronous) resources.""" url = URL.create("sqlite+aiosqlite", database=":memory:") - component = SQLAlchemyComponent(url=url, **component_opts) + component = SQLAlchemyComponent(url=url) async with Context(): await component.start() - get_resource_nowait(AsyncEngine, *args) - async_session_class = get_resource_nowait(async_sessionmaker, *args) - get_resource_nowait(AsyncSession, *args) - sync_session_class = get_resource_nowait(sessionmaker, *args) + get_resource_nowait(AsyncEngine) + async_session_class = get_resource_nowait(async_sessionmaker) + get_resource_nowait(AsyncSession) + sync_session_class = get_resource_nowait(sessionmaker) assert async_session_class.kw["sync_session_class"] is sync_session_class From 3c54df02439601da28f7251dbfa9328f83a3066c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 30 Dec 2024 17:30:29 +0200 Subject: [PATCH 23/29] Updated pre-commit modules --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2da49ab..727687f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,18 +16,18 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.3 + rev: v0.8.4 hooks: - id: ruff args: [--fix, --show-fixes] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.13.0 + rev: v1.14.0 hooks: - id: mypy additional_dependencies: - - asphalt@git+https://github.com/asphalt-framework/asphalt@5.0 + - asphalt@git+https://github.com/asphalt-framework/asphalt - pytest - sqlalchemy From d3af4d12e056c157b3bb5919a5fcb980da3c3214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 30 Dec 2024 17:33:58 +0200 Subject: [PATCH 24/29] Updated the changelog --- docs/versionhistory.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/versionhistory.rst b/docs/versionhistory.rst index fb0543d..fd03e66 100644 --- a/docs/versionhistory.rst +++ b/docs/versionhistory.rst @@ -3,6 +3,13 @@ Version history This library adheres to `Semantic Versioning 2.0 `_. +**UNRELEASED** + +- Dropped support for Python 3.8 +- **BACKWARD INCOMPATIBLE** Bumped minimum Asphalt version to 5.0 +- **BACKWARD INCOMPATIBLE** Dropped the ``resource_name`` option in favor of Asphalt's + native alternate resource name syntax + **5.1.0** (2024-01-16) - Dropped support for Python 3.7 From b29faf5f6ef218c181177a00ea0843e4c43a1eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 30 Dec 2024 17:37:09 +0200 Subject: [PATCH 25/29] Fixed mypy errors --- tests/test_component.py | 6 +++--- tests/test_utils.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_component.py b/tests/test_component.py index bc714ea..2117f46 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -4,7 +4,7 @@ from contextlib import AsyncExitStack from pathlib import Path from threading import Thread, current_thread -from typing import Any +from typing import Any, cast import pytest from asphalt.core import ( @@ -151,7 +151,7 @@ async def test_close_twice_sync(psycopg_url: str) -> None: await component.start() session = get_resource_nowait(Session) assert isinstance(session.bind, Engine) - pool = session.bind.pool + pool = cast(QueuePool, session.bind.pool) assert isinstance(pool, QueuePool) session.execute(text("SELECT 1")) assert pool.checkedout() == 1 @@ -170,7 +170,7 @@ async def test_close_twice_async(psycopg_url_async: str) -> None: await component.start() session = get_resource_nowait(AsyncSession) assert isinstance(session.bind, AsyncEngine) - pool = session.bind.pool + pool = cast(QueuePool, session.bind.pool) assert isinstance(pool, AsyncAdaptedQueuePool) await session.execute(text("SELECT 1")) assert pool.checkedout() == 1 diff --git a/tests/test_utils.py b/tests/test_utils.py index 5e21c70..d7d1a05 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,7 +19,7 @@ def connection(sync_engine: Engine) -> Generator[Connection, Any, None]: Table("table", metadata, Column("column1", Integer, primary_key=True)) Table("table2", metadata, Column("fk_column", ForeignKey("table.column1"))) if conn.dialect.name != "sqlite": - conn.execute(CreateSchema("altschema")) # type: ignore[no-untyped-call] + conn.execute(CreateSchema("altschema")) Table("table3", metadata, Column("fk_column", Integer), schema="altschema") metadata.create_all(conn) @@ -28,7 +28,7 @@ def connection(sync_engine: Engine) -> Generator[Connection, Any, None]: if conn.dialect.name != "sqlite": metadata.drop_all(conn) - conn.execute(DropSchema("altschema")) # type: ignore[no-untyped-call] + conn.execute(DropSchema("altschema")) def test_clear_database(connection: Connection) -> None: From 4d9f1c74a73aa28b3d47da091372224357d18cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 30 Dec 2024 23:36:25 +0200 Subject: [PATCH 26/29] Added mypy_path --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index c5a4d6c..7abac1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ explicit_package_bases = true source = ["asphalt.sqlalchemy"] relative_files = true branch = true +mypy_path = ["src", "tests", "examples"] [tool.coverage.report] show_missing = true From a5bafe297a405c94cb7dbdd30d05c2a0c5548e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 30 Dec 2024 23:36:38 +0200 Subject: [PATCH 27/29] Updated the examples for Asphalt 5 --- examples/csvimport.py | 16 +++++++--------- examples/csvimport_orm.py | 16 +++++++--------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/examples/csvimport.py b/examples/csvimport.py index e3c15f6..0efe01c 100644 --- a/examples/csvimport.py +++ b/examples/csvimport.py @@ -37,19 +37,17 @@ class CSVImporterComponent(CLIApplicationComponent): def __init__(self) -> None: super().__init__() self.csv_path = Path(__file__).with_name("people.csv") - - async def start(self) -> None: - # Remove the db file if it exists - db_path = self.csv_path.with_name("people.db") - if db_path.exists(): - db_path.unlink() - + self.db_path = self.csv_path.with_name("people.db") self.add_component( "sqlalchemy", - url=f"sqlite:///{db_path}", + url=f"sqlite:///{self.db_path}", ready_callback=lambda bind, factory: metadata.create_all(bind), ) - await super().start() + + async def start(self) -> None: + # Remove the db file if it exists + if self.db_path.exists(): + self.db_path.unlink() @inject async def run(self, *, dbsession: Session = resource()) -> None: diff --git a/examples/csvimport_orm.py b/examples/csvimport_orm.py index 79e753e..1bab954 100644 --- a/examples/csvimport_orm.py +++ b/examples/csvimport_orm.py @@ -39,21 +39,19 @@ class CSVImporterComponent(CLIApplicationComponent): def __init__(self) -> None: super().__init__() self.csv_path = Path(__file__).with_name("people.csv") - - async def start(self) -> None: - # Remove the db file if it exists - db_path = self.csv_path.with_name("people.db") - if db_path.exists(): - db_path.unlink() - + self.db_path = self.csv_path.with_name("people.db") self.add_component( "sqlalchemy", - url=f"sqlite:///{db_path}", + url=f"sqlite:///{self.db_path}", ready_callback=lambda bind, factory: DeclarativeBase.metadata.create_all( bind ), ) - await super().start() + + async def start(self) -> None: + # Remove the db file if it exists + if self.db_path.exists(): + self.db_path.unlink() @inject async def run(self, *, dbsession: Session = resource()) -> None: From cb889714ab34e662692fce8599e331ff66ae731e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 30 Dec 2024 23:47:18 +0200 Subject: [PATCH 28/29] Removed the existing DB in prepare() rather than start() --- examples/csvimport.py | 2 +- examples/csvimport_orm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/csvimport.py b/examples/csvimport.py index 0efe01c..e766b91 100644 --- a/examples/csvimport.py +++ b/examples/csvimport.py @@ -44,7 +44,7 @@ def __init__(self) -> None: ready_callback=lambda bind, factory: metadata.create_all(bind), ) - async def start(self) -> None: + async def prepare(self) -> None: # Remove the db file if it exists if self.db_path.exists(): self.db_path.unlink() diff --git a/examples/csvimport_orm.py b/examples/csvimport_orm.py index 1bab954..7c5efad 100644 --- a/examples/csvimport_orm.py +++ b/examples/csvimport_orm.py @@ -48,7 +48,7 @@ def __init__(self) -> None: ), ) - async def start(self) -> None: + async def prepare(self) -> None: # Remove the db file if it exists if self.db_path.exists(): self.db_path.unlink() From 3ff52f2a1ad121a012bd86dd17d4febc5616347f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 31 Dec 2024 02:34:08 +0200 Subject: [PATCH 29/29] Fixed config oopsie and increased test coverage --- pyproject.toml | 2 +- src/asphalt/sqlalchemy/_component.py | 9 ++++-- tests/test_component.py | 9 ++++-- tests/test_utils.py | 43 ++++++++++++++++++++++++++-- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7abac1c..361fe8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,12 +84,12 @@ testpaths = ["tests"] python_version = "3.9" strict = true explicit_package_bases = true +mypy_path = ["src", "tests", "examples"] [tool.coverage.run] source = ["asphalt.sqlalchemy"] relative_files = true branch = true -mypy_path = ["src", "tests", "examples"] [tool.coverage.report] show_missing = true diff --git a/src/asphalt/sqlalchemy/_component.py b/src/asphalt/sqlalchemy/_component.py index 4813efa..b877aa4 100644 --- a/src/asphalt/sqlalchemy/_component.py +++ b/src/asphalt/sqlalchemy/_component.py @@ -91,7 +91,10 @@ def __init__( engine_args: dict[str, Any] | None = None, session_args: dict[str, Any] | None = None, commit_executor_workers: int = 50, - ready_callback: Callable[[Engine, sessionmaker[Any]], Any] | str | None = None, + ready_callback: Callable[[Engine, sessionmaker[Any]], Any] + | Callable[[AsyncEngine, async_sessionmaker[Any]], Any] + | str + | None = None, poolclass: str | type[Pool] | None = None, ): self.commit_thread_limiter = CapacityLimiter(commit_executor_workers) @@ -112,7 +115,7 @@ def __init__( elif isinstance(bind, AsyncEngine): self._engine = self._async_bind = bind else: - raise TypeError(f"Incompatible bind argument: {qualified_name(bind)}") + raise TypeError(f"incompatible bind argument: {qualified_name(bind)}") else: if isinstance(url, dict): url = URL.create(**url) @@ -131,7 +134,7 @@ def __init__( if isinstance(poolclass, str): poolclass = resolve_reference(poolclass) - pool_class = cast("type[Pool]", poolclass) + pool_class = cast(type[Pool], poolclass) if prefer_async: try: self._engine = self._async_bind = create_async_engine( diff --git a/tests/test_component.py b/tests/test_component.py index 2117f46..e3b438f 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -34,6 +34,11 @@ pytestmark = pytest.mark.anyio +def test_bad_bind_argument() -> None: + with pytest.raises(TypeError, match="incompatible bind argument: str"): + SQLAlchemyComponent(bind="bad") # type: ignore[arg-type] + + async def test_component_start_sync() -> None: """Test that the component creates all the expected (synchronous) resources.""" url = URL.create("sqlite", database=":memory:") @@ -285,9 +290,7 @@ def listener(session: Session) -> None: engine = get_resource_nowait(AsyncEngine) dbsession = get_resource_nowait(AsyncSession) await dbsession.run_sync( - lambda session: Person.metadata.create_all( - session.bind # type: ignore[arg-type] - ) + lambda session: Person.metadata.create_all(session.bind) ) dbsession.add(Person(name="Test person")) diff --git a/tests/test_utils.py b/tests/test_utils.py index d7d1a05..98edd05 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,15 +1,18 @@ from __future__ import annotations -from collections.abc import Generator +from collections.abc import AsyncGenerator, Generator from typing import Any import pytest from sqlalchemy.engine import Connection, Engine +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine from sqlalchemy.sql.ddl import CreateSchema, DropSchema from sqlalchemy.sql.schema import Column, ForeignKey, MetaData, Table from sqlalchemy.sql.sqltypes import Integer -from asphalt.sqlalchemy import clear_database +from asphalt.sqlalchemy import clear_async_database, clear_database + +pytestmark = pytest.mark.anyio @pytest.fixture @@ -31,6 +34,27 @@ def connection(sync_engine: Engine) -> Generator[Connection, Any, None]: conn.execute(DropSchema("altschema")) +@pytest.fixture +async def async_connection( + async_engine: AsyncEngine, +) -> AsyncGenerator[AsyncConnection]: + async with async_engine.connect() as conn: + metadata = MetaData() + Table("table", metadata, Column("column1", Integer, primary_key=True)) + Table("table2", metadata, Column("fk_column", ForeignKey("table.column1"))) + if conn.dialect.name != "sqlite": + await conn.execute(CreateSchema("altschema")) + Table("table3", metadata, Column("fk_column", Integer), schema="altschema") + + await conn.run_sync(metadata.create_all) + + yield conn + + if conn.dialect.name != "sqlite": + await conn.run_sync(metadata.drop_all) + await conn.execute(DropSchema("altschema")) + + def test_clear_database(connection: Connection) -> None: clear_database( connection, ["altschema"] if connection.dialect.name != "sqlite" else [] @@ -43,3 +67,18 @@ def test_clear_database(connection: Connection) -> None: alt_metadata = MetaData(schema="altschema") alt_metadata.reflect(connection) assert len(alt_metadata.tables) == 0 + + +async def test_clear_async_database(async_connection: AsyncConnection) -> None: + await clear_async_database( + async_connection, + ["altschema"] if async_connection.dialect.name != "sqlite" else [], + ) + metadata = MetaData() + await async_connection.run_sync(metadata.reflect) + assert len(metadata.tables) == 0 + + if async_connection.dialect.name != "sqlite": + alt_metadata = MetaData(schema="altschema") + await async_connection.run_sync(alt_metadata.reflect) + assert len(alt_metadata.tables) == 0