diff --git a/docs/1_getting_started.md b/docs/1_getting_started.md index debd2c0..8be5550 100644 --- a/docs/1_getting_started.md +++ b/docs/1_getting_started.md @@ -26,9 +26,66 @@ from typedal.for_py4web import TypeDAL db = TypeDAL("sqlite:memory") ``` -TypeDAL accepts the same connection string format and other arguments as `pydal.DAL`. Again, -see [their documentation](http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#The-DAL-A-quick-tour) -for more info about this. +TypeDAL accepts the same connection string format and other arguments as `pydal.DAL`. +Again, see +[their documentation](http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#The-DAL-A-quick-tour) +for more info about this. For additional configuration options specific to TypeDAL, see the [7. Advanced Configuration](./7_configuration.md) page. When using py4web, it is recommended to import the py4web-specific TypeDAL, which is a Fixture that handles database -connections on request (just like the py4web specific DAL class does). +connections on request (just like the py4web specific DAL class does). More information about this can be found on [5. py4web](./5_py4web.md) + +### Simple Queries + +For direct SQL access, use `executesql()`: + +```python +rows = db.executesql("SELECT * FROM some_table") +``` + +#### Safely Injecting Variables + +Use t-strings (Python 3.14+) for automatic SQL escaping: + +```python +name = "Robert'); DROP TABLE Students;--" +rows = db.executesql(t"SELECT * FROM some_table WHERE name = {name}") +``` + +Or use the `placeholders` argument with positional or named parameters: + +```python +# Positional +rows = db.executesql( + "SELECT * FROM some_table WHERE name = %s AND age > %s", + placeholders=[name, 18] +) + +# Named +rows = db.executesql( + "SELECT * FROM some_table WHERE name = %(name)s AND age > %(age)s", + placeholders={"name": name, "age": 18} +) +``` + +#### Result Formatting + +By default, `executesql()` returns rows as tuples. To map results to specific fields, use `fields` (takes +Field/TypedField objects) or `colnames` (takes column name strings): + +```python +rows = db.executesql( + "SELECT id, name FROM some_table", + colnames=["id", "name"] +) + +rows = db.executesql( + "SELECT id, name FROM some_table", + fields=[some_table.id, some_table.name] # Requires table definition +) +``` + +You can also use `as_dict` or `as_ordered_dict` to return dictionaries instead of tuples. + +Most of the time, you probably don't want to write raw queries. For that, you'll need to define some tables! +Head to [2. Defining Tables](./2_defining_tables.md) to learn how. + diff --git a/docs/2_defining_tables.md b/docs/2_defining_tables.md index 51e77dc..c40c34e 100644 --- a/docs/2_defining_tables.md +++ b/docs/2_defining_tables.md @@ -102,5 +102,6 @@ def my_after_delete(query: Set): row.delete_record() # to trigger MyTable.where(...).delete() # to trigger - ``` + +"Now that we have some tables, it's time to actually query them! Let's go to [3. Building Queries](./3_building_queries.md) to learn how. \ No newline at end of file diff --git a/docs/3_building_queries.md b/docs/3_building_queries.md index 3fa98c2..d8207b0 100644 --- a/docs/3_building_queries.md +++ b/docs/3_building_queries.md @@ -30,27 +30,33 @@ The query builder uses the builder pattern, so you can keep adding to it (in any data: ```python -Person +builder = Person .where(Person.id > 0) -.where(Person.id < 99, Person.id == 101) +.where(Person.id < 99, Person.id == 101) # id < 99 OR id == 101 .select(Reference.id, Reference.title) .join('reference') .select(Person.ALL) -.paginate(limit=5, page=2) # final step: actually runs the query + +# final step: actually runs the query: +builder.paginate(limit=5, page=2) + +# to get the SQL (for debugging or subselects), you can use: +builder.to_sql() ``` ```sql -SELECT "person".*, - "reference"."id", - "reference"."title" -FROM "person", - "reference" -WHERE (("person"."id" IN (SELECT "person"."id" - FROM "person" - WHERE ((("person"."id" > 0)) AND - (("person"."id" < 99) OR ("person"."id" = 101))) - ORDER BY "person"."id" LIMIT 1 OFFSET 0)) - AND ("person"."reference" = "reference"."id") ); +SELECT "person".* + , "reference"."id" + , "reference"."title" + FROM "person" + , "reference" + WHERE (("person"."id" IN (SELECT "person"."id" + FROM "person" + WHERE ((("person"."id" > 0)) AND + (("person"."id" < 99) OR ("person"."id" = 101))) + ORDER BY "person"."id" + LIMIT 1 OFFSET 0)) + AND ("person"."reference" = "reference"."id")); ``` ### where @@ -58,27 +64,81 @@ WHERE (("person"."id" IN (SELECT "person"."id" In pydal, this is the part that would be in `db(...)`. Can be used in multiple ways: -- `.where(Query)` -> with a direct query such as `Table.id == 5` +- `.where(query)` -> with a direct query such as `query = (Table.id == 5)` - `.where(lambda table: table.id == 5)` -> with a query via a lambda - `.where(id=5)` -> via keyword arguments +- `.where({"id": 5})` -> via a dictionary (equivalent to keyword args) +- `.where(Table.field)` -> with a Field directly, checks if field is not null + +When using multiple `.where()` calls, they will be ANDed together: +`.where(lambda table: table.id == 5).where(active=True)` equals `(table.id == 5) & (table.active == True)` -When using multiple where's, they will be ANDed: -`.where(lambda table: table.id == 5).where(lambda table: table.id == 6)` equals `(table.id == 5) & (table.id=6)` -When passing multiple queries to a single .where, they will be ORed: -`.where(lambda table: table.id == 5, lambda table: table.id == 6)` equals `(table.id == 5) | (table.id=6)` +When passing multiple arguments to a single `.where()`, they will be ORed: +`.where({"id": 5}, {"id": 6})` equals `(table.id == 5) | (table.id == 6)` ### select -Here you can enter any number of fields as arguments: database columns by name ('id'), by field reference (table.id) or -other (e.g. table.ALL). +Here you can enter any number of fields as arguments: database columns by name ('id'), by field reference (Table.id), +other (e.g. Table.ALL), or Expression objects. ```python Person.select('id', Person.name, Person.ALL) # defaults to Person.ALL if select is omitted. ``` -You can also specify extra options such as `orderby` here. For supported keyword arguments, see +You can also specify extra options as keyword arguments. Supported options are: `orderby`, `groupby`, `limitby`, +`distinct`, `having`, `orderby_on_limitby`, `join`, `left`, `cache`, see also the [web2py docs](http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#orderby-groupby-limitby-distinct-having-orderby_on_limitby-join-left-cache). +```python +Person.select(Person.name, distinct=True) +``` + +If you only want a list of name strings (in this example) instead of Person instances, you could use the column() method +instead: + +```python +Person.column(Person.name, distinct=True) +``` + +You can use `.orderby(*fields)` as an alternative to `select(orderby=...)`: + +```python +Person.where(...).orderby(~Person.name, "age") +``` + +`.orderby()` accepts field references (`Table.field` or `"field_name""`), reverse ordering (`~Table.field`), or the +literal `""`. Multiple field references can be passed (except when using ``). + +#### Raw SQL Expressions + +For complex SQL that can't be expressed with field references, use `sql_expression()`: + +```python +# Simple arithmetic +expr = db.sql_expression("age * 2") +Person.select(expr) + +# Safe parameter injection with t-strings (Python 3.14+) +min_age = 21 +expr = db.sql_expression(t"age >= {min_age}", output_type="boolean") +Person.where(expr).select() + +# Positional arguments +expr = db.sql_expression("age > %s AND status = %s", 18, "active", output_type="boolean") +Person.where(expr).select() + +# Named arguments +expr = db.sql_expression( + "EXTRACT(year FROM %(date_col)s) = %(year)s", + date_col="created_at", + year=2023, + output_type="boolean" +) +Person.where(expr).select() +``` + +Expressions can be used in `where()`, `select()`, `orderby()`, and other query methods. + ### join Include relationship fields in the result. @@ -93,6 +153,8 @@ This can be overwritten with the `method` keyword argument (left or inner) Person.join('articles', method='inner') # will only yield persons that have related articles ``` +For more details about relationships and joins, see [4. Relationships](./4_relationships.md). + ### cache ```python @@ -120,7 +182,7 @@ time. class ... ``` -In order to enable this functionality, TypeDAL adds a `before update` and `before delete` hook to your tables, +In order to enable this functionality, TypeDAL adds a `before update` and `before delete` hook to your tables, which manages the dependencies. You can disable this behavior by passing `cache_dependency=False` to `db.define`. Be aware doing this might break some caching functionality! @@ -136,12 +198,15 @@ The Query Builder has a few operations that don't return a new builder instance: - paginate: this works similarly to `collect`, but returns a PaginatedRows instead, which has a `.next()` and `.previous()` method to easily load more pages. - collect_or_fail: where `collect` may return an empty result, this variant will raise an error if there are no results. +- execute: get the raw rows matching your query as returned by pydal, without entity mapping or relationship loading. + Useful for subqueries or when you need lower-level control. - first: get the first entity matching your query, possibly with relationships loaded (if .join was used) - first_or_fail: where `first` may return an empty result, this variant will raise an error if there are no results. +- to_sql: get the SQL string that would run, useful for debugging, subqueries and other advanced SQL operations. - update: instead of selecting rows, update those matching the current query (see [Delete](#delete)) - delete: instead of selecting rows, delete those matching the current query (see [Update](#update)) -Additionally, you can directly call `.all()`, `.collect()`, `.count()`, `.first()` on a model. +Additionally, you can directly call `.all()`, `.collect()`, `.count()`, `.first()` on a model (e.g. `User.all()`). ## Update diff --git a/docs/4_relationships.md b/docs/4_relationships.md index 3e728bf..a76d9b2 100644 --- a/docs/4_relationships.md +++ b/docs/4_relationships.md @@ -20,25 +20,37 @@ class Post(TypedTable): author: Author -authors_with_roles = Author.join('role').collect() +authors_with_roles = Author.join('roles').collect() posts_with_author = Post.join().collect() # join can be called without arguments to join all relationships (in this case only 'author') +post_deep = Post.join("author.roles").collect() # nested relationship, accessible via post.author.roles ``` In this example, the `Post` table contains a reference to the `Author` table. In that case, the `Author` `id` is stored -in the -`Post`'s `author` column. +in the `Post`'s `author` column. Furthermore, the `Author` table contains a `list:reference` to `list[Role]`. This means multiple `id`s from the `Role` -table -can be stored in the `roles` column of `Author`. +table can be stored in the `roles` column of `Author`. For these two cases, a Relationship is set-up automatically, which means `.join()` can work with those. +### Alternative Join Syntax + +You can pass the relationship object directly instead of its name as a string: + +```python +posts_with_author = Post.join(Post.author).collect() +``` + +This works, but note that `Post.author` is typed as `Relationship[Author]` at the class level, while `row.author` is +typed as `Author` at the instance level. Some editors may complain about type mismatches when using this syntax (e.g., +reporting that `list[Tag]` isn't a `Relationship`). If you encounter type checking issues, use the string syntax +instead. + ## Other Relationships To get the reverse relationship, you'll have to tell TypeDAL how the two tables relate to each other (since guessing is complex and unreliable). -For example, to set up the reverse relationshp from author to posts: +For example, to set up the reverse relationship from author to posts: ```python @db.define() @@ -46,7 +58,6 @@ class Author(TypedTable): name: str posts = relationship(list["Post"], condition=lambda author, post: author.id == post.author, join="left") - ``` Note that `"Post"` is in quotes. This is because the `Post` class is defined later, so a reference to it is not @@ -62,7 +73,7 @@ class Role(TypedTable): authors = relationship(list["Author"], condition=lambda role, author: author.roles.contains(role.id), join="left") ``` -Here, contains is used since `Author.roles` is a `list:reference`. +Here, `contains` is used since `Author.roles` is a `list:reference`. See [the web2py docs](http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#list-type-and-contains) for more details. @@ -82,8 +93,8 @@ class Sidekick(TypedTable): superhero: SuperHero ``` -In this example, `Relationship["Sidekick"]` is added as an extra type hint, since the reference to the table -in `relationship("Sidekick", ...)` is a string. This has to be passed as a string, since the Sidekick class is defined +In this example, `Relationship["Sidekick"]` is added as an extra type hint, since the reference to the table in +`relationship("Sidekick", ...)` is a string. This has to be passed as a string, since the Sidekick class is defined after the superhero class. Adding the `Relationship["Sidekick"]` hint is optional, but recommended to improve editor support. @@ -107,6 +118,7 @@ class Post(TypedTable): tag.on(tag.id == tagged.tag), ]) + # without unique alias: @db.define() @@ -126,7 +138,65 @@ class Tagged(TypedTable): ``` Instead of a `condition`, it is recommended to define an `on`. Using a condition is possible, but could lead to pydal -generation a `CROSS JOIN` instead of a `LEFT JOIN`, which is bad for performance. +generating a `CROSS JOIN` instead of a `LEFT JOIN`, which is bad for performance. In this example, `Tag` is connected to `Post` and vice versa via the `Tagged` table. It is recommended to use the tables received as arguments from the lambda (e.g. `tag.on` instead of `Tag.on` directly), since these use aliases under the hood, which prevents conflicts when joining the same table multiple times. + +## Lazy Loading and Explicit Relationships + +### Lazy Policy + +The `lazy` parameter on a relationship controls what happens when you access relationship data without explicitly +joining it first: + +```python +@db.define() +class User(TypedTable): + name: str + posts = relationship(list["Post"], condition=lambda user, post: user.id == post.author, lazy="forbid") +``` + +Available policies: + +- **`"forbid"`**: Raises an error. Prevents N+1 query problems by making them fail fast. +- **`"warn"`**: Returns an empty value (empty list or `None`) with a console warning. +- **`"ignore"`**: Returns an empty value silently. +- **`"tolerate"`**: Fetches the data but logs a warning about potential performance issues. +- **`"allow"`**: Fetches the data silently. + +If `lazy=None` (the default), the relationship uses the database's default lazy policy. You can set this globally via +`TypeDAL`'s `lazy_policy` option (see [7. Advanced Configuration](./7_configuration.md) for configuration details), +which defaults to `"tolerate"`. + +### Explicit Relationships + +Use `explicit=True` for relationships that are expensive to join or rarely needed: + +```python +@db.define() +class User(TypedTable): + name: str + audit_logs = relationship(list["AuditLog"], condition=lambda user, log: user.id == log.user, explicit=True) +``` + +When you call `.join()` without arguments, explicit relationships are skipped: + +```python +user = User.join().first() # user.audit_logs follows the lazy policy (empty/error/warning depending on setting) +``` + +To include an explicit relationship, reference it by name: + +```python +user = User.join("audit_logs").first() # now user.audit_logs is populated +``` + +## What's Next? + +Depending on your setup: + +- **Using py4web or web2py?** → [5. py4web & web2py](./5_py4web.md) +- **Ready to manage your database?** → [6. Migrations](./6_migrations.md) +- **Dive deeper into functionality?** → [8.: Mixins](./8_mixins.md) + diff --git a/docs/6_migrations.md b/docs/6_migrations.md index d150b25..009b447 100644 --- a/docs/6_migrations.md +++ b/docs/6_migrations.md @@ -2,114 +2,73 @@ By default, pydal manages migrations in your database schema automatically ([See also: the web2py docs](http://www.web2py.com/books/default/chapter/29/06/the-database-abstraction-layer#Migrations)). -This can however be problemantic in a more production environment. -In some cases, you want to disable automatic table changes and manage these by hand. +This can be problematic in a production environment where you want to disable automatic table changes and manage these +by hand. -TypeDAL integrates with [`edwh-migrate`](https://pypi.org/project/edwh-migrate/) to make this easier. -With this tool, you write migrations (`CREATE`, `ALTER`, `DROP` statements) in SQL and it keeps track of which actions -have already been executed on your database. +TypeDAL integrates with [`edwh-migrate`](https://pypi.org/project/edwh-migrate/) to make this easier. With this tool, +you write migrations (`CREATE`, `ALTER`, `DROP` statements) in SQL and it keeps track of which actions have already been +executed on your database. -In order to make this process easier, TypeDAL also integrates -with [`pydal2sql`](https://pypi.org/project/pydal2sql/), which can convert your pydal/TypeDAL table definitions -into `CREATE` statements if it's a new table, or `ALTER` statements if it's an existing table. +To make this process easier, TypeDAL also integrates with [`pydal2sql`](https://pypi.org/project/pydal2sql/), which can +convert your pydal/TypeDAL table definitions into `CREATE` statements for new tables, or `ALTER` statements for existing +tables. ## Installation -To enable the migrations functionality within TypeDAL, you'll need to install it with the specific migrations extra -dependencies. Run the following command: +To enable the migrations functionality within TypeDAL, install it with the migrations extra: ```bash pip install typedal[migrations] # also included in typedal[all] ``` -This extra option is necessary as it adds a few dependencies that aren't essential for the core functionality of -TypeDAL. Enabling the migrations explicitly ensures that you have the additional tools and features available for -managing migrations effectively. +## Minimal Configuration -## Config +To use migrations, you need to configure TypeDAL in your `pyproject.toml`. At minimum, you must set: -TypeDAL's migration behavior and some other features can be customized using a section in your `pyproject.toml`. -An example config can look like this: +- `database`: Your database URI +- `dialect`: The database type (e.g., `sqlite`, `postgres`) +- `migrate`: Set to `false` to disable pydal's automatic migrations +- `flag_location`: Where edwh-migrate stores its migration tracking +- `input`: Path to your table definitions (e.g., `data_model.py`) +- `output`: Where generated migrations are written (e.g., `migrations/`) -```toml -[tool.typedal] -database = "storage.sqlite" -dialect = "sqlite" -folder = "databases" -caching = true -pool_size = 1 -database_to_restore = "data/backup.sql" -migrate_table = "typedal_implemented_features" -flag_location = "databases/flags" -create_flag_location = true -schema = "public" -migrate = false # disable pydal's automatic migration behavior -fake_migrate = false -``` - -To generate such a configuration interactively, use `typedal setup`. If you already have `[tool.pydal2sql]` -and/or `[tool.migrate]` sections, setup will incorporate their settings as defaults. For only essential prompts, add -`--minimal`; sensible defaults will fill in the rest. - -For dynamic properties or secrets (like a database in postgres with credentials), exclude them from the toml and add -them to your .env file (optionally prefixed with `TYPEDAL`_): +Optionally: -```env -TYPEDAL_DATABASE = "psql://user:password@host:5432/database" -``` +- `database_to_restore`: Path to a SQL file to restore before running migrations on a fresh database. -Settings passed directly to `TypeDAL()` will overwrite config values. - -### Multiple Connections - -Thie configuration allows you to define multiple database connections and specify which one `TypeDAL()` will use through environment -variables. +Here's a minimal example: ```toml [tool.typedal] -default = "development" - -[tool.typedal.development] database = "sqlite://" dialect = "sqlite" -migrate = true - -[tool.typedal.production] -# database from .env -dialect = "postgres" migrate = false +flag_location = "migrations/.flags" +input = "path/to/data_model.py" +output = "path/to/migrations.py" ``` +For dynamic properties or secrets (like a database with credentials), +add them to your `.env` file or set them as environment variables (optionally prefixed with `TYPEDAL_`): + ```env -TYPEDAL_CONNECTION="production" -TYPEDAL_DATABASE="psql://..." +TYPEDAL_DATABASE = "psql://user:password@host:5432/database" ``` -In the pyproject.toml file, under` [tool.typedal]`, you can set a default connection key, which here is set to " -development". -This key corresponds to a section named `[tool.typedal.development]` where you define configuration details for the -development environment, such as the database URL (database) and dialect. +> **Full configuration reference**: For all available options, multiple connections, environment overrides, and other +> settings, see [7. Configuration](./7_configuration.md). -Similarly, another section `[tool.typedal.production]` holds configuration details for the production environment. In -this -example, the database parameter is fetched from the `.env` file using the environment variable `TYPEDAL_DATABASE`. The -`dialect` specifies the type of database being used, and `migrate` is set to `false` here, disabling automatic -migrations in the production environment. +You can generate a config interactively with `typedal setup`, or view your current config with `typedal --show-config`. -The .env file contains environment variables like `TYPEDAL_CONNECTION`, which dictates the current active connection -("production" in this case), and `TYPEDAL_DATABASE`, holding the database URI for the production environment. - -This setup allows you to easily switch between different database configurations by changing the `TYPEDAL_CONNECTION` -variable in the `.env` file, enabling you to seamlessly manage different database settings for distinct environments like -development, testing, and production while keeping every (non-secret) config setting documented. +## Generate Migrations (pydal2sql) -To see the currently active configuration settings, you can run `typedal --show-config`. +With your config in place, generate migrations from your table definitions: -## Generate Migrations (pydal2sql) +```bash +typedal migrations.generate +``` -Assuming your configuration is properly set up, `typedal migrations.generate` should execute without additional -arguments. -You can however overwrite the behavior as defined in the config. See the following command for all options: +You can override config values with CLI flags. See all options: ```bash typedal migrations.generate --help @@ -117,8 +76,14 @@ typedal migrations.generate --help ## Run Migrations (edwh-migrate) -With a correctly configured setup, running `typedal migrations.run` should function without extra arguments. -You can however overwrite the behavior as defined in the config. See the following command for all options: +Apply your migrations to the database: + +```bash +typedal migrations.run +``` + +With a correctly configured setup, this should function without extra arguments. +You can however overwrite the behavior as defined in the config. See all options: ```bash typedal migrations.run --help diff --git a/docs/7_configuration.md b/docs/7_configuration.md new file mode 100644 index 0000000..86666fb --- /dev/null +++ b/docs/7_configuration.md @@ -0,0 +1,225 @@ +# 7. Configuration + +TypeDAL configuration is managed through `pyproject.toml`, `.env` file, environment variables, and `TypeDAL()` kwargs. +This page documents all available configuration options. + +## Basic Setup + +Configuration goes under `[tool.typedal]` in your `pyproject.toml`: + +```toml +[tool.typedal] +database = "sqlite://path/to/database.sqlite" +folder = "databases" +caching = true +pool_size = 0 +lazy_policy = "tolerate" +# keys may also be written as pool-size, lazy-policy +``` + +### Core Options + +- **`database`**: Your database URI (required) +- **`folder`**: Directory for database files (default: `"databases"`). Primarily used by SQLite. +- **`caching`**: Enable query caching (default: `true`) +- **`pool_size`**: Connection pool size (default: `1` for sqlite, otherwise `3`). Set to `0` for no pooling. +- **`connection`**: Which connection config to use in multi-connection setups (default: `"default"`). + See "Multiple Connections" below. +- **`lazy_policy`**: Default policy for implicit relationship loading. + Values: `forbid`, `warn`, `ignore`, `tolerate`, `allow` (default: `"tolerate"`). + Can be overridden per relationship. See [4. Relationships](./4_relationships.md) for details. + +## Migrations + +For the minimal configuration required to use migrations, see [6. Migrations](./6_migrations.md). +Options for generating and running migrations: + +```toml +[tool.typedal] +input = "path/to/data_model.py" +output = "path/to/migrations.py" +dialect = "sqlite" +magic = true +function = "define_tables" +flag_location = "migrations/.flags" +migrate_table = "typedal_implemented_features" +create_flag_location = true +schema = "public" +migrate = false +fake_migrate = false +database_to_restore = "data/backup.sql" +tables = ["users", "posts"] +noop = false +``` + +### Generating Migrations (pydal2sql) + +- **`input`**: Path to your TypeDAL table definitions file +- **`output`**: Path to the generated migration `.py` fil +- **`dialect`**: Database type: `sqlite`, `postgres`, `mysql`, etc. (if unclear from database uri) +- **`magic`**: Insert missing variables to prevent crashes (default: `true`). + See [pydal2sql docs](https://github.com/robinvandernoord/pydal2sql#configuration). +- **`function`**: Function name containing your `db.define()` calls (default: `"define_tables"`) +- **`tables`**: Specific tables to generate migrations for (optional; usually set via CLI instead) +- **`noop`**: Don't write to output (usually set via CLI) + +### Running Migrations (edwh-migrate) + +- **`output`**: Path to the migration `.py` file containing your migrations +- **`flag_location`**: Directory where migration flags are stored +- **`create_flag_location`**: Auto-create flag location if missing (default: `true`) +- **`migrate_table`**: Table name tracking executed migrations (default: `"typedal_implemented_features"`) +- **`schema`**: Database schema to use (default: `"public"`, mainly for PostgreSQL) +- **`database_to_restore`**: Optional SQL file to restore before running migrations + +For advanced options like `migrate_cat_command`, `schema_version`, and `redis_host`, see +the [edwh-migrate documentation](https://github.com/educationwarehouse/migrate#documentation). + +### Migration Behavior + +- **`migrate`**: Enable pydal's automatic migration behavior (default: `true`). Set to `false` in production to use + manual migrations. +- **`fake_migrate`**: Mark migrations as executed without running them (default: `false`). Useful for initial setup or + recovery. + +For full details on pydal2sql options, see +the [pydal2sql configuration documentation](https://github.com/robinvandernoord/pydal2sql#configuration). + +## Multiple Connections + +Configure multiple database connections and switch between them: + +```toml +[tool.typedal] +default = "development" + +[tool.typedal.development] +database = "sqlite://" +dialect = "sqlite" +migrate = true + +[tool.typedal.production] +dialect = "postgres" +migrate = false +``` + +```env +TYPEDAL_CONNECTION="production" +TYPEDAL_DATABASE="psql://user:password@host:5432/database" +``` + +- Set `default` to the connection used when `TYPEDAL_CONNECTION` is not set +- Each connection can have its own `[tool.typedal.connection_name]` section +- Environment variables override config values; use `TYPEDAL_CONNECTION` to switch active connections +- Secrets (like database URIs) should go in `.env` prefixed with `TYPEDAL_` + +## Configuration Priority + +TypeDAL loads configuration in this order (highest priority last): + +1. **`pyproject.toml`** — Base configuration +2. **`.env` file** — Environment-specific overrides +3. **Environment variables** — System env vars with `TYPEDAL_` prefix (override `.env`) +4. **`TypeDAL()` kwargs** — Runtime arguments passed to the constructor + +Example with all layers: + +```toml +# pyproject.toml +[tool.typedal] +database = "sqlite://" +pool_size = 5 +``` + +```env +# .env +TYPEDAL_DATABASE="psql://localhost/dev" +TYPEDAL_POOL_SIZE=10 +``` + +```bash +# shell +export TYPEDAL_POOL_SIZE=20 +``` + +```python +# code (highest priority) +db = TypeDAL(database="psql://prod/db") +``` + +The resulting config uses: `database="psql://prod/db"` and `pool_size=20` (from the environment variable). + +## Environment Variables & Variable Interpolation + +Override any config value using environment variables prefixed with `TYPEDAL_`: + +```env +TYPEDAL_DATABASE="psql://..." +TYPEDAL_FOLDER="custom_folder" +TYPEDAL_POOL_SIZE=10 +TYPEDAL_CACHING=false +TYPEDAL_LAZY_POLICY="forbid" +``` + +You can also use variable interpolation in your config to reference environment variables or `.env` values: + +```toml +# pyproject.toml +[tool.typedal] +database = "psql://user:${DB_PASSWORD:defaultpass}@host:5432/database" +``` + +```env +# .env +DB_PASSWORD="secretpassword" +``` + +Use `${VAR:default}` syntax. If `VAR` is not set, `default` is used. + +## TypeDAL() Constructor Options + +Pass configuration directly when instantiating TypeDAL: + +```python +from typedal import TypeDAL + +db = TypeDAL( + "sqlite://...", + # *other pydal configuration + # optional extra typedal settings: + use_pyproject=True, + use_env=True, + connection="production", + lazy_policy="forbid", + enable_typedal_caching=True, +) +``` + +- **`use_pyproject`**: Load config from `pyproject.toml` (default: `True`). Set to a string path to use a custom file. +- **`use_env`**: Load config from `.env` and environment variables (default: `True`). Set to a string path to use a + custom `.env` file, or `False` to disable. +- **`connection`**: Which connection to use in multi-connection setups +- **`lazy_policy`**: Override default lazy loading policy +- **`enable_typedal_caching`**: Override caching setting +- **`config`**: Pass a pre-built `TypeDALConfig` object directly + +## Setup & Inspection + +Generate a config interactively: + +```bash +typedal setup # full setup wizard +typedal setup --minimal # skip non-essential prompts +``` + +View your current active configuration: + +```bash +typedal --show-config +``` + +--- + +You've conquered the boring bits. +Ready for something more interesting? +Head to [8. Mixins](./8_mixins.md) to create powerful, reusable logic with mixins. diff --git a/docs/7_mixins.md b/docs/8_mixins.md similarity index 72% rename from docs/7_mixins.md rename to docs/8_mixins.md index 647cf2a..91e7a4c 100644 --- a/docs/7_mixins.md +++ b/docs/8_mixins.md @@ -1,4 +1,4 @@ -# 7. Mixins +# 8. Mixins Mixins allow you to encapsulate reusable fields and behaviors that can be easily added to your database models. On this page, we'll walk through the usage @@ -40,7 +40,7 @@ from typedal.mixins import SlugMixin # Define your table with SlugMixin, specifying the field to base the slug on -class MyTable(TypedTable, SlugMixin, slug_field="title"): # optionally add `slug_suffix` here. +class MyTable(TypedTable, SlugMixin, slug_field="title"): title: str # Assuming 'title' is a field in your table # Define other fields here @@ -59,21 +59,23 @@ To create your own mixins for additional functionality, follow these steps: Here's a basic example of how to create and use a custom mixin: ```python -from typedal import TypeDAL, TypedTable -from typedal.mixins import Mixin +import datetime as dt +import typing as t + +from typedal import TypeDAL, TypedTable, QueryBuilder +from typedal.mixins import Mixin, TimestampsMixin from typedal.fields import UploadField from py4web import URL from yatl import IMG - class HasImageMixin(Mixin): """ A custom mixin example. """ # Define your mixin fields here - image: UploadField(uploadfolder="/shared_uploads", autodelete=True, notnull=False) + image = UploadField(uploadfolder="/shared_uploads", autodelete=True, notnull=False) def img(self, **options) -> IMG: """ @@ -90,16 +92,32 @@ class HasImageMixin(Mixin): super().__on_define__(db) # Add any custom initialization logic here -# Now you can use CustomMixin in your table definitions along with other mixins or base classes. -class Article(TypedTable, HasImageMixin): +# Now you can use HasImageMixin in your table definitions along with other mixins or base classes. + +class Article(TypedTable, TimestampsMixin, HasImageMixin): title: str -# ... insert article row with image here ... # - -Article(id=1).img() # -> - + # this could also be a class method of Timestamps Mixin: + @classmethod + def recently_updated(cls, hours: int = 24) -> QueryBuilder[t.Self]: + """Return records updated in the last N hours.""" + cutoff = dt.datetime.now() - dt.timedelta(hours=hours) + return QueryBuilder(cls).where(cls.updated_at >= cutoff) + +# Retrieve a record and use the custom method +article = Article(id=1) +article.img() # -> + +# Use the classmethod to get recently updated articles +recent_articles = ( + Article.recently_updated(hours=12) + .where(published=True) + .collect() +) ``` +> **Note:** The `img()` example uses py4web utilities (URL, IMG), but the mixin itself works identically in any setup. + By using these mixins, you can enhance the functionality of your models in a modular and reusable manner, saving you time and effort in your development process. diff --git a/docs/index.md b/docs/index.md index e155d23..c77116e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,9 +1,10 @@ # 0. Table of Contents -1. [Getting Started](1_getting_started.md) -2. [Defining Tables](2_defining_tables.md) -3. [Building Queries](3_building_queries.md) -4. [Relationships](4_relationships.md) +1. [Getting Started](./1_getting_started.md) +2. [Defining Tables](./2_defining_tables.md) +3. [Building Queries](./3_building_queries.md) +4. [Relationships](./4_relationships.md) 5. [py4web & web2py](./5_py4web.md) 6. [Migrations](./6_migrations.md) -7. [Mixins](./7_mixins.md) +7. [Advanced Configuration](./7_configuration.md) +8. [Mixins](./8_mixins.md) diff --git a/mkdocs.yml b/mkdocs.yml index 162717c..432039b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,7 +8,8 @@ nav: - 4. Relationships: 4_relationships.md - 5. py4web: 5_py4web.md - 6. Migrations: 6_migrations.md - - 7. Mixins: 7_mixins.md + - 7. Configuration: 7_configuration.md + - 8. Mixins: 8_mixins.md extra: version: default: stable diff --git a/pyproject.toml b/pyproject.toml index b12fd12..c0755e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -236,3 +236,5 @@ pythonpath = [ ] [tool.typedal] +# e.g. +# lazy-policy = "forbid" diff --git a/src/typedal/caching.py b/src/typedal/caching.py index 4add6e0..83d17da 100644 --- a/src/typedal/caching.py +++ b/src/typedal/caching.py @@ -260,7 +260,7 @@ def _load_from_cache(key: str, db: "TypeDAL") -> t.Any | None: inst.db = db inst.model = db._class_map[inst.model] - inst.model._setup_instance_methods(inst.model) # type: ignore + inst.model._setup_instance_methods(inst.model) return inst diff --git a/src/typedal/cli.py b/src/typedal/cli.py index ae6c7a0..d18204a 100644 --- a/src/typedal/cli.py +++ b/src/typedal/cli.py @@ -392,7 +392,8 @@ def fake_migrations( previously_migrated = ( db( - db.ewh_implemented_features.name.belongs(to_fake) & (db.ewh_implemented_features.installed == True) # noqa E712 + db.ewh_implemented_features.name.belongs(to_fake) + & (db.ewh_implemented_features.installed == True) # noqa E712 ) .select(db.ewh_implemented_features.name) .column("name") diff --git a/src/typedal/config.py b/src/typedal/config.py index 26bf2d8..9939c28 100644 --- a/src/typedal/config.py +++ b/src/typedal/config.py @@ -20,6 +20,8 @@ from edwh_migrate import Config as MigrateConfig from pydal2sql.typer_support import Config as P2SConfig +LazyPolicy = t.Literal["forbid", "warn", "ignore", "tolerate", "allow"] + class TypeDALConfig(TypedConfig): """ @@ -34,6 +36,7 @@ class TypeDALConfig(TypedConfig): pool_size: int = 0 pyproject: str connection: str = "default" + lazy_policy: LazyPolicy = "tolerate" # pydal2sql: input: str = "" diff --git a/src/typedal/core.py b/src/typedal/core.py index 6f67653..b592aff 100644 --- a/src/typedal/core.py +++ b/src/typedal/core.py @@ -12,7 +12,7 @@ import pydal -from .config import TypeDALConfig, load_config +from .config import LazyPolicy, TypeDALConfig, load_config from .helpers import ( SYSTEM_SUPPORTS_TEMPLATES, default_representer, @@ -138,7 +138,7 @@ def resolve_annotation(ftype: str) -> type: # pragma: no cover return resolve_annotation_314(ftype) -class TypeDAL(pydal.DAL): # type: ignore +class TypeDAL(pydal.DAL): """ Drop-in replacement for pyDAL with layer to convert class-based table definitions to classical pydal define_tables. """ @@ -176,6 +176,7 @@ def __init__( use_env: bool | str = True, connection: Optional[str] = None, config: Optional[TypeDALConfig] = None, + lazy_policy: LazyPolicy | None = None, ) -> None: """ Adds some internal tables after calling pydal's default init. @@ -191,6 +192,7 @@ def __init__( fake_migrate=fake_migrate, caching=enable_typedal_caching, pool_size=pool_size, + lazy_policy=lazy_policy, ) self._config = config diff --git a/src/typedal/define.py b/src/typedal/define.py index 42f0f54..8a59bb1 100644 --- a/src/typedal/define.py +++ b/src/typedal/define.py @@ -65,7 +65,7 @@ def define(self, cls: t.Type[T], **kwargs: t.Any) -> t.Type[T]: k: instanciate(v, True) for k, v in annotations.items() if is_typed_field(v) } - relationships: dict[str, type[Relationship[t.Any]]] = filter_out(annotations, Relationship) + relationships: dict[str, type[Relationship[t.Any]]] = filter_out(annotations, Relationship) # type: ignore fields = {fname: self.to_field(fname, ftype) for fname, ftype in annotations.items()} other_kwargs = kwargs | { @@ -77,7 +77,7 @@ def define(self, cls: t.Type[T], **kwargs: t.Any) -> t.Type[T]: setattr(cls, key, clone) typedfields[key] = clone - relationships = filter_out(full_dict, Relationship) | relationships | filter_out(other_kwargs, Relationship) + relationships = filter_out(full_dict, Relationship) | relationships | filter_out(other_kwargs, Relationship) # type: ignore reference_field_keys = [ k for k, v in fields.items() if str(v.type).split(" ")[0] in ("list:reference", "reference") @@ -157,7 +157,7 @@ def annotation_to_pydal_fieldtype( elif origin_is_subclass(ftype, TypedField): # TypedField[int] return self.annotation_to_pydal_fieldtype(t.get_args(ftype)[0], mut_kw) - elif isinstance(ftype, types.GenericAlias) and t.get_origin(ftype) in (list, TypedField): # type: ignore + elif isinstance(ftype, types.GenericAlias) and t.get_origin(ftype) in (list, TypedField): # list[str] -> str -> string -> list:string _child_type = t.get_args(ftype)[0] _child_type = self.annotation_to_pydal_fieldtype(_child_type, mut_kw) diff --git a/src/typedal/for_py4web.py b/src/typedal/for_py4web.py index 2456df1..446b072 100644 --- a/src/typedal/for_py4web.py +++ b/src/typedal/for_py4web.py @@ -14,7 +14,7 @@ from .web2py_py4web_shared import AuthUser -class Fixture(_Fixture): # type: ignore +class Fixture(_Fixture): """ Make mypy happy. """ diff --git a/src/typedal/helpers.py b/src/typedal/helpers.py index 5ba6031..4a9fe9c 100644 --- a/src/typedal/helpers.py +++ b/src/typedal/helpers.py @@ -189,7 +189,7 @@ def looks_like(v: t.Any, _type: type[t.Any]) -> bool: return isinstance(v, _type) or (isinstance(v, type) and issubclass(v, _type)) or origin_is_subclass(v, _type) -def filter_out(mut_dict: dict[K, V], _type: type[T]) -> dict[K, type[T]]: +def filter_out(mut_dict: dict[K, V], _type: type[T]) -> dict[K, T]: """ Split a dictionary into things matching _type and the rest. diff --git a/src/typedal/query_builder.py b/src/typedal/query_builder.py index 3c26893..c21aacf 100644 --- a/src/typedal/query_builder.py +++ b/src/typedal/query_builder.py @@ -14,7 +14,14 @@ from .constants import DEFAULT_JOIN_OPTION, JOIN_OPTIONS from .core import TypeDAL from .fields import TypedField, is_typed_field -from .helpers import DummyQuery, as_lambda, looks_like, normalize_table_keys, throw +from .helpers import ( + DummyQuery, + as_lambda, + filter_out, + looks_like, + normalize_table_keys, + throw, +) from .tables import TypedTable from .types import ( CacheMetadata, @@ -61,7 +68,7 @@ def __init__( """ self.model = model table = model._ensure_table_defined() - default_query = t.cast(Query, table.id > 0) + default_query = table.id > 0 self.query = add_query or default_query self.select_args = select_args or [] self.select_kwargs = select_kwargs or {} @@ -93,7 +100,7 @@ def __bool__(self) -> bool: Querybuilder is truthy if it has t.Any conditions. """ table = self.model._ensure_table_defined() - default_query = t.cast(Query, table.id > 0) + default_query = table.id > 0 return any( [ self.query != default_query, @@ -261,7 +268,7 @@ def _parse_relationships( def join( self, - *fields: str | t.Type[TypedTable], + *fields: str | t.Type[TypedTable] | Relationship[t.Any], method: JOIN_OPTIONS = None, on: OnQuery | list[Expression] | Expression = None, condition: Condition = None, @@ -270,6 +277,15 @@ def join( """ Include relationship fields in the result. + Supports: + - join("example") + - join(Table.example, "second", method="left") + - join(Table.example, on=...) + - join(Table.example, condition=...) + + `fields` can be names or Relationship instances. + If no fields are passed, all relationships will be joined. + `fields` can be names of Relationships on the current model. If no fields are passed, all will be used. @@ -282,21 +298,29 @@ def join( # todo: it would be nice if 'fields' could be an actual relationship # (Article.tags = list[Tag]) and you could change the .condition and .on # this could deprecate condition_and - relationships = self.model.get_relationships() if condition and on: - raise ValueError("condition and on can not be used together!") - elif condition: + raise Relationship._error_duplicate_condition(condition, on) # type: ignore + + relationships: dict[str, Relationship[t.Any]] + + if condition: if len(fields) != 1: raise ValueError("join(field, condition=...) can only be used with exactly one field!") if isinstance(condition, pydal.objects.Query): condition = as_lambda(condition) - to_field = t.cast(t.Type[TypedTable], fields[0]) - relationships = { - str(to_field): Relationship(to_field, condition=condition, join=method, condition_and=condition_and) - } + field = fields[0] + if isinstance(field, Relationship) and field.name: + relationships = { + field.name: field.clone(condition=condition, on=None, join=method, condition_and=condition_and) + } + else: + to_field = t.cast(t.Type[TypedTable], field) + relationships = { + str(to_field): Relationship(to_field, condition=condition, join=method, condition_and=condition_and) + } elif on: if len(fields) != 1: raise ValueError("join(field, on=...) can only be used with exactly one field!") @@ -307,26 +331,50 @@ def join( if isinstance(on, list): on = as_lambda(on) - to_field = t.cast(t.Type[TypedTable], fields[0]) - relationships = {str(to_field): Relationship(to_field, on=on, join=method, condition_and=condition_and)} - - else: - if fields: - # join on every relationship - # simple: 'relationship' - # -> {'relationship': Relationship('relationship')} - # complex with one: relationship.with_nested - # -> {'relationship': Relationship('relationship', nested=[Relationship('with_nested')]) - # complex with two: relationship.with_nested, relationship.no2 - # -> {'relationship': Relationship('relationship', - # nested=[Relationship('with_nested'), Relationship('no2')]) - - relationships = self._parse_relationships(fields, method=method, condition_and=condition_and) - - if method: + field = fields[0] + if isinstance(field, Relationship) and field.name: relationships = { - str(k): r.clone(join=method, condition_and=condition_and) for k, r in relationships.items() + field.name: field.clone(on=on, join=method, condition=None, condition_and=condition_and) } + else: + to_field = t.cast(t.Type[TypedTable], field) + relationships = {str(to_field): Relationship(to_field, on=on, join=method, condition_and=condition_and)} + elif fields: + # join on every relationship + # simple: 'relationship' + # -> {'relationship': Relationship('relationship')} + # complex with one: relationship.with_nested + # -> {'relationship': Relationship('relationship', nested=[Relationship('with_nested')]) + # complex with two: relationship.with_nested, relationship.no2 + # -> {'relationship': Relationship('relationship', + # nested=[Relationship('with_nested'), Relationship('no2')]) + + # fields is a tuple so that's not mutable, filter_out requires a mutable dict: + other_fields = {idx: field for idx, field in enumerate(fields)} + relationship_instances = filter_out(other_fields, Relationship) + + relationships = {} + + # Clone direct Relationship instances (preserving their settings) + for relationship in relationship_instances.values(): + if relationship.name: + relationships[relationship.name] = relationship.clone( + join=method, + condition_and=condition_and, + ) + + # Parse and merge string/table fields + if other_fields: + parsed_relationships = self._parse_relationships( + other_fields.values(), # type: ignore + method=method, + condition_and=condition_and, + ) + # Explicit Relationship instances take precedence + relationships = parsed_relationships | relationships + + else: + relationships = {k: v for k, v in self.model.get_relationships().items() if not v.explicit} return self._extend(relationships=relationships) @@ -465,7 +513,7 @@ def execute(self, add_id: bool = False) -> Rows: Raw version of .collect which only executes the SQL, without performing t.Any magic afterwards. """ db = self._get_db() - metadata = t.cast(Metadata, self.metadata.copy()) + metadata = self.metadata.copy() query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id) @@ -484,7 +532,7 @@ def collect( _to = TypedRows db = self._get_db() - metadata = t.cast(Metadata, self.metadata.copy()) + metadata = self.metadata.copy() if metadata.get("cache", {}).get("enabled") and (result := self._collect_cached(metadata)): return result @@ -842,7 +890,7 @@ def _process_relationship_data( row=row, relation=relation, instance=instance, - parent_id=parent_id, + # parent_id=parent_id, seen_relations=seen_relations, db=db, path=current_path, @@ -864,7 +912,6 @@ def _process_nested_relationships( row: t.Any, relation: Relationship[t.Any], instance: t.Any, - parent_id: t.Any, seen_relations: dict[str, set[str]], db: t.Any, path: str, @@ -876,7 +923,6 @@ def _process_nested_relationships( row: The database row containing relationship data relation: The parent Relationship object containing nested relationships instance: The instance to attach nested data to - parent_id: ID of the root parent for tracking seen_relations: Dict tracking which relationships we've already processed db: Database instance path: Current relationship path diff --git a/src/typedal/relationships.py b/src/typedal/relationships.py index 81d0151..10c5475 100644 --- a/src/typedal/relationships.py +++ b/src/typedal/relationships.py @@ -8,6 +8,7 @@ import pydal.objects +from .config import LazyPolicy from .constants import JOIN_OPTIONS from .core import TypeDAL from .fields import TypedField @@ -17,19 +18,25 @@ To_Type = t.TypeVar("To_Type") +# default lazy policy is defined at the TypeDAL() instance settings level + + class Relationship(t.Generic[To_Type]): """ Define a relationship to another table. """ _type: t.Type[To_Type] - table: t.Type["TypedTable"] | type | str + table: t.Type["TypedTable"] | type | str # use get_table() to resolve later on condition: Condition condition_and: Condition on: OnQuery multiple: bool join: JOIN_OPTIONS + _lazy: LazyPolicy | None nested: dict[str, t.Self] + explicit: bool + name: str | None = None # set by __set_name__ def __init__( self, @@ -39,19 +46,21 @@ def __init__( on: OnQuery = None, condition_and: Condition = None, nested: dict[str, t.Self] = None, + lazy: LazyPolicy | None = None, + explicit: bool = False, ): """ Should not be called directly, use relationship() instead! """ if condition and on: - warnings.warn(f"Relation | Both specified! {condition=} {on=} {_type=}") - raise ValueError("Please specify either a condition or an 'on' statement for this relationship!") + raise self._error_duplicate_condition(condition, on) self._type = _type self.condition = condition self.join = "left" if on else join # .on is always left join! self.on = on self.condition_and = condition_and + self._lazy = lazy if args := t.get_args(_type): self.table = unwrap_type(args[0]) @@ -63,21 +72,34 @@ def __init__( if isinstance(self.table, str): self.table = TypeDAL.to_snake(self.table) + self.explicit = explicit self.nested = nested or {} def clone(self, **update: t.Any) -> "Relationship[To_Type]": """ Create a copy of the relationship, possibly updated. """ + condition = update.get("condition") + on = update.get("on") + + if on and condition: # pragma: no cover + raise self._error_duplicate_condition(condition, on) + return self.__class__( update.get("_type") or self._type, - update.get("condition") or self.condition, + None if on else (condition or self.condition), update.get("join") or self.join, - update.get("on") or self.on, + None if condition else (on or self.on), update.get("condition_and") or self.condition_and, (self.nested | extra) if (extra := update.get("nested")) else self.nested, # type: ignore + update.get("lazy") or self._lazy, ) + @staticmethod + def _error_duplicate_condition(condition: Condition, on: OnQuery) -> t.Never: + warnings.warn(f"Relation | Both specified! {condition=} {on=}") + raise ValueError("Please specify either a condition or an 'on' statement for this relationship!") + def __repr__(self) -> str: """ Representation of the relationship. @@ -93,7 +115,12 @@ def __repr__(self) -> str: src_code = f"to {cls_name} (missing condition)" join = f":{self.join}" if self.join else "" - return f"" + lazy_str = f" lazy={self.lazy}" if self.lazy != "warn" else "" + return f"" + + def __set_name__(self, owner: t.Type["TypedTable"], name: str) -> None: + """Called automatically when assigned to a class attribute.""" + self.name = name def get_table(self, db: "TypeDAL") -> t.Type["TypedTable"]: """ @@ -111,6 +138,36 @@ def get_table(self, db: "TypeDAL") -> t.Type["TypedTable"]: return table + def get_db(self) -> TypeDAL | None: + """ + Retrieves the database instance associated with the table. + + Returns: + TypeDAL | None: The database instance if it exists, or None otherwise. + """ + return getattr(self.table, "_db", None) + + @property + def lazy(self) -> LazyPolicy: + """ + Gets the lazy policy configured in the current context. + + The method first checks for a customized lazy policy for this relationship. + If not found, it attempts to retrieve the lazy policy from the database. + If neither option is available, it returns a conservative fallback value. + + Returns: + LazyPolicy or str: The configured lazy policy or a fallback value. + """ + if customized := self._lazy: + return customized + + if db := self.get_db(): + return db._config.lazy_policy + + # conservative fallback: + return "warn" + def get_table_name(self) -> str: """ Get the name of the table this relationship is bound to. @@ -129,36 +186,105 @@ def get_table_name(self) -> str: return str(table) - def __get__(self, instance: t.Any, owner: t.Any) -> "t.Optional[list[t.Any]] | Relationship[To_Type]": + def __get__( + self, + instance: "TypedTable", + owner: t.Type["TypedTable"], + ) -> "t.Optional[list[t.Any]] | Relationship[To_Type]": """ Relationship is a descriptor class, which can be returned from a class but not an instance. For an instance, using .join() will replace the Relationship with the actual data. - If you forgot to join, a warning will be shown and empty data will be returned. + Behavior when accessed without joining depends on the lazy policy. """ if not instance: # relationship queried on class, that's allowed return self - warnings.warn( - "Trying to get data from a relationship object! Did you forget to join it?", - category=RuntimeWarning, - ) - if self.multiple: - return [] - else: - return None + # instance: TypedTable instance + # owner: TypedTable class + + if not self.name: # pragma: no cover + raise ValueError("Relationship does not seem to be connected to a table field.") + + # Handle different lazy policies + if self.lazy == "forbid": # pragma: no cover + raise AttributeError( + f"Accessing relationship '{self.name}' without joining is forbidden. " + f"Use .join('{self.name}') in your query or set lazy='allow' if this is intentional.", + ) + + fallback_value: t.Optional[list[t.Any]] = [] if self.multiple else None + + if self.lazy == "ignore": # pragma: no cover + # Return empty silently + return fallback_value + + if self.lazy == "warn": + # Warn and return empty + warnings.warn( + f"Trying to access relationship '{self.name}' without joining. " + f"Did you forget to use .join('{self.name}')? Returning empty value.", + category=RuntimeWarning, + ) + return fallback_value + + # For "tolerate" and "allow", we fetch the data + try: + resolved_table = self.get_table(instance._db) + + builder = owner.where(id=instance.id).join(self.name) + if issubclass(resolved_table, TypedTable) or isinstance(resolved_table, pydal.objects.Table): + # is a table so we can select ALL and ignore non-required fields of parent row: + builder = builder.select(owner.id, resolved_table.ALL) + + if self.lazy == "tolerate": + warnings.warn( + f"Lazy loading relationship '{self.name}'. " + "This performs an extra database query. " + f"Consider using .join('{self.name}') for better performance.", + category=RuntimeWarning, + ) + + return builder.first()[self.name] # type: ignore + except Exception as e: # pragma: no cover + warnings.warn( + f"Failed to lazy load relationship '{self.name}': {e}", + category=RuntimeWarning, + source=e, + ) + + return fallback_value def relationship( - _type: t.Type[To_Type], + _type: t.Type[To_Type] | str, condition: Condition = None, join: JOIN_OPTIONS = None, on: OnQuery = None, + lazy: LazyPolicy | None = None, + explicit: bool = False, ) -> To_Type: """ Define a relationship to another table, when its id is not stored in the current table. + Args: + _type: The type of the related table. Use list[Type] for one-to-many relationships. + condition: Lambda function defining the join condition between tables. + Example: lambda self, post: self.id == post.author + join: Join strategy ('left', 'inner', etc.). Defaults to 'left' when using 'on'. + on: Alternative to condition for complex queries with pivot tables. + Allows specifying multiple join conditions to avoid cross joins. + lazy: Controls behavior when accessing relationship data without explicitly joining: + - "forbid": Raise an error (strictest, prevents N+1 queries) + - "warn": Return empty value with warning + - "ignore": Return empty value silently + - "tolerate": Fetch data with warning (convenient but warns about performance) + - "allow": Fetch data silently (most permissive, use only for known cheap queries) + explicit: If True, this relationship is only joined when explicitly requested + (e.g. User.join("tags")). Bare User.join() calls will skip it. + Useful for expensive or rarely-needed relationships. Defaults to False. + Example: class User(TypedTable): name: str @@ -174,7 +300,7 @@ class Post(TypedTable): Here, Post stores the User ID, but `relationship(list["Post"])` still allows you to get the user's posts. In this case, the join strategy is set to LEFT so users without posts are also still selected. - For complex queries with a pivot table, a `on` can be set insteaad of `condition`: + For complex queries with a pivot table, 'on' can be set instead of 'condition': class User(TypedTable): ... @@ -190,7 +316,7 @@ class User(TypedTable): # so for ease of use, just cast to the refered type for now! # e.g. x = relationship(Author) -> x: Author To_Type, - Relationship(_type, condition, join, on), + Relationship(_type, condition, join, on, lazy=lazy, explicit=explicit), # type: ignore ) diff --git a/src/typedal/rows.py b/src/typedal/rows.py index 6590f91..4d21626 100644 --- a/src/typedal/rows.py +++ b/src/typedal/rows.py @@ -491,7 +491,7 @@ def as_dict(self, *_: t.Any, **__: t.Any) -> PaginateDict: # type: ignore return {"data": super().as_dict(), "pagination": self.pagination} -class TypedSet(pydal.objects.Set): # type: ignore # pragma: no cover +class TypedSet(pydal.objects.Set): # pragma: no cover """ Used to make pydal Set more typed. diff --git a/src/typedal/tables.py b/src/typedal/tables.py index 2ad6699..bc9d296 100644 --- a/src/typedal/tables.py +++ b/src/typedal/tables.py @@ -358,7 +358,7 @@ def first_or_fail(self: t.Type[T_MetaInstance]) -> T_MetaInstance: def join( self: t.Type[T_MetaInstance], - *fields: str | t.Type["TypedTable"], + *fields: str | t.Type[TypedTable] | Relationship[t.Any], method: JOIN_OPTIONS = None, on: OnQuery | list[Expression] | Expression = None, condition: Condition = None, @@ -778,6 +778,15 @@ def __getattr__(self, item: str) -> t.Any: raise AttributeError(item) + def __eq__(self, other: t.Any) -> bool: + """ + Compare equal classes via their _row, since the other data is irrelevant for equality. + """ + if type(self) is not type(other): + return False + + return self._row == other._row # type: ignore + def keys(self) -> list[str]: """ Return the combination of row + relationship keys. diff --git a/src/typedal/types.py b/src/typedal/types.py index b9810b9..9d69352 100644 --- a/src/typedal/types.py +++ b/src/typedal/types.py @@ -88,15 +88,15 @@ def open(self, file: str, mode: str = "r") -> t.IO[t.Any]: # --------------------------------------------------------------------------- -class Query(_Query): # type: ignore +class Query(_Query): """Pydal Query object. Makes mypy happy.""" -class Expression(_Expression): # type: ignore +class Expression(_Expression): """Pydal Expression object. Make mypy happy.""" -class Set(_Set): # type: ignore +class Set(_Set): """Pydal Set object. Make mypy happy.""" @@ -117,19 +117,19 @@ def __setitem__(self, key: str, value: t.Any) -> None: else: - class OpRow(_OpRow): # type: ignore + class OpRow(_OpRow): """Runtime OpRow, using pydal's version.""" -class Reference(_Reference): # type: ignore +class Reference(_Reference): """Pydal Reference object. Make mypy happy.""" -class Field(_Field): # type: ignore +class Field(_Field): """Pydal Field object. Make mypy happy.""" -class Rows(_Rows): # type: ignore +class Rows(_Rows): """Pydal Rows object. Make mypy happy.""" def column(self, column: t.Any = None) -> list[t.Any]: @@ -146,11 +146,11 @@ class Row(_Row): """Pydal Row object. Make mypy happy.""" -class Validator(_Validator): # type: ignore +class Validator(_Validator): """Pydal Validator object. Make mypy happy.""" -class Table(_Table, TableProtocol): # type: ignore +class Table(_Table, TableProtocol): """Table with protocol support. Make mypy happy.""" diff --git a/tests/test_relationships.py b/tests/test_relationships.py index 332167c..6627f1f 100644 --- a/tests/test_relationships.py +++ b/tests/test_relationships.py @@ -1,6 +1,9 @@ +import contextlib +import json import time import types import typing +import warnings from uuid import uuid4 import pytest @@ -13,6 +16,7 @@ clear_expired, remove_cache, ) +from src.typedal.serializers import as_json db = TypeDAL("sqlite:memory") @@ -28,8 +32,22 @@ class TaggableMixin: tagged.on(tagged.entity == entity.gid), tag.on((tagged.tag == tag.id)), ], + lazy="warn", # default behavior + ) + + tags_tolerate = relationship( + list["Tag"], + # lambda self, _: (Tagged.entity == self.gid) & (Tagged.tag == Tag.id) + # doing an .on with and & inside can lead to a cross join, + # for relationships with pivot tables a manual on query with aliases is prefered: + on=lambda entity, tag: [ + tagged := Tagged.unique_alias(), + tagged.on(tagged.entity == entity.gid), + tag.on((tagged.tag == tag.id)), + ], + lazy="tolerate", # load but with warning + explicit=True, # only load when requested ) - # tags = relationship(list["Tag"], tagged) @db.define() @@ -84,7 +102,8 @@ class Tagged(TypedTable): # pivot table @db.define() -class Empty(TypedTable): ... +class Empty(TypedTable): + ... def _setup_data(): @@ -184,7 +203,7 @@ def test_typedal_way(): Empty.first_or_fail() # user through article: 1 - many - all_articles = Article.join().collect().as_dict() + all_articles = Article.join("author", "secondary_author", "final_editor", "tags").collect().as_dict() assert all_articles[3]["final_editor"]["name"] == "Editor 1" assert all_articles[4]["secondary_author"]["name"] == "Editor 1" @@ -208,7 +227,9 @@ def test_typedal_way(): with pytest.warns(RuntimeWarning): assert article2.tags == [] - articles1 = Article.where(title="Article 1").join().first_or_fail() + articles1 = ( + Article.where(title="Article 1").join("author", "secondary_author", "final_editor", "tags").first_or_fail() + ) assert articles1.final_editor.name == "Editor 1" @@ -286,14 +307,15 @@ def test_typedal_way(): # from role to users: BelongsToMany via list:reference - role_writer = Role.where(Role.name == "writer").join().first_or_fail() + role_writer = Role.where(Role.name == "writer").join("users", "tags").first_or_fail() assert len(role_writer.users) == 2 - author1 = User.where(id=4).join().first() + author1 = User.where(id=4).join("articles").first() assert ( - len(author1.as_dict()["articles"]) == len(author1.__dict__["articles"]) == len(dict(author1)["articles"]) == 2 + len(author1.as_dict()["articles"]) == len(author1.__dict__["articles"]) == len( + dict(author1)["articles"]) == 2 ) @@ -338,6 +360,35 @@ def test_reprs(): assert "AND" in repr(relation) and "Hank" in repr(relation) +@contextlib.contextmanager +def no_warnings(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + yield + if caught: + raise AssertionError(f"Unexpected warnings: {[w.message for w in caught]}") + + +def test_get_relationship_after_initial(): + _setup_data() + + article1 = Article.where(title="Article 1").first_or_fail() + article2 = Article.where(title="Article 1").join(Article.tags).first_or_fail() + + with pytest.warns(RuntimeWarning): + assert article1.tags == [] + + with pytest.warns(RuntimeWarning): + # tolerate includes warning + tags_tolerate = article1.tags_tolerate + assert tags_tolerate + + with no_warnings(): + # expect no warnings + assert as_json.encode(tags_tolerate) == as_json.encode(article2.tags) + assert tags_tolerate == article2.tags + + @db.define() class CacheFirst(TypedTable): name: str @@ -401,6 +452,8 @@ def test_join_with_different_condition(): def test_caching(): + _setup_data() + uncached = User.join().collect_or_fail() cached = User.cache().join().collect_or_fail() # not actually cached yet! cached_user_only = User.join().cache(User.id).collect_or_fail() # idem @@ -430,12 +483,12 @@ def test_caching(): cached_user_only2 = User.join().cache(User.id).collect_or_fail() assert ( - len(uncached2) - == len(uncached) - == len(cached2) - == len(cached) - == len(cached_user_only2) - == len(cached_user_only) + len(uncached2) + == len(uncached) + == len(cached2) + == len(cached) + == len(cached_user_only2) + == len(cached_user_only) ) assert uncached.as_json() == uncached2.as_json() == cached.as_json() == cached2.as_json() @@ -443,9 +496,9 @@ def test_caching(): assert cached.first().gid == cached2.first().gid assert ( - [_.name for _ in uncached2.first().roles] - == [_.name for _ in cached.first().roles] - == [_.name for _ in cached2.first().roles] + [_.name for _ in uncached2.first().roles] + == [_.name for _ in cached.first().roles] + == [_.name for _ in cached2.first().roles] ) assert not uncached2.metadata.get("cache", {}).get("enabled") @@ -583,10 +636,31 @@ def test_caching_dependencies(): def test_illegal(): with pytest.raises(ValueError), pytest.warns(UserWarning): - class HasRelationship: something = relationship("...", condition=lambda: 1, on=lambda: 2) + with pytest.raises(ValueError), pytest.warns(UserWarning): + Tag.join(Tag.articles, condition=lambda: 1, on=lambda: 2) + +def test_join_relationship_custom_on(): + _setup_data() + + rows1 = Tag.join(Tag.articles, + condition=lambda tag, article: (Tagged.tag == tag.id) & (article.gid == Tagged.entity) & (article.author == 3), + method="inner", + ) + + rows2 = Tag.join(Tag.articles, + on=lambda tag, article: [ + tagged := Tagged.unique_alias(), + (tagged.tag == tag.id) & (article.gid == tagged.entity) & (article.author == 3) + ], + method="inner", + ) + + assert all([row.articles for row in rows1]) + assert all([row.articles for row in rows2]) + def test_join_with_select(): _setup_data() @@ -669,7 +743,9 @@ def test_nested_relationships(): # more complex: role = Role.where(name="reader").join( - "users.bestie", "users.articles.final_editor", "users.articles.secondary_author" + "users.bestie", + "users.articles.final_editor", + "users.articles.secondary_author", ) nested_article = role.first().users[2].articles[0] @@ -681,28 +757,16 @@ def test_nested_relationships(): # complex, inner: role_inner = Role.where(name="reader").join( - "users.bestie", "users.articles.final_editor", "users.articles.secondary_author", method="inner" + "users.bestie", + "users.articles.final_editor", + "users.articles.secondary_author", + method="inner", ) # no final_editor -> inner join should fail: assert not role_inner.first() -""" -In production I noticed this bug: - -When joining nested data like `process = Process.join("pros_and_cons.product")` -process.pros_and_cons[0].product # is fine -process.pros_and_cons[1].product # is None -while they both share the same `product_gid` -This indicates that something goes wrong when collecting the nested data. -`pros_and_cons` is very specific to that production use-case so please think of a more general example and write a test for it. -1. define the required tables and relationships -2. define the required data -3. perform the query and check the results -""" - - class City(TypedTable): gid = TypedField(str, default=uuid4) name: str