From 51fcdfef0b45069a0659fba2bd0d8d0a150a6e44 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Thu, 29 Jan 2026 15:18:52 +0100 Subject: [PATCH 01/15] Don't delete _NoDefault type, rename to _NoDefaultType --- src/validataclass/dataclasses/defaults.py | 8 ++++---- tests/mypy/dataclasses/test_defaults.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/validataclass/dataclasses/defaults.py b/src/validataclass/dataclasses/defaults.py index 0eafa86..645a80b 100644 --- a/src/validataclass/dataclasses/defaults.py +++ b/src/validataclass/dataclasses/defaults.py @@ -20,6 +20,7 @@ 'DefaultFactory', 'DefaultUnset', 'NoDefault', + '_NoDefaultType', ] # Helper objects for setting default values for validator fields @@ -189,7 +190,7 @@ def needs_factory(self) -> bool: # Temporary class to create the NoDefault sentinel, class will be deleted afterwards -class _NoDefault(BaseDefault[Never]): +class _NoDefaultType(BaseDefault[Never]): """ Class for creating the sentinel object `NoDefault` which specifies that a field has no default value, i.e. the field is required. @@ -227,6 +228,5 @@ def __call__(self) -> Self: # Create sentinel object NoDefault, redefine __new__ to always return the same instance, and delete temporary class -NoDefault = _NoDefault() -_NoDefault.__new__ = lambda cls: NoDefault # type: ignore[assignment, method-assign, return-value] -del _NoDefault +NoDefault = _NoDefaultType() +_NoDefaultType.__new__ = lambda cls: NoDefault # type: ignore[assignment, method-assign, return-value] diff --git a/tests/mypy/dataclasses/test_defaults.yml b/tests/mypy/dataclasses/test_defaults.yml index f2259e6..5fd29f1 100644 --- a/tests/mypy/dataclasses/test_defaults.yml +++ b/tests/mypy/dataclasses/test_defaults.yml @@ -76,6 +76,6 @@ var1: BaseDefault[Never] = NoDefault # correct var2: BaseDefault[str] = NoDefault # error (9) out: | - main:3: note: Revealed type is "validataclass.dataclasses.defaults._NoDefault" + main:3: note: Revealed type is "validataclass.dataclasses.defaults._NoDefaultType" main:5: note: Revealed type is "Never" - main:9: error: Incompatible types in assignment (expression has type "_NoDefault", variable has type "BaseDefault[str]") [assignment] + main:9: error: Incompatible types in assignment (expression has type "_NoDefaultType", variable has type "BaseDefault[str]") [assignment] From 6a23032e8ef66544224cef293fcaa6314e72eb0b Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Thu, 29 Jan 2026 15:21:25 +0100 Subject: [PATCH 02/15] validataclass_field: Change 'default' to kw-only argument This is a minor breaking change, but it's unlikely to affect a lot of code. --- src/validataclass/dataclasses/validataclass_field.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/validataclass/dataclasses/validataclass_field.py b/src/validataclass/dataclasses/validataclass_field.py index 34d6f52..59fe150 100644 --- a/src/validataclass/dataclasses/validataclass_field.py +++ b/src/validataclass/dataclasses/validataclass_field.py @@ -17,8 +17,8 @@ def validataclass_field( validator: Validator[Any], - default: Any = NoDefault, *, + default: Any = NoDefault, metadata: dict[str, Any] | None = None, **kwargs: Any, ) -> Any: From d8383323a2951226b86e4e92cfd122dd938e1e61 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Thu, 29 Jan 2026 18:11:21 +0100 Subject: [PATCH 03/15] validataclass_field: Add proper typing with overloads --- .../dataclasses/validataclass_field.py | 59 +++++++++++++++++-- .../dataclasses/validataclass_field_test.py | 46 ++++++++++----- 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/src/validataclass/dataclasses/validataclass_field.py b/src/validataclass/dataclasses/validataclass_field.py index 59fe150..ffb8089 100644 --- a/src/validataclass/dataclasses/validataclass_field.py +++ b/src/validataclass/dataclasses/validataclass_field.py @@ -7,21 +7,71 @@ import dataclasses from typing import Any +from typing_extensions import TypeVar, overload + from validataclass.validators import Validator -from .defaults import BaseDefault, Default, NoDefault +from .defaults import BaseDefault, Default, NoDefault, _NoDefaultType __all__ = [ 'validataclass_field', ] +T_Validated = TypeVar('T_Validated') +T_Default = TypeVar('T_Default') +T_RawDefault = TypeVar('T_RawDefault') + + +@overload +def validataclass_field( + validator: Validator[T_Validated], + *, + default: _NoDefaultType = NoDefault, + metadata: dict[str, Any] | None = None, + **kwargs: Any, +) -> dataclasses.Field[T_Validated]: + ... + + +@overload +def validataclass_field( + validator: Validator[T_Validated], + *, + default: dataclasses._MISSING_TYPE, + metadata: dict[str, Any] | None = None, + **kwargs: Any, +) -> dataclasses.Field[T_Validated]: + ... + + +@overload +def validataclass_field( + validator: Validator[T_Validated], + *, + default: BaseDefault[T_Default], + metadata: dict[str, Any] | None = None, + **kwargs: Any, +) -> dataclasses.Field[T_Validated | T_Default]: + ... + + +@overload +def validataclass_field( + validator: Validator[T_Validated], + *, + default: T_RawDefault, + metadata: dict[str, Any] | None = None, + **kwargs: Any, +) -> dataclasses.Field[T_Validated | T_RawDefault]: + ... + def validataclass_field( - validator: Validator[Any], + validator: Validator[T_Validated], *, default: Any = NoDefault, metadata: dict[str, Any] | None = None, **kwargs: Any, -) -> Any: +) -> dataclasses.Field[Any]: """ Defines a dataclass field compatible with DataclassValidator. @@ -75,4 +125,5 @@ def validataclass_field( kwargs['default'] = default.get_value() # Create a dataclass field with our metadata - return dataclasses.field(metadata=metadata, **kwargs) + # NOTE: typeshed types the field() function as 'T' rather than 'Field[T]', but the actual return type is a Field. + return dataclasses.field(metadata=metadata, **kwargs) # type: ignore[no-any-return] diff --git a/tests/unit/dataclasses/validataclass_field_test.py b/tests/unit/dataclasses/validataclass_field_test.py index dae1f23..9a85f04 100644 --- a/tests/unit/dataclasses/validataclass_field_test.py +++ b/tests/unit/dataclasses/validataclass_field_test.py @@ -67,10 +67,13 @@ def test_validataclass_field_with_default(param_default, expected_default): field = validataclass_field(IntegerValidator(), default=param_default) # Check field metadata - assert type(field.metadata.get('validator')) is IntegerValidator - assert isinstance(field.metadata.get('validator_default'), Default) - assert field.metadata.get('validator_default').get_value() == expected_default - assert field.metadata.get('validator_default').needs_factory() is False + metadata_validator = field.metadata.get('validator') + metadata_default = field.metadata.get('validator_default') + + assert type(metadata_validator) is IntegerValidator + assert type(metadata_default) is Default + assert metadata_default.get_value() == expected_default + assert metadata_default.needs_factory() is False # Check field default and default_factory assert field.default == expected_default @@ -93,14 +96,17 @@ def test_validataclass_field_with_default_factory(param_default, expected_defaul field = validataclass_field(IntegerValidator(), default=param_default) # Check field metadata - assert type(field.metadata.get('validator')) is IntegerValidator - assert isinstance(field.metadata.get('validator_default'), BaseDefault) - assert isinstance(field.metadata.get('validator_default'), expected_default_cls) - assert field.metadata.get('validator_default').get_value() == expected_default - assert field.metadata.get('validator_default').needs_factory() is True + metadata_validator = field.metadata.get('validator') + metadata_default = field.metadata.get('validator_default') + + assert type(metadata_validator) is IntegerValidator + assert type(metadata_default) is expected_default_cls + assert metadata_default.get_value() == expected_default + assert metadata_default.needs_factory() is True # Check field default and default_factory assert field.default is dataclasses.MISSING + assert field.default_factory is not dataclasses.MISSING assert field.default_factory() == expected_default @staticmethod @@ -124,14 +130,18 @@ def needs_factory(self) -> bool: field = validataclass_field(IntegerValidator(), default=CustomDefault()) # Check field metadata - assert type(field.metadata.get('validator')) is IntegerValidator - assert isinstance(field.metadata.get('validator_default'), CustomDefault) - assert field.metadata.get('validator_default').get_value() == 1 - assert field.metadata.get('validator_default').get_value() == 2 - assert field.metadata.get('validator_default').needs_factory() is True + metadata_validator = field.metadata.get('validator') + metadata_default = field.metadata.get('validator_default') + + assert type(metadata_validator) is IntegerValidator + assert type(metadata_default) is CustomDefault + assert metadata_default.get_value() == 1 + assert metadata_default.get_value() == 2 + assert metadata_default.needs_factory() is True # Check field default and default_factory assert field.default is dataclasses.MISSING + assert field.default_factory is not dataclasses.MISSING assert field.default_factory() == 3 assert field.default_factory() == 4 @@ -150,8 +160,12 @@ def test_validataclass_field_with_metadata(): ) # Check field metadata - assert type(field.metadata.get('validator')) is IntegerValidator - assert field.metadata.get('validator_default').get_value() == 42 + metadata_validator = field.metadata.get('validator') + metadata_default = field.metadata.get('validator_default') + + assert type(metadata_validator) is IntegerValidator + assert type(metadata_default) is Default + assert metadata_default.get_value() == 42 assert field.metadata.get('unittest') == 123 @staticmethod From 57de939c2e40eb267ebaff691e547b5603a145ac Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Thu, 29 Jan 2026 19:22:29 +0100 Subject: [PATCH 04/15] mypy plugin: Add base implementation for simple dataclasses --- pyproject.toml | 9 + run_mypy.py | 22 ++ src/validataclass/mypy/__init__.py | 5 + src/validataclass/mypy/plugin/__init__.py | 7 + src/validataclass/mypy/plugin/constants.py | 39 +++ src/validataclass/mypy/plugin/helpers.py | 38 +++ src/validataclass/mypy/plugin/plugin.py | 141 ++++++++++ .../mypy/plugin/validataclass_transformer.py | 216 +++++++++++++++ .../mypy/plugin/virtual_field_resolver.py | 258 ++++++++++++++++++ tests/mypy/pytest_mypy.ini | 4 + tests/unit/dataclasses/validataclass_test.py | 2 +- tox.ini | 7 +- 12 files changed, 745 insertions(+), 3 deletions(-) create mode 100644 run_mypy.py create mode 100644 src/validataclass/mypy/__init__.py create mode 100644 src/validataclass/mypy/plugin/__init__.py create mode 100644 src/validataclass/mypy/plugin/constants.py create mode 100644 src/validataclass/mypy/plugin/helpers.py create mode 100644 src/validataclass/mypy/plugin/plugin.py create mode 100644 src/validataclass/mypy/plugin/validataclass_transformer.py create mode 100644 src/validataclass/mypy/plugin/virtual_field_resolver.py diff --git a/pyproject.toml b/pyproject.toml index 65b3491..07794ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,15 @@ files = ["src/", "tests/"] mypy_path = "src/" explicit_package_bases = true +# Enable mypy plugins +plugins = [ + # Our custom mypy plugin for type checking validataclasses + "validataclass.mypy.plugin", + + # Plugin to type check mypy plugins (only for plugin development) + "mypy.plugins.proper_plugin", +] + # Enable strict type checking strict = true diff --git a/run_mypy.py b/run_mypy.py new file mode 100644 index 0000000..e131788 --- /dev/null +++ b/run_mypy.py @@ -0,0 +1,22 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +# Helper script that wraps running mypy for easier debugging in PyCharm. + +import sys +from mypy import api + +result = api.run(['--show-traceback', '--no-incremental'] + sys.argv[1:]) + +if result[0]: + print('\nType checking report:\n') + print(result[0]) # stdout + +if result[1]: + print('\nError report:\n') + print(result[1]) # stderr + +print('\nExit status:', result[2]) diff --git a/src/validataclass/mypy/__init__.py b/src/validataclass/mypy/__init__.py new file mode 100644 index 0000000..36e1829 --- /dev/null +++ b/src/validataclass/mypy/__init__.py @@ -0,0 +1,5 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" diff --git a/src/validataclass/mypy/plugin/__init__.py b/src/validataclass/mypy/plugin/__init__.py new file mode 100644 index 0000000..b0530a8 --- /dev/null +++ b/src/validataclass/mypy/plugin/__init__.py @@ -0,0 +1,7 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +from .plugin import plugin diff --git a/src/validataclass/mypy/plugin/constants.py b/src/validataclass/mypy/plugin/constants.py new file mode 100644 index 0000000..0296114 --- /dev/null +++ b/src/validataclass/mypy/plugin/constants.py @@ -0,0 +1,39 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +from typing import Final + +from mypy.errorcodes import ErrorCode + +# Decorators that turn a class to a validataclass +# (Not Final so it can be modified by other plugins that add their own validataclass-style decorators.) +VALIDATACLASS_DECORATORS = { + 'validataclass.dataclasses.validataclass.validataclass', +} + +# Full and short name of the validataclass_field() +VALIDATACLASS_FIELD_FUNC: Final = 'validataclass.dataclasses.validataclass_field.validataclass_field' +VALIDATACLASS_FIELD_FUNC_NAME: Final = 'validataclass_field' + +# Full name of base class for validators +VALIDATOR_BASE_CLASS: Final = 'validataclass.validators.validator.Validator' + +# Full name of base class for default objects +FIELD_DEFAULT_BASE_CLASS: Final = 'validataclass.dataclasses.defaults.BaseDefault' + +# Full name of the type of the special NoDefault object +FIELD_NO_DEFAULT_CLASS: Final = 'validataclass.dataclasses.defaults._NoDefaultType' + +# Full and short name of the virtual field wrapper function +# NOTE: This function does not actually exist in the code, it only exists for mypy. +VIRTUAL_FIELD_WRAPPER_FUNC: Final = 'validataclass.mypy._virtual_field_wrapper' +VIRTUAL_FIELD_WRAPPER_FUNC_NAME: Final = '_virtual_field_wrapper' + +# Custom error codes for validataclass +# TODO: Maybe add multiple error codes (as sub codes of the validataclass code). +ERROR_CODE_VALIDATACLASS: Final = ErrorCode( + 'validataclass', 'Check that validataclasses are defined correctly', 'Plugin' +) diff --git a/src/validataclass/mypy/plugin/helpers.py b/src/validataclass/mypy/plugin/helpers.py new file mode 100644 index 0000000..b489e19 --- /dev/null +++ b/src/validataclass/mypy/plugin/helpers.py @@ -0,0 +1,38 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +from typing import Any + + +class DebugLogger: + """ + Logger class for plugin development and debugging. + """ + + debug_mode: bool + + def __init__(self, debug_mode: bool): + self.debug_mode = debug_mode + + def log(self, level: str, context: str | None, msg: str, *objects: Any) -> None: + """ + Log a message on a given level, optionally with a context (e.g. file/line) and with object dumps. + Debug level messages are only logged if debug mode is enabled. + """ + level = level.upper() + if level == 'DEBUG' and not self.debug_mode: + return + + if context is not None: + msg = f'{context}: {msg}' + + obj_dumps = [f'{type(obj)} ({obj!s})' for obj in objects] + if len(obj_dumps) == 1: + msg = f'{msg}: {obj_dumps[0]})' + elif len(obj_dumps) > 1: + msg = '\n- '.join([f'{msg}:', *obj_dumps]) + + print(f'[validataclass.mypy] [{level}] {msg}') diff --git a/src/validataclass/mypy/plugin/plugin.py b/src/validataclass/mypy/plugin/plugin.py new file mode 100644 index 0000000..e4d3561 --- /dev/null +++ b/src/validataclass/mypy/plugin/plugin.py @@ -0,0 +1,141 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +import os +from typing import Callable + +from mypy.options import Options +from mypy.plugin import ClassDefContext, FunctionContext, Plugin +from mypy.plugins.dataclasses import dataclass_tag_callback +from mypy.types import Type +from typing_extensions import override + +from .constants import VALIDATACLASS_DECORATORS, VIRTUAL_FIELD_WRAPPER_FUNC +from .helpers import DebugLogger +from .validataclass_transformer import ValidataclassTransformer +from .virtual_field_resolver import VirtualFieldResolver + + +def _is_debug_mode() -> bool: + return bool(os.getenv('VALIDATACLASS_MYPY_DEBUG', False)) + + +class ValidataclassPlugin(Plugin): + """ + Custom mypy plugin for validataclass support. + """ + + _logger: DebugLogger + + def __init__(self, options: Options): + super().__init__(options) + + self._logger = DebugLogger( + debug_mode=_is_debug_mode(), + ) + + @override + def get_class_decorator_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: + """ + Update class definition of classes decorated with the given decorator (here, the `@validataclass` decorator). + + (For details, see docs for `mypy.plugin.Plugin`.) + + In this plugin, we use this hook to tag all validataclasses so that we can recognize them later. + We also call the corresponding callback of the dataclass plugin because it wouldn't be called otherwise. + """ + # Tag all classes decorated with `@validataclass` for later. + if fullname in VALIDATACLASS_DECORATORS: + return self._validataclass_decorator_tag_callback + + return None + + @override + def get_class_decorator_hook_2(self, fullname: str) -> Callable[[ClassDefContext], bool] | None: + """ + Update class definition of classes decorated with the given decorator (here, the `@validataclass` decorator). + + Similar to `get_class_decorator_hook`, but this runs in a later pass when placeholders have been resolved. + The hook can return False if some base class hasn't been processed yet, in which case the hook will be called + another time later. + + (For details, see docs for `mypy.plugin.Plugin`.) + + In this plugin, we transform the definition of validataclasses by wrapping each field definition (e.g. tuples of + validator and default) in a virtual wrapper function which is analyzed later in `get_function_hook`. + """ + # Update classes decorated with `@validataclass` or similar. + if fullname in VALIDATACLASS_DECORATORS: + return self._validataclass_decorator_transform_callback + + return None + + @override + def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: + """ + Adjust the return type of a function call. + + (For details, see docs for `mypy.plugin.Plugin`.) + + In this plugin, we use this hook to analyze validataclass field definitions that have been wrapped in a virtual + wrapper function during `get_class_decorator_hook_2`. We check the field definition for validators and default + objects and adjust the return type to be the combined type of validator output and default value. + + For example, given a validataclass with the following field definition: + + example: str | None = StringValidator(), Default(None) + + The right-hand side of this assignment would be wrapped in a virtual function call ("virtual" meaning it only + exists for the type checker) by the class decorator hook, changing the field to something similar to: + + example: str | None = _virtual_field_wrapper((StringValidator(), Default(None))) + + In the function hook, we can now analyze the wrapped expression (in this case a tuple) and find the validator + and default object. Then we evaluate their types (e.g. the StringValidator returns `str`, the default has a + value of `None`) and adjust the return type of the virtual wrapper function to the actual type that this field + can have according to its validator and default. In the example, the return type is changed to `str | None`. + + Now if the return type doesn't match the field annotation on the left-hand side, mypy can report a proper error. + """ + # Adjust return type of all calls to the virtual field wrapper function. + if fullname == VIRTUAL_FIELD_WRAPPER_FUNC: + return self._virtual_field_wrapper_callback + + return None + + @staticmethod + def _validataclass_decorator_tag_callback(ctx: ClassDefContext) -> None: + """ + Callback for the class decorator hook for classes decorated with `@validataclass`. + + Tag all `@validataclass`-decorated classes both as a validataclass and a regular dataclass. + """ + # TODO: Tag class with "validataclass_tag" (like the dataclass plugin) + dataclass_tag_callback(ctx) + + def _validataclass_decorator_transform_callback(self, ctx: ClassDefContext) -> bool: + """ + Callback for the second-pass class decorator hook for classes decorated with `@validataclass`. + + Transform all `@validataclass`-decorated classes for later analysis. + """ + transformer = ValidataclassTransformer(ctx, self._logger) + return transformer.transform() + + def _virtual_field_wrapper_callback(self, ctx: FunctionContext) -> Type: + """ + Callback for the function hook for the virtual wrapper function. + + Analyze type of field definition and adjust function return type. + """ + resolver = VirtualFieldResolver(ctx, self._logger) + return resolver.resolve() + + +# This function is expected by mypy to load the plugin +def plugin(_version: str) -> type[ValidataclassPlugin]: + # Ignore version argument if the plugin works with all mypy versions + return ValidataclassPlugin diff --git a/src/validataclass/mypy/plugin/validataclass_transformer.py b/src/validataclass/mypy/plugin/validataclass_transformer.py new file mode 100644 index 0000000..fb833cb --- /dev/null +++ b/src/validataclass/mypy/plugin/validataclass_transformer.py @@ -0,0 +1,216 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +from typing import Any + +from mypy.errorcodes import ErrorCode +from mypy.nodes import ( + Argument, + ArgKind, + AssignmentStmt, + CallExpr, + ClassDef, + Context, + Expression, + FuncDef, + GDEF, + NameExpr, + TempNode, + Var, +) +from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface +from mypy.plugins.dataclasses import dataclass_class_maker_callback, DATACLASS_FIELD_SPECIFIERS +from mypy.types import AnyType, CallableType, TypeOfAny, UnboundType + +from .constants import ( + ERROR_CODE_VALIDATACLASS, + VALIDATACLASS_FIELD_FUNC, + VIRTUAL_FIELD_WRAPPER_FUNC, + VIRTUAL_FIELD_WRAPPER_FUNC_NAME, +) +from .helpers import DebugLogger + + +class ValidataclassTransformer: + """ + Handler for class decorator hook to transform a `@validataclass`-decorated class for type checking. + + Ensures that the default dataclass plugin callbacks are called as well if necessary. + """ + + _ctx: ClassDefContext + + # Class definition that is being transformed + _class_def: ClassDef + + # The decorator expression that's applied to the class + _decorator: Expression + + # Interface to mypy's semantic analyzer + _api: SemanticAnalyzerPluginInterface + + # Logger for plugin development and debugging + _logger: DebugLogger + + def __init__(self, ctx: ClassDefContext, logger: DebugLogger): + self._ctx = ctx + self._class_def = ctx.cls + self._decorator = ctx.reason + self._api = ctx.api + self._logger = logger + + def _get_logger_context(self, context: Context | None) -> str: + """ + Returns a string representation of the given context, i.e. the full class name and line number. + """ + if context is None: + context = self._decorator + return f'{self._class_def.fullname}:{context.line}' + + def _log_warn(self, context: Context | None, msg: str, *objects: Any) -> None: + """ + Logs a message on WARN level. + Use this to warn about unhandled expressions/types or features that are not implemented yet. + """ + return self._logger.log('WARN', self._get_logger_context(context), msg, *objects) + + def _log_debug(self, context: Context | None, msg: str, *objects: Any) -> None: + """ + Logs a message on DEBUG level. Only printed if debug mode is enabled. + Use this for better traceability and debugging of what the plugin is doing. + """ + return self._logger.log('DEBUG', self._get_logger_context(context), msg, *objects) + + def _fail(self, msg: str, context: Context, *, code: ErrorCode | None = None) -> None: + """ + Reports a mypy error to the user. Default code is "validataclass". + """ + self._api.fail(msg, context, code=code or ERROR_CODE_VALIDATACLASS) + + def transform(self) -> bool: + """ + Update the class definition of a validataclass by wrapping the right-hand side of field definitions in the + virtual field wrapper function, so that the types can be analyzed later in the function hook. + + Additionally take care of running the hooks of the mypy dataclass plugin, which generates the definitions of + special functions like `__init__`, and modify these definitions if necessary. + + Called for every class that is decorated with `@validataclass` (or an equivalent decorator). + """ + self._log_debug(None, 'Analyzing decorated validataclass') + + # TODO: Parse base classes in reverse MRO and collect fields defined in these base classes. + # TODO: Make sure all base classes have been processed already. + self._log_debug(None, 'Base classes', self._class_def.info.bases) + bases = [str(base) for base in self._class_def.info.bases] + if bases != ['builtins.object']: + self._log_warn(None, 'validataclass with base classes, not implemented yet', bases) + + # Iterate over all statements of the class body + for stmt in self._class_def.defs.body: + # There are a lot of different types of statements that we could theoretically encounter here. Most of them + # are either irrelevant for us here (e.g. method definitions) or because they don't make sense within a + # validataclass definition (e.g. loops, conditionals, throw/return statements, ...). + # For now we are only interested in assignment statements like `field: type = SomeValidator()`. + if not isinstance(stmt, AssignmentStmt): + self._log_debug(stmt, f'Skipping statement of type {type(stmt)}') + continue + + # There are more edge cases for the left-hand side that we can skip for now, like multiple assignments + # (`x, y = z`) or chained assignments (`x = y = z`). + # TODO: Check if there are edge cases that we should handle, or that we should even report an error for! + lvalue = stmt.lvalues[0] if len(stmt.lvalues) == 1 else None + if lvalue is None or not isinstance(lvalue, NameExpr): + self._log_warn(stmt, 'Skipping weird edge case for assignment LHS', stmt.lvalues) + continue + + # Edge case for right-hand side: We can have a type annotation statement without an assignment (`x: int`), + # which mypy represents as an AssignmentStmt with a TempNode(AnyType, no_rhs=True) as rvalue + # TODO: This isn't allowed in validataclasses, we should report an error for this + if isinstance(stmt.rvalue, TempNode) and stmt.rvalue.no_rhs: + self._log_warn(stmt, f'Skipping assignment for "{lvalue.name}" without RHS') + continue + + # The validataclass decorator ignores attributes without annotations and those that start with a '_'. + if stmt.type is None: + self._log_debug(stmt, f'Skipping assignment for "{lvalue.name}" without type annotation') + continue + if lvalue.name.startswith('_'): + self._log_debug(stmt, f'Skipping assignment for "{lvalue.name}" because it starts with an underscore') + continue + + # Skip fields that are already wrapped or created with field specifier functions + if isinstance(stmt.rvalue, CallExpr) and isinstance(stmt.rvalue.callee, NameExpr): + callee_name = stmt.rvalue.callee.fullname + + # Ignore fields that have been created with the regular dataclasses.field() or similar + if callee_name in DATACLASS_FIELD_SPECIFIERS: + self._log_debug(stmt, f'Skipping field "{lvalue.name}" with {callee_name}') + continue + + # Skip fields that have been created with validataclass_field() + # TODO: I think we need to wrap these too actually :/ + if callee_name == VALIDATACLASS_FIELD_FUNC: + # TODO: Normalize `default` - if it's NoDefault, remove the default instead? + # TODO: (Can probably be ignored because we'll have to fix __init__ manually anyway.) + self._log_debug(stmt, f'Skipping field "{lvalue.name}" with validataclass_field()') + continue + + # Skip field if it has already been wrapped in a previous pass + if callee_name == VIRTUAL_FIELD_WRAPPER_FUNC: + self._log_debug(stmt, f'Skipping already wrapped field "{lvalue.name}"') + continue + + # If we're still here, we might actually have a validataclass field definition that we care about! + # Wrap it in the virtual field wrapper function. + self._log_debug(stmt, f'Wrapping field "{lvalue.name}" with rvalue', stmt.rvalue) + stmt.rvalue = self._wrap_in_virtual_field_wrapper(stmt.rvalue) + + # Let the default mypy plugin for dataclasses process the validataclass too. + # This is needed to get auto-generated methods like __init__ right. + # TODO: Currently, all fields will be seen as optional by the dataclass plugin, because every assignment has + # a right-hand side. We either need to hook into the dataclass plugin, or modify the generated __init__ + # function, or generate the __init__ function all by ourselves. + return dataclass_class_maker_callback(self._ctx) + + def _wrap_in_virtual_field_wrapper(self, rvalue: Expression) -> CallExpr: + """ + Wrap an expression (the right-hand side of an assignment in a validataclass) in a CallExpr for the virtual + field wrapper function. + """ + # TODO: We probably don't need to recreate the FuncDef everytime. Make it reusable. + virtual_wrapper_funcdef = FuncDef( + name=VIRTUAL_FIELD_WRAPPER_FUNC_NAME, + arguments=[ + Argument(Var('args'), None, None, ArgKind.ARG_POS), + ], + typ=CallableType( + arg_types=[AnyType(TypeOfAny.implementation_artifact)], + arg_kinds=[ArgKind.ARG_POS], + arg_names=['args'], + # TODO: Does the UnboundType make sense as a placeholder? We will always replace the return type anyway, + # but an UnboundType or similar might help us find errors where we the return type wasn't replaced. + # Maybe UninhabitedType (i.e. Never) would work for that too. + ret_type=UnboundType('UNKNOWN_TYPE'), + fallback=self._api.named_type('builtins.function'), + name=VIRTUAL_FIELD_WRAPPER_FUNC_NAME, + ), + ) + + # Generate a call expression for the virtual field wrapper + virtual_wrapper_nameexpr = NameExpr(VIRTUAL_FIELD_WRAPPER_FUNC_NAME) + virtual_wrapper_nameexpr.fullname = VIRTUAL_FIELD_WRAPPER_FUNC + virtual_wrapper_nameexpr.node = virtual_wrapper_funcdef + virtual_wrapper_nameexpr.kind = GDEF + + virtual_call_expr = CallExpr( + callee=virtual_wrapper_nameexpr, + args=[rvalue], + arg_kinds=[ArgKind.ARG_POS], + arg_names=[None], + ) + virtual_call_expr.set_line(rvalue) + return virtual_call_expr diff --git a/src/validataclass/mypy/plugin/virtual_field_resolver.py b/src/validataclass/mypy/plugin/virtual_field_resolver.py new file mode 100644 index 0000000..cbc407e --- /dev/null +++ b/src/validataclass/mypy/plugin/virtual_field_resolver.py @@ -0,0 +1,258 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +from typing import Any + +from mypy.errorcodes import ErrorCode +from mypy.nodes import Context, Expression, MemberExpr, TempNode +from mypy.plugin import CheckerPluginInterface, FunctionContext +from mypy.typeops import make_simplified_union +from mypy.types import ( + AnyType, + CallableType, + Instance, + Overloaded, + ProperType, + TupleType, + Type, + TypeOfAny, + get_proper_type, +) + +from .constants import ERROR_CODE_VALIDATACLASS, FIELD_DEFAULT_BASE_CLASS, VALIDATOR_BASE_CLASS +from .helpers import DebugLogger + + +class ParsedValidataclassField: + """ + Internal representation of the types of a validataclass field. + """ + + # True if there was an error while analyzing the field + error: bool = False + + # If None, the field does not have an explicit validator/default, but could still have one from the base class. + validated_type: ProperType | None = None + default_type: ProperType | None = None + + +class VirtualFieldResolver: + """ + Handler for function hook to analyse the wrapped expression in virtual field wrapper calls, i.e. the right-hand side + expressions of assignments in a validataclass. + """ + + _ctx: FunctionContext + + # Interface to mypy's type checker + _api: CheckerPluginInterface + + # Logger for plugin development and debugging + _logger: DebugLogger + + def __init__(self, ctx: FunctionContext, logger: DebugLogger): + self._ctx = ctx + self._api = ctx.api + self._logger = logger + + def _get_logger_context(self) -> str: + """ + Return a string representation of the current context, i.e. the file path and line number of this call. + """ + return f'{self._api.path}:{self._ctx.context.line}' + + def _log_warn(self, msg: str, *objects: Any) -> None: + """ + Log a message on WARN level. + Use this to warn about unhandled expressions/types or features that are not implemented yet. + """ + return self._logger.log('WARN', self._get_logger_context(), msg, *objects) + + def _log_debug(self, msg: str, *objects: Any) -> None: + """ + Log a message on DEBUG level. Only printed if debug mode is enabled. + Use this for better traceability and debugging of what the plugin is doing. + """ + return self._logger.log('DEBUG', self._get_logger_context(), msg, *objects) + + def _fail(self, msg: str, context: Context | None = None, *, code: ErrorCode | None = None) -> None: + """ + Report a mypy error to the user. Default code is "validataclass". + """ + self._api.fail( + msg, + context or self._ctx.context, + code=code or ERROR_CODE_VALIDATACLASS, + ) + + def resolve(self) -> Type: + """ + Analyze the wrapped expression (right-hand side of a validataclass field assignment) for validator and default + objects. Return the field type by combining the validator result type and the default value type. + """ + + # TODO: These assertions just lead to "internal errors". I guess here that's fine? + assert len(self._ctx.args) == 1 + assert len(self._ctx.args[0]) == 1, 'Function was called without an argument?' + + # Get only argument of virtual function call, this is the entire RHS of the validataclass field assignment + rhs_expr = self._ctx.args[0][0] + + # Parse right-hand side expression based on its type(s) + parsed_field = self._parse_rhs_expression(rhs_expr) + + # Stop here if there was an error parsing the tuple (fallback to returning Any) + if parsed_field is None: + return AnyType(TypeOfAny.from_error) + + # These conditions should always be true, otherwise we have an error in our code + assert parsed_field.error is False + assert parsed_field.validated_type is not None + + # Get the combined type of the field (union of validated type and default type) + combined_type = self._get_combined_type(parsed_field) + self._log_debug(' => Combined type of tuple', combined_type) + return combined_type + + def _parse_rhs_expression(self, rhs_expr: Expression) -> ParsedValidataclassField | None: + """ + Analyze the right-hand side expression of a validataclass field, determining its type based on validator and + default objects found in the type of the expression. + """ + # Get type of field assignment RHS + rhs_type = get_proper_type(self._api.get_expression_type(rhs_expr)) + + # TODO: Handle validataclass_field() calls properly. + + parsed_field = ParsedValidataclassField() + + if isinstance(rhs_type, TupleType): + # We have a tuple on the right-hand side, probably a tuple with validator and default object + self._log_debug('Field RHS is a tuple', rhs_type) + + for index, item_type in enumerate(rhs_type.items): + item_type = get_proper_type(item_type) + self._parse_rhs_item(item_type, parsed_field) + else: + # We have a single Instance on the right-hand side, probably a validator or default object + self._log_debug('Field RHS is a single item', rhs_type) + self._parse_rhs_item(rhs_type, parsed_field) + + # Stop analyzing if we have already found errors in the field definition + if parsed_field.error: + self._log_debug('Field has errors, stop checking any further') + return None + + # Make sure the field has at least a validator instance + if parsed_field.validated_type is None: + # TODO: Consider base classes here too + self._fail('No Validator found in field definition') + return None + + return parsed_field + + def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassField) -> None: + """ + Analyze the type of a single item in the right-hand side of a validataclass field, i.e. an item in a tuple or + a the whole right-hand side if it's not a tuple. + + The result will be written into the given ParsedValidataclassField. May report errors, e.g. for unknown types or + for another validator/default instance if the parsed field already has one. + """ + item_type = get_proper_type(item_type) + + # Usually we're expecting an instance of class here, namely of a validator class or default class + if isinstance(item_type, Instance): + # Check for validator instances (base class Validator) + if item_type.type.has_base(VALIDATOR_BASE_CLASS): + # Make sure only one validator is set + if parsed_field.validated_type is not None: + parsed_field.error = True + self._fail('Multiple validator instances found in field definition') + else: + # Find out the output type of this validator + parsed_field.validated_type = self._get_method_return_type(item_type, 'validate') + self._log_debug(f' - Validator: {item_type.type.name} -> {parsed_field.validated_type}') + return + + # Check for default instances (base class BaseDefault) + if item_type.type.has_base(FIELD_DEFAULT_BASE_CLASS): + # Make sure only one default object is set + if parsed_field.default_type is not None: + parsed_field.error = True + self._fail('Multiple default objects found in field definition') + else: + # Find out the value type of this default object + parsed_field.default_type = self._get_method_return_type(item_type, 'get_value') + self._log_debug(f' - Default object: {item_type.type.name} -> {parsed_field.default_type}') + return + + # One easy mistake is writing a validator class name without parentheses (e.g. `field: int = IntegerValidator`). + # Let's provide a more helpful error message (the default string representation would be long and confusing). + if isinstance(item_type, CallableType): + callable_ret_type = get_proper_type(item_type.ret_type) + item_type_str = f'Callable[..., {callable_ret_type}]' + + # Check if the callable's return type is a validator instance + if isinstance(callable_ret_type, Instance) and callable_ret_type.type.has_base(VALIDATOR_BASE_CLASS): + parsed_field.error = True + self._fail( + f'Unexpected type "{item_type_str}" in field definition (did you mean ' + f'"{callable_ret_type.type.name}()"?)' + ) + return + else: + item_type_str = str(item_type) + + # Everything else is probably an error + parsed_field.error = True + self._fail(f'Unexpected type "{item_type_str}" in field definition (expected Validator or BaseDefault)') + + def _get_method_return_type(self, instance_type: Instance, method_name: str) -> ProperType | None: + """ + Construct a MemberExpr for the given instance type and method name, resolve expression type and return the + return type of the method. Report an error if it's not a valid callable. + + In other words, returns the return type of `instance.method_name()`. + """ + # We don't need to construct an actual CallExpr here, the MemberExpr is enough. Evaluating a CallExpr type can + # have side effects, like mypy falsely reporting "code unreachable" if we have a RejectValidator or NoDefault. + constructed_memberexpr = MemberExpr(TempNode(instance_type), method_name) + constructed_method_type = get_proper_type( + self._api.get_expression_type(constructed_memberexpr) + ) + + # Special case: It's an overloaded method. Combine all possible return types in a union. + if isinstance(constructed_method_type, Overloaded): + return make_simplified_union( + [item.ret_type for item in constructed_method_type.items], + line=self._ctx.context.line, + column=self._ctx.context.column, + ) + + # Make sure it's a callable. If it isn't, either we forgot some edge case, or the user is doing something *very* + # weird (namely overriding validate or get_value to be something that's not callable, which mypy should notice + # anyway). We should report an error here, but can return None as if the validator/default didn't exist. + if not isinstance(constructed_method_type, CallableType): + self._fail(f'"{instance_type.type.name}.{method_name}" is not a callable') + return None + + return get_proper_type(constructed_method_type.ret_type) + + def _get_combined_type(self, parsed_field: ParsedValidataclassField) -> ProperType: + """ + Returns the combined type of a field, i.e. a union of validated type and default type. + """ + assert parsed_field.validated_type is not None, 'No validator. This should have been caught earlier.' + + if parsed_field.default_type is None: + return parsed_field.validated_type + else: + return make_simplified_union( + [parsed_field.validated_type, parsed_field.default_type], + line=self._ctx.context.line, + column=self._ctx.context.column, + ) diff --git a/tests/mypy/pytest_mypy.ini b/tests/mypy/pytest_mypy.ini index 3cece81..f24241f 100644 --- a/tests/mypy/pytest_mypy.ini +++ b/tests/mypy/pytest_mypy.ini @@ -18,3 +18,7 @@ enable_error_code = truthy-bool, truthy-iterable, unused-awaitable + +# Enable validataclass mypy plugin +plugins = + validataclass.mypy.plugin diff --git a/tests/unit/dataclasses/validataclass_test.py b/tests/unit/dataclasses/validataclass_test.py index a06b601..3938954 100644 --- a/tests/unit/dataclasses/validataclass_test.py +++ b/tests/unit/dataclasses/validataclass_test.py @@ -478,7 +478,7 @@ def test_validataclass_with_invalid_field_tuples(field_tuple, expected_exception with pytest.raises(DataclassValidatorFieldException) as exception_info: @validataclass class InvalidDataclass: - foo: int = field_tuple + foo: int = field_tuple # type: ignore[validataclass] assert str(exception_info.value) == expected_exception_msg diff --git a/tox.ini b/tox.ini index 8db5212..358838a 100644 --- a/tox.ini +++ b/tox.ini @@ -18,14 +18,17 @@ extras = testing commands = python -m pytest --cov --cov-append {posargs} [testenv:pytest-mypy] -extras = testing commands = python -m pytest {posargs:tests/mypy} [testenv:flake8] commands = flake8 src/ tests/ [testenv:mypy,py{310,311,312,313,314}-mypy] -commands = mypy +commands = mypy {posargs} + +[testenv:mypy-debug] +# Use no-incremental to disable mypy caching when developing the mypy plugin +commands = mypy --show-traceback --no-incremental {posargs} [testenv:clean] commands = coverage erase From cb3495d28082659b749d32e6e8efaf05d12e9f23 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Wed, 4 Feb 2026 20:36:53 +0100 Subject: [PATCH 05/15] mypy plugin: Add tests; handle more edge cases --- pytest.ini | 3 + .../mypy/plugin/validataclass_transformer.py | 16 +- tests/mypy/dataclasses/test_validataclass.yml | 225 ++++++++++++++++++ tests/unit/dataclasses/validataclass_test.py | 2 +- 4 files changed, 238 insertions(+), 8 deletions(-) create mode 100644 tests/mypy/dataclasses/test_validataclass.yml diff --git a/pytest.ini b/pytest.ini index c1835d0..d913327 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,6 +6,9 @@ addopts = --cov-report= --mypy-ini-file=tests/mypy/pytest_mypy.ini --mypy-only-local-stub + # TODO: This is needed to include the mypy plugin in coverage. However the docs say: + # TODO "Useful for debugging, will create problems with import cache" - better solution? + --mypy-same-process testpaths = tests python_files = *_test.py *Test.py diff --git a/src/validataclass/mypy/plugin/validataclass_transformer.py b/src/validataclass/mypy/plugin/validataclass_transformer.py index fb833cb..1c731f0 100644 --- a/src/validataclass/mypy/plugin/validataclass_transformer.py +++ b/src/validataclass/mypy/plugin/validataclass_transformer.py @@ -129,18 +129,20 @@ def transform(self) -> bool: # Edge case for right-hand side: We can have a type annotation statement without an assignment (`x: int`), # which mypy represents as an AssignmentStmt with a TempNode(AnyType, no_rhs=True) as rvalue - # TODO: This isn't allowed in validataclasses, we should report an error for this if isinstance(stmt.rvalue, TempNode) and stmt.rvalue.no_rhs: - self._log_warn(stmt, f'Skipping assignment for "{lvalue.name}" without RHS') + self._fail( + 'Annotated field without assignment (missing Validator or BaseDefault on right-hand side)', + stmt, + ) continue - # The validataclass decorator ignores attributes without annotations and those that start with a '_'. - if stmt.type is None: + # The validataclass decorator ignores attributes without annotations (in most cases) + if stmt.unanalyzed_type is None: + # TODO: Unless the attribute starts with an underscore, the decorator actually checks the type of the + # rhs of an annotation-less field and raises an error if it contains a validator or default (because + # it means you probably forgot the annotation). It's a bit more difficult to check this here though. self._log_debug(stmt, f'Skipping assignment for "{lvalue.name}" without type annotation') continue - if lvalue.name.startswith('_'): - self._log_debug(stmt, f'Skipping assignment for "{lvalue.name}" because it starts with an underscore') - continue # Skip fields that are already wrapped or created with field specifier functions if isinstance(stmt.rvalue, CallExpr) and isinstance(stmt.rvalue.callee, NameExpr): diff --git a/tests/mypy/dataclasses/test_validataclass.yml b/tests/mypy/dataclasses/test_validataclass.yml new file mode 100644 index 0000000..7929302 --- /dev/null +++ b/tests/mypy/dataclasses/test_validataclass.yml @@ -0,0 +1,225 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/typeddjango/pytest-mypy-plugins/master/pytest_mypy_plugins/schema.json + +# Create a trivial validataclass without any fields +- case: empty_validataclass + main: | + from validataclass.dataclasses import validataclass + + @validataclass + class EmptyDataclass: + pass + + obj = EmptyDataclass() + reveal_type(obj) # N: Revealed type is "main.EmptyDataclass" + +# Create a simple validataclass with several different validators but without defaults +# TODO: Missing init parameters aren't reported as errors yet. +- case: simple_validataclass + main: | + from validataclass.dataclasses import validataclass + from validataclass.validators import ListValidator, IntegerValidator, Noneable, StringValidator + + @validataclass + class SimpleDataclass: + """ Example dataclass (docstring should be ignored!). """ + name: str = StringValidator(min_length=1) + noneable: int | None = Noneable(IntegerValidator()) + numbers: list[int] = ListValidator(IntegerValidator(min_value=0)) + + # Valid initializations + valid1 = SimpleDataclass( + name='test', + noneable=None, + numbers=[], + ) + valid2 = SimpleDataclass( + name='test', + noneable=42, + numbers=[1, 2, 3], + ) + reveal_type(valid1) # N: Revealed type is "main.SimpleDataclass" + reveal_type(valid2) # N: Revealed type is "main.SimpleDataclass" + + # Invalid initializations + invalid1 = SimpleDataclass() # TODO: This should be an error! + invalid2 = SimpleDataclass( # E: Unexpected keyword argument "unknown_field" for "SimpleDataclass" [call-arg] + name=42, # E: Argument "name" to "SimpleDataclass" has incompatible type "int"; expected "str" [arg-type] + noneable='foo', # E: Argument "noneable" to "SimpleDataclass" has incompatible type "str"; expected "int | None" [arg-type] + numbers=['foo'], # E: List item 0 has incompatible type "str"; expected "int" [list-item] + unknown_field=42, # (error reported for first line of statement) + ) + + # Positional arguments shouldn't be allowed + invalid3 = SimpleDataclass('', 1, []) # E: Too many positional arguments for "SimpleDataclass" [misc] + +# Create a validataclass with validator-default tuples +# TODO: Missing init parameters aren't reported as errors yet. +- case: validataclass_with_defaults + main: | + from validataclass.dataclasses import validataclass, Default, DefaultFactory, DefaultUnset, NoDefault + from validataclass.helpers import UnsetValue, UnsetValueType + from validataclass.validators import IntegerValidator, StringValidator + + @validataclass + class DataclassWithDefaults: + required_implicit: int = IntegerValidator() + required_explicit: int = IntegerValidator(), NoDefault + + optional_str: str = StringValidator(), Default('') + optional_none: str | None = StringValidator(), Default(None) + optional_unset: str | UnsetValueType = StringValidator(), DefaultUnset + optional_factory: int = IntegerValidator(), DefaultFactory(lambda: 42) + + # Valid initializations + valid1 = DataclassWithDefaults( + required_implicit=1, + required_explicit=2, + ) + valid2 = DataclassWithDefaults( + required_implicit=1, + required_explicit=2, + optional_str='example', + optional_none='example', + optional_unset='example', + optional_factory=42, + ) + valid3 = DataclassWithDefaults( + required_implicit=1, + required_explicit=2, + optional_str='example', + optional_none=None, + optional_unset=UnsetValue, + optional_factory=42, + ) + reveal_type(valid1) # N: Revealed type is "main.DataclassWithDefaults" + reveal_type(valid2) # N: Revealed type is "main.DataclassWithDefaults" + reveal_type(valid3) # N: Revealed type is "main.DataclassWithDefaults" + + # Invalid initializations + invalid1 = DataclassWithDefaults() # TODO: This should be an error! + invalid2 = DataclassWithDefaults( + required_implicit=None, # E: Argument "required_implicit" to "DataclassWithDefaults" has incompatible type "None"; expected "int" [arg-type] + required_explicit=None, # E: Argument "required_explicit" to "DataclassWithDefaults" has incompatible type "None"; expected "int" [arg-type] + optional_str=None, # E: Argument "optional_str" to "DataclassWithDefaults" has incompatible type "None"; expected "str" [arg-type] + optional_none=UnsetValue, # E: Argument "optional_none" to "DataclassWithDefaults" has incompatible type "UnsetValueType"; expected "str | None" [arg-type] + optional_unset=None, # E: Argument "optional_unset" to "DataclassWithDefaults" has incompatible type "None"; expected "str | UnsetValueType" [arg-type] + optional_factory='foo', # E: Argument "optional_factory" to "DataclassWithDefaults" has incompatible type "str"; expected "int" [arg-type] + ) + +# Invalid validataclasses with errors that the mypy plugin should detect +- case: validataclass_invalid + main: | + from validataclass.dataclasses import validataclass, Default, DefaultFactory, DefaultUnset, NoDefault + from validataclass.helpers import UnsetValue, UnsetValueType + from validataclass.validators import IntegerValidator, StringValidator, Validator + + class BadValidator(Validator[None]): + """ Validator class with invalid validate method. """ + validate = None # type: ignore[assignment] + + @validataclass + class BadValidataclass: + # Valid definitions, incorrect typing + wrong_validator: int = StringValidator() # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] + wrong_default: int = IntegerValidator(), Default('') # E: Incompatible types in assignment (expression has type "int | str", variable has type "int") [assignment] + wrong_both: int = StringValidator(), Default(None) # E: Incompatible types in assignment (expression has type "str | None", variable has type "int") [assignment] + + # Annotated field without assignment right-hand side + missing_rhs: int # E: Annotated field without assignment (missing Validator or BaseDefault on right-hand side) [validataclass] + + # Missing validator + missing_validator: int = Default(42) # E: No Validator found in field definition [validataclass] + + # Too many validators/defaults + double_validator: int = IntegerValidator(), Default(42), StringValidator() # E: Multiple validator instances found in field definition [validataclass] + double_default: int = IntegerValidator(), Default(42), Default(None) # E: Multiple default objects found in field definition [validataclass] + + # Missing parentheses (class instead of instance) + missing_parentheses: int = IntegerValidator # E: Unexpected type "Callable[..., validataclass.validators.integer_validator.IntegerValidator]" in field definition (did you mean "IntegerValidator()"?) [validataclass] + missing_parentheses_tuple: int = IntegerValidator(), Default # E: Unexpected type "Callable[..., validataclass.dataclasses.defaults.Default[T_Default`1]]" in field definition (expected Validator or BaseDefault) [validataclass] + + # Miscellaneous nonsense + nonsense1: int = 42 # E: Unexpected type "Literal[42]?" in field definition (expected Validator or BaseDefault) [validataclass] + nonsense2: int = [42] # E: Unexpected type "builtins.list[builtins.int]" in field definition (expected Validator or BaseDefault) [validataclass] + nonsense3: int = list # E: Unexpected type "Overload(def [_T] () -> builtins.list[_T`1], def [_T] (typing.Iterable[_T`1]) -> builtins.list[_T`1])" in field definition (expected Validator or BaseDefault) [validataclass] + nonsense4: int = lambda: 42 # E: Unexpected type "Callable[..., Literal[42]?]" in field definition (expected Validator or BaseDefault) [validataclass] + + # Bad validator (.validate is not a function) + bad_validator: None = BadValidator() # (multiple errors, see below) + out: | + main:37: error: "BadValidator.validate" is not a callable [validataclass] + main:37: error: No Validator found in field definition [validataclass] + +# Validataclass with various edge cases (that are all allowed / should be ignored) +- case: validataclass_valid_edge_cases + main: | + from typing_extensions import Any, overload, override + + from validataclass.dataclasses import validataclass, Default + from validataclass.exceptions import InvalidTypeError + from validataclass.validators import IntegerValidator, StringValidator, Validator + + class OverloadedValidator(Validator[int | str]): + @overload + def validate(self, input_data: int, **kwargs: Any) -> int: ... + + @overload + def validate(self, input_data: str, **kwargs: Any) -> str: ... + + @override + def validate(self, input_data: Any, **kwargs: Any) -> int | str: + if isinstance(input_data, (int, str)): + return input_data + raise InvalidTypeError(expected_types=[int, str]) + + # Predefined objects to use in the validataclass + predefined_validator = StringValidator() + predefined_default = Default(42) + predefined_tuple = StringValidator(), Default(42) + + def validator_generator() -> Validator[int]: + return IntegerValidator() + + def validator_tuple_generator() -> tuple[Validator[int], Default[None]]: + return IntegerValidator(), Default(None) + + @validataclass + class OddValidataclass: + """ + Docstrings are statements in the class that should be ignored. + """ + + # Annotation-less fields with a validator are allowed if they start with an underscore + _reusable_validator = IntegerValidator() + + # Annotation-less fields with values that are not validators or defaults are allowed, even without underscore + some_constant = 42 + + # Predefined validators, defaults or tuples + field1: str = predefined_validator + field2: str | int = predefined_validator, predefined_default + field3: str | int = predefined_tuple + field4: int = validator_generator() + field5: int | None = validator_tuple_generator() + + # Validator with @overload-ed validate() method + overloaded_validator: int | str = OverloadedValidator() + + # Inner classes should be ignored + class InnerClass: pass + + # Methods should be ignored as well + def __post_init__(self) -> None: + pass + def __post_validate__(self, **kwargs: Any) -> None: + pass + def get_some_value(self) -> str: + return self.field1 + + @classmethod + def some_class_method(cls) -> None: + pass + + @staticmethod + def some_static_method() -> None: + pass diff --git a/tests/unit/dataclasses/validataclass_test.py b/tests/unit/dataclasses/validataclass_test.py index 3938954..f2e8d15 100644 --- a/tests/unit/dataclasses/validataclass_test.py +++ b/tests/unit/dataclasses/validataclass_test.py @@ -428,7 +428,7 @@ def test_validataclass_with_invalid_values(): with pytest.raises(DataclassValidatorFieldException) as exception_info: @validataclass class InvalidDataclass: - foo: int + foo: int # type: ignore[validataclass] assert str(exception_info.value) == 'Dataclass field "foo" must specify a validator.' From 33f10cff8c70cfc727723cc313bd24adc32abe17 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Thu, 26 Feb 2026 18:27:43 +0100 Subject: [PATCH 06/15] mypy plugin: Implement support for subclasses --- pyproject.toml | 5 - pytest.ini | 1 + run_mypy.py | 10 +- .../mypy/plugin/parsed_field_cache.py | 69 ++++ src/validataclass/mypy/plugin/plugin.py | 49 ++- .../mypy/plugin/validataclass_transformer.py | 289 +++++++++++--- .../mypy/plugin/virtual_field_resolver.py | 191 ++++++--- .../test_validataclass_subclasses.yml | 373 ++++++++++++++++++ tests/unit/dataclasses/validataclass_test.py | 30 +- .../unit/validators/integer_validator_test.py | 4 +- 10 files changed, 878 insertions(+), 143 deletions(-) create mode 100644 src/validataclass/mypy/plugin/parsed_field_cache.py create mode 100644 tests/mypy/dataclasses/test_validataclass_subclasses.yml diff --git a/pyproject.toml b/pyproject.toml index 07794ee..e50cb8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,3 @@ module = 'tests.*' # Don't enforce typed definitions in tests, this is a lot of unnecessary work (most parameters would be Any anyway). allow_untyped_defs = true - -# TODO: This is the main issue with mypy and validataclass right now. -# Defining dataclasses with validators using the @validataclass decorator, like `some_field: str = StringValidator()`, -# will cause "Incompatible types in assignment" errors. Until we find a way to solve this, ignore this error for now. -disable_error_code = "assignment" diff --git a/pytest.ini b/pytest.ini index d913327..6d5b134 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,7 @@ addopts = -ra --import-mode=importlib + --cov-config=.coveragerc --cov-context=test --cov-report= --mypy-ini-file=tests/mypy/pytest_mypy.ini diff --git a/run_mypy.py b/run_mypy.py index e131788..61cec5a 100644 --- a/run_mypy.py +++ b/run_mypy.py @@ -9,7 +9,15 @@ import sys from mypy import api -result = api.run(['--show-traceback', '--no-incremental'] + sys.argv[1:]) +# Set this to True to disable the mypy cache (or pass --no-incremental via CLI args) +disable_mypy_cache = False + +# Run mypy +mypy_args = ['--show-traceback'] +if disable_mypy_cache: + mypy_args.append('--no-incremental') + +result = api.run(mypy_args + sys.argv[1:]) if result[0]: print('\nType checking report:\n') diff --git a/src/validataclass/mypy/plugin/parsed_field_cache.py b/src/validataclass/mypy/plugin/parsed_field_cache.py new file mode 100644 index 0000000..d035920 --- /dev/null +++ b/src/validataclass/mypy/plugin/parsed_field_cache.py @@ -0,0 +1,69 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +from mypy.types import ProperType +from typing_extensions import Self + + +class ParsedValidataclassField: + """ + Internal representation of the types of a parsed validataclass field. + """ + + # True if there was an error while analyzing the field + error: bool = False + + # If None, the field does not have an explicit validator/default, but could still have one from the base class. + validated_type: ProperType | None = None + default_type: ProperType | None = None + + def merge(self, other: Self) -> None: + """ + Merge another instance of this class into this one. + Validated type and default type are replaced if the attributes in the other instance are not None. + The error flag is set to True if it's true for any instance. + """ + self.error = self.error or other.error + + if other.validated_type is not None: + self.validated_type = other.validated_type + if other.default_type is not None: + self.default_type = other.default_type + + +class ParsedFieldCache: + """ + Cache for ParsedValidataclassFields, i.e. the parsed validator and default types of a validataclass field. + + This cache is globally shared for the entire plugin, meaning that all VirtualFieldResolver instances can access it. + It's used to store the result of the field resolver, so that later instances can access the parsed types of a field + in a base class. + + Important: This cache is NOT related to the mypy cache. It is also NOT persisted across multiple mypy runs. + (Although it is possible that we might persist the cache in a future release to speed up continuous runs of mypy.) + """ + + # Nested dictionary, outer key is the fully qualified name of a validataclass, inner key is the name of a field, + # values are objects representing parsed validataclass fields. + _fields: dict[str, dict[str, ParsedValidataclassField]] + + def __init__(self) -> None: + self._fields = {} + + def get_field(self, class_name: str, field_name: str) -> ParsedValidataclassField: + """ + Retrieve a parsed validataclass field for a given field in a given class from the cache. + + The class and field must exist in the cache, otherwise a KeyError is raised. + """ + return self._fields[class_name][field_name] + + def set_field(self, class_name: str, field_name: str, parsed_field: ParsedValidataclassField) -> None: + """ + Store a parsed validataclass field in the cache. + """ + class_fields = self._fields.setdefault(class_name, {}) + class_fields[field_name] = parsed_field diff --git a/src/validataclass/mypy/plugin/plugin.py b/src/validataclass/mypy/plugin/plugin.py index e4d3561..48e02e2 100644 --- a/src/validataclass/mypy/plugin/plugin.py +++ b/src/validataclass/mypy/plugin/plugin.py @@ -5,16 +5,18 @@ """ import os -from typing import Callable +from datetime import datetime +from typing import Any, Callable from mypy.options import Options -from mypy.plugin import ClassDefContext, FunctionContext, Plugin +from mypy.plugin import ClassDefContext, FunctionContext, Plugin, ReportConfigContext from mypy.plugins.dataclasses import dataclass_tag_callback from mypy.types import Type from typing_extensions import override from .constants import VALIDATACLASS_DECORATORS, VIRTUAL_FIELD_WRAPPER_FUNC from .helpers import DebugLogger +from .parsed_field_cache import ParsedFieldCache from .validataclass_transformer import ValidataclassTransformer from .virtual_field_resolver import VirtualFieldResolver @@ -28,14 +30,48 @@ class ValidataclassPlugin(Plugin): Custom mypy plugin for validataclass support. """ + # Logger class for easier debugging _logger: DebugLogger + # Internal cache for parsed validataclass fields, shared between all instances of VirtualFieldResolver + _parsed_field_cache: ParsedFieldCache + def __init__(self, options: Options): super().__init__(options) self._logger = DebugLogger( debug_mode=_is_debug_mode(), ) + self._parsed_field_cache = ParsedFieldCache() + + @override + def report_config_data(self, ctx: ReportConfigContext) -> Any: + """ + Get representation of configuration data for a module. + + This hook is called once or twice for every module (i.e. Python file): Once after loading the metadata cache to + check if the cache for this module needs to be invalidated, and if yes, another time at the end to write new + cache information. + + It's intended for custom plugin configuration, so that mypy rechecks cached files if the plugin config has been + changed. + + We misuse this hook a little bit here to work around the limitations of mypy's caching system. With the current + approach, we need to parse every validataclass in every run and cannot rely on the mypy cache. This means we + need to invalidate the cache for every module that defines a validataclass. Since all these classes depend on + the `@validataclass` decorator, it's enough to invalidate the cache for the module that defines this decorator, + all dependent files will automatically be rechecked. To force cache invalidation, we can just return the current + timestamp here. + + TODO: This is a workaround that partially disables caching (which is still better than disabling caching + completely). We should find a better solution that works without cache invalidation, but for now this is fine. + """ + # Always invalidate cache for the module that defines the validataclass decorator to trigger rechecking of all + # files that define validataclasses. + if ctx.id == 'validataclass.dataclasses.validataclass': + return str(datetime.now()) + + return None @override def get_class_decorator_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: @@ -91,7 +127,7 @@ def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] The right-hand side of this assignment would be wrapped in a virtual function call ("virtual" meaning it only exists for the type checker) by the class decorator hook, changing the field to something similar to: - example: str | None = _virtual_field_wrapper((StringValidator(), Default(None))) + example: str | None = _virtual_field_wrapper((StringValidator(), Default(None)), [additional metadata]) In the function hook, we can now analyze the wrapped expression (in this case a tuple) and find the validator and default object. Then we evaluate their types (e.g. the StringValidator returns `str`, the default has a @@ -113,7 +149,10 @@ def _validataclass_decorator_tag_callback(ctx: ClassDefContext) -> None: Tag all `@validataclass`-decorated classes both as a validataclass and a regular dataclass. """ - # TODO: Tag class with "validataclass_tag" (like the dataclass plugin) + # Tag class so we can recognize it as a validataclass later (the value of the tag is ignored) + ctx.cls.info.metadata["validataclass_tag"] = {} + + # Default dataclass plugin: Tag class as a dataclass dataclass_tag_callback(ctx) def _validataclass_decorator_transform_callback(self, ctx: ClassDefContext) -> bool: @@ -131,7 +170,7 @@ def _virtual_field_wrapper_callback(self, ctx: FunctionContext) -> Type: Analyze type of field definition and adjust function return type. """ - resolver = VirtualFieldResolver(ctx, self._logger) + resolver = VirtualFieldResolver(ctx, self._logger, self._parsed_field_cache) return resolver.resolve() diff --git a/src/validataclass/mypy/plugin/validataclass_transformer.py b/src/validataclass/mypy/plugin/validataclass_transformer.py index 1c731f0..31d1d48 100644 --- a/src/validataclass/mypy/plugin/validataclass_transformer.py +++ b/src/validataclass/mypy/plugin/validataclass_transformer.py @@ -4,26 +4,31 @@ Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. """ -from typing import Any +from typing import Any, Iterator from mypy.errorcodes import ErrorCode from mypy.nodes import ( Argument, ArgKind, AssignmentStmt, + Block, CallExpr, ClassDef, Context, Expression, FuncDef, GDEF, + ListExpr, NameExpr, + StrExpr, TempNode, Var, ) from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface from mypy.plugins.dataclasses import dataclass_class_maker_callback, DATACLASS_FIELD_SPECIFIERS +from mypy.server.trigger import make_wildcard_trigger from mypy.types import AnyType, CallableType, TypeOfAny, UnboundType +from typing_extensions import override from .constants import ( ERROR_CODE_VALIDATACLASS, @@ -34,6 +39,71 @@ from .helpers import DebugLogger +class ValidataclassField: + """ + Internal representation of a validataclass field in the semantic analysis pass, storing information and references + to expressions for analysis. + """ + + # The name of the field + name: str + + # Fully qualified names of base validataclasses in which this field has been defined or overridden, in reverse MRO + # (oldest class first). Does not include the current class, nor base classes that are not validataclasses. + # Empty for fields that have been created in the current class and don't exist in any base class. + base_classes: list[str] + + # Assignment statement for this field from the class body. + # None if the field is not defined explicitly in the current class (i.e. it has only been defined in base classes). + assignment_stmt: AssignmentStmt | None + + def __init__(self, name: str): + self.name = name + self.base_classes = [] + self.assignment_stmt = None + + @override + def __repr__(self) -> str: + return f'ValidataclassField({self.name})' + + @property + def lvalue(self) -> NameExpr: + """ + NameExpr of the field name, i.e. the left-hand side of the assignment statement. + """ + assert self.assignment_stmt is not None + lvalue = self.assignment_stmt.lvalues[0] + assert isinstance(lvalue, NameExpr) + return lvalue + + @property + def lvalue_var(self) -> Var: + """ + Variable symbol that the lvalue NameExpr references. + """ + # We can assume that the node is always a Var. + lvalue_node = self.lvalue.node + assert lvalue_node is not None and isinstance(lvalue_node, Var) + return lvalue_node + + @property + def rvalue(self) -> Expression: + """ + Right-hand side expression of the assignment statement. + """ + assert self.assignment_stmt is not None + return self.assignment_stmt.rvalue + + def serialize(self) -> dict[str, str]: + """ + Serialize this object to save it in a ClassDef's metadata (potentially cached by mypy). + Currently only contains the field name. + """ + return { + "name": self.name, + } + + class ValidataclassTransformer: """ Handler for class decorator hook to transform a `@validataclass`-decorated class for type checking. @@ -55,6 +125,9 @@ class ValidataclassTransformer: # Logger for plugin development and debugging _logger: DebugLogger + # FuncDef for the virtual field wrapper function. Created on first use and cached here as a class variable. + _virtual_field_wrapper_funcdef: FuncDef | None = None + def __init__(self, ctx: ClassDefContext, logger: DebugLogger): self._ctx = ctx self._class_def = ctx.cls @@ -100,23 +173,86 @@ def transform(self) -> bool: Called for every class that is decorated with `@validataclass` (or an equivalent decorator). """ - self._log_debug(None, 'Analyzing decorated validataclass') - - # TODO: Parse base classes in reverse MRO and collect fields defined in these base classes. - # TODO: Make sure all base classes have been processed already. - self._log_debug(None, 'Base classes', self._class_def.info.bases) - bases = [str(base) for base in self._class_def.info.bases] - if bases != ['builtins.object']: - self._log_warn(None, 'validataclass with base classes, not implemented yet', bases) - - # Iterate over all statements of the class body - for stmt in self._class_def.defs.body: - # There are a lot of different types of statements that we could theoretically encounter here. Most of them - # are either irrelevant for us here (e.g. method definitions) or because they don't make sense within a - # validataclass definition (e.g. loops, conditionals, throw/return statements, ...). - # For now we are only interested in assignment statements like `field: type = SomeValidator()`. - if not isinstance(stmt, AssignmentStmt): + # Collect all validataclass fields in this class, including fields defined in base classes + current_fields = self._collect_fields() + if current_fields is None: + return False + + # Iterate over all fields and modify the assignment statements + for field in current_fields: + # Skip fields that haven't been defined in this class (i.e. only in base classes) + if field.assignment_stmt is None: + continue + + # Update the class definition for this field (wrap assignment rvalue and add virtual attribute) + self._transform_field_in_class(field) + + # Store information about all fields (for now, only their names) in the class metadata for subclasses + self._class_def.info.metadata['validataclass'] = { + 'fields': [field.serialize() for field in current_fields if field.assignment_stmt is not None], + } + + # Let the default mypy plugin for dataclasses process the validataclass too. + # This is needed to get auto-generated methods like __init__ right. + # TODO: Currently, all fields will be seen as optional by the dataclass plugin, because every assignment has + # a right-hand side. We either need to hook into the dataclass plugin, or modify the generated __init__ + # function, or generate the __init__ function all by ourselves. + return dataclass_class_maker_callback(self._ctx) + + def _get_assignment_statements_from_block(self, block: Block) -> Iterator[AssignmentStmt]: + """ + Iterate over all assignment statements of a block. + """ + for stmt in block.body: + if isinstance(stmt, AssignmentStmt): + yield stmt + else: self._log_debug(stmt, f'Skipping statement of type {type(stmt)}') + + def _collect_fields(self) -> list[ValidataclassField] | None: + """ + Collect all fields defined in the validataclass and its base classes. + """ + current_fields: dict[str, ValidataclassField] = {} + + # First, collect all fields defined in subclasses (in reverse MRO, oldest class first) + for base_typeinfo in reversed(self._class_def.info.mro[1:-1]): + base_fullname = base_typeinfo.fullname + + # Ignore base classes that are not validataclasses + if 'validataclass_tag' not in base_typeinfo.metadata: + self._log_debug(None, f'Skipping base class "{base_fullname}" (no validataclass tag)') + continue + + # Ensure base class has already been processed by this plugin, otherwise we need another pass + if 'validataclass' not in base_typeinfo.metadata: + self._log_debug(None, f'Base class "{base_fullname}" not processsed yet, need another pass') + return None + + # Gather all fields previously collected by this very function when the base class was analyzed + for field_data in base_typeinfo.metadata['validataclass'].get('fields', []): + field_name: str = field_data['name'] + + if field_name not in current_fields: + current_fields[field_name] = ValidataclassField(field_name) + + # Construct list of base classes that define this field + current_fields[field_name].base_classes.append(base_fullname) + + # Set dependency so that changing the base class will trigger reprocessing of this class + self._api.add_plugin_dependency(make_wildcard_trigger(base_fullname)) + + # Second, collect fields that have been defined in this class (update existing fields if necessary) by + # iterating over all assignment statements in the class body + for stmt in self._get_assignment_statements_from_block(self._class_def.defs): + # TODO: Handle InitVars? + + # The validataclass decorator ignores attributes without annotations (in most cases) + if not stmt.new_syntax: + # TODO: Unless the attribute starts with an underscore, the decorator actually checks the type of the + # RHS of an annotation-less field and raises an error if it contains a validator or default (because + # it means you probably forgot the annotation). It's a bit more difficult to check this here though. + self._log_debug(stmt, f'Skipping assignment for "{stmt.lvalues}" without type annotation') continue # There are more edge cases for the left-hand side that we can skip for now, like multiple assignments @@ -136,20 +272,13 @@ def transform(self) -> bool: ) continue - # The validataclass decorator ignores attributes without annotations (in most cases) - if stmt.unanalyzed_type is None: - # TODO: Unless the attribute starts with an underscore, the decorator actually checks the type of the - # rhs of an annotation-less field and raises an error if it contains a validator or default (because - # it means you probably forgot the annotation). It's a bit more difficult to check this here though. - self._log_debug(stmt, f'Skipping assignment for "{lvalue.name}" without type annotation') - continue - - # Skip fields that are already wrapped or created with field specifier functions + # Handle fields that are already wrapped or created with field specifier functions if isinstance(stmt.rvalue, CallExpr) and isinstance(stmt.rvalue.callee, NameExpr): callee_name = stmt.rvalue.callee.fullname # Ignore fields that have been created with the regular dataclasses.field() or similar if callee_name in DATACLASS_FIELD_SPECIFIERS: + # TODO: Handle these properly. self._log_debug(stmt, f'Skipping field "{lvalue.name}" with {callee_name}') continue @@ -167,52 +296,92 @@ def transform(self) -> bool: continue # If we're still here, we might actually have a validataclass field definition that we care about! - # Wrap it in the virtual field wrapper function. - self._log_debug(stmt, f'Wrapping field "{lvalue.name}" with rvalue', stmt.rvalue) - stmt.rvalue = self._wrap_in_virtual_field_wrapper(stmt.rvalue) + if lvalue.name not in current_fields: + current_fields[lvalue.name] = ValidataclassField(lvalue.name) + current_fields[lvalue.name].assignment_stmt = stmt - # Let the default mypy plugin for dataclasses process the validataclass too. - # This is needed to get auto-generated methods like __init__ right. - # TODO: Currently, all fields will be seen as optional by the dataclass plugin, because every assignment has - # a right-hand side. We either need to hook into the dataclass plugin, or modify the generated __init__ - # function, or generate the __init__ function all by ourselves. - return dataclass_class_maker_callback(self._ctx) + return list(current_fields.values()) + + def _transform_field_in_class(self, field: ValidataclassField) -> None: + """ + Update the class definition for a single validataclass field for later analysis. - def _wrap_in_virtual_field_wrapper(self, rvalue: Expression) -> CallExpr: + This does primarily one thing: Wrap the right-hand side of the field's assignment statement in the virtual field + wrapper function call (modifying the assignment in place). Additional arguments (class nane, field name, list + of base classes) are passed to the wrapper. """ - Wrap an expression (the right-hand side of an assignment in a validataclass) in a CallExpr for the virtual - field wrapper function. + assert field.assignment_stmt is not None + + # Allow incompatible overrides of fields in validataclasses + # TODO: This is necessary because historically we've allowed to override the type of a field in an incompatible + # way in a subclass. This actually isn't very type-safe, though. We probably should discourage this and + # provide an option to allow incompatible overrides for compatibility. (Maybe a strict mode for this plugin?) + field.lvalue_var.allow_incompatible_override = True + + # Wrap the right-hand side of the assignment in a call to the virtual field wrapper function, changing the + # assignment statement in the class body in-place. + self._log_debug(field.assignment_stmt, f'Wrapping field "{field.name}" with RHS', field.rvalue) + field.assignment_stmt.rvalue = self._construct_virtual_wrapper_callexpr( + field.rvalue, + field.name, + field.base_classes, + ) + + def _construct_virtual_wrapper_callexpr( + self, + field_rhs: Expression, + field_name: str, + base_classes: list[str], + ) -> CallExpr: """ - # TODO: We probably don't need to recreate the FuncDef everytime. Make it reusable. - virtual_wrapper_funcdef = FuncDef( - name=VIRTUAL_FIELD_WRAPPER_FUNC_NAME, - arguments=[ - Argument(Var('args'), None, None, ArgKind.ARG_POS), - ], - typ=CallableType( - arg_types=[AnyType(TypeOfAny.implementation_artifact)], - arg_kinds=[ArgKind.ARG_POS], - arg_names=['args'], - # TODO: Does the UnboundType make sense as a placeholder? We will always replace the return type anyway, - # but an UnboundType or similar might help us find errors where we the return type wasn't replaced. - # Maybe UninhabitedType (i.e. Never) would work for that too. - ret_type=UnboundType('UNKNOWN_TYPE'), - fallback=self._api.named_type('builtins.function'), + Construct a CallExpr for the virtual field wrapper function with the given expressions as arguments. + + First argument: Right-hand side expression of the field assignment. + Second argument: Fully qualified name of the current class. + Third argument: Name of the field. + Fourth argument: List of the fully qualified names of all base classes that define this field (in reverse MRO). + """ + cls = type(self) + + # Create the virtual wrapper FuncDef only once and cache it as a class variable + if cls._virtual_field_wrapper_funcdef is None: + any_type = AnyType(TypeOfAny.explicit) + cls._virtual_field_wrapper_funcdef = FuncDef( name=VIRTUAL_FIELD_WRAPPER_FUNC_NAME, - ), - ) + arguments=[ + Argument(Var('field_rhs'), any_type, None, ArgKind.ARG_POS), + Argument(Var('class_name'), any_type, None, ArgKind.ARG_POS), + Argument(Var('field_name'), any_type, None, ArgKind.ARG_POS), + Argument(Var('base_classes'), any_type, None, ArgKind.ARG_POS), + ], + typ=CallableType( + name=VIRTUAL_FIELD_WRAPPER_FUNC_NAME, + arg_types=[any_type, any_type, any_type, any_type], + arg_kinds=[ArgKind.ARG_POS, ArgKind.ARG_POS, ArgKind.ARG_POS, ArgKind.ARG_POS], + arg_names=['field_rhs', 'class_name', 'field_name', 'base_classes'], + # Return type will always be overridden by the function hook, so this one doesn't matter too much. + # We use an UnboundType because if it ever slips through, mypy should complain about it. + ret_type=UnboundType('UNKNOWN_TYPE'), + fallback=self._api.named_type('builtins.function'), + ), + ) # Generate a call expression for the virtual field wrapper virtual_wrapper_nameexpr = NameExpr(VIRTUAL_FIELD_WRAPPER_FUNC_NAME) virtual_wrapper_nameexpr.fullname = VIRTUAL_FIELD_WRAPPER_FUNC - virtual_wrapper_nameexpr.node = virtual_wrapper_funcdef + virtual_wrapper_nameexpr.node = cls._virtual_field_wrapper_funcdef virtual_wrapper_nameexpr.kind = GDEF virtual_call_expr = CallExpr( callee=virtual_wrapper_nameexpr, - args=[rvalue], - arg_kinds=[ArgKind.ARG_POS], - arg_names=[None], + args=[ + field_rhs, + StrExpr(self._class_def.fullname), + StrExpr(field_name), + ListExpr([StrExpr(base_class) for base_class in base_classes]), + ], + arg_kinds=[ArgKind.ARG_POS, ArgKind.ARG_POS, ArgKind.ARG_POS, ArgKind.ARG_POS], + arg_names=[None, None, None, None], ) - virtual_call_expr.set_line(rvalue) + virtual_call_expr.set_line(field_rhs) return virtual_call_expr diff --git a/src/validataclass/mypy/plugin/virtual_field_resolver.py b/src/validataclass/mypy/plugin/virtual_field_resolver.py index cbc407e..3883c7d 100644 --- a/src/validataclass/mypy/plugin/virtual_field_resolver.py +++ b/src/validataclass/mypy/plugin/virtual_field_resolver.py @@ -4,10 +4,10 @@ Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. """ -from typing import Any +from typing import Any, cast from mypy.errorcodes import ErrorCode -from mypy.nodes import Context, Expression, MemberExpr, TempNode +from mypy.nodes import Context, Expression, ListExpr, MemberExpr, StrExpr, TempNode from mypy.plugin import CheckerPluginInterface, FunctionContext from mypy.typeops import make_simplified_union from mypy.types import ( @@ -24,19 +24,7 @@ from .constants import ERROR_CODE_VALIDATACLASS, FIELD_DEFAULT_BASE_CLASS, VALIDATOR_BASE_CLASS from .helpers import DebugLogger - - -class ParsedValidataclassField: - """ - Internal representation of the types of a validataclass field. - """ - - # True if there was an error while analyzing the field - error: bool = False - - # If None, the field does not have an explicit validator/default, but could still have one from the base class. - validated_type: ProperType | None = None - default_type: ProperType | None = None +from .parsed_field_cache import ParsedFieldCache, ParsedValidataclassField class VirtualFieldResolver: @@ -53,10 +41,14 @@ class VirtualFieldResolver: # Logger for plugin development and debugging _logger: DebugLogger - def __init__(self, ctx: FunctionContext, logger: DebugLogger): + # Internal cache for parsed validataclass fields (i.e. parsed types), shared across instances of this class + _parsed_field_cache: ParsedFieldCache + + def __init__(self, ctx: FunctionContext, logger: DebugLogger, parsed_field_cache: ParsedFieldCache): self._ctx = ctx self._api = ctx.api self._logger = logger + self._parsed_field_cache = parsed_field_cache def _get_logger_context(self) -> str: """ @@ -90,34 +82,117 @@ def _fail(self, msg: str, context: Context | None = None, *, code: ErrorCode | N def resolve(self) -> Type: """ - Analyze the wrapped expression (right-hand side of a validataclass field assignment) for validator and default - objects. Return the field type by combining the validator result type and the default value type. - """ + Analyze the wrapped expression (right-hand side of a validataclass field assignment) to find validator and + default objects within it and determine the type that this field can have, taking base classes into account. - # TODO: These assertions just lead to "internal errors". I guess here that's fine? - assert len(self._ctx.args) == 1 - assert len(self._ctx.args[0]) == 1, 'Function was called without an argument?' + Return the expected field type by combining the validator result type and the default value type. + """ + # Parse call arguments of the virtual field wrapper call + field_rhs_expr, class_name, field_name, base_classes = self._get_call_args() - # Get only argument of virtual function call, this is the entire RHS of the validataclass field assignment - rhs_expr = self._ctx.args[0][0] + # Parse entire field definition (parse RHS expression and merge with parsed fields from base classes) + parsed_field = self._parse_field_definition(field_name, field_rhs_expr, base_classes) - # Parse right-hand side expression based on its type(s) - parsed_field = self._parse_rhs_expression(rhs_expr) + # Store parsed types in internal cache (not persisted between mypy runs!) + self._parsed_field_cache.set_field(class_name, field_name, parsed_field) # Stop here if there was an error parsing the tuple (fallback to returning Any) - if parsed_field is None: + if parsed_field.error: return AnyType(TypeOfAny.from_error) - # These conditions should always be true, otherwise we have an error in our code - assert parsed_field.error is False - assert parsed_field.validated_type is not None - # Get the combined type of the field (union of validated type and default type) combined_type = self._get_combined_type(parsed_field) self._log_debug(' => Combined type of tuple', combined_type) return combined_type - def _parse_rhs_expression(self, rhs_expr: Expression) -> ParsedValidataclassField | None: + def _get_call_args(self) -> tuple[Expression, str, str, list[str]]: + """ + Parse the call argument expressions of the virtual field wrapper call. + Return a tuple with the field RHS expression, the class name, the field name and a list of base class names. + """ + # We can assume that every call to the virtual field wrapper was constructed by us, and that every call has + # the expected number of arguments, so using assert is safe here. + assert len(self._ctx.args) == 4, 'Virtual field wrapper was called with invalid number of arguments' + assert all(len(arg) == 1 for arg in self._ctx.args) + + return ( + # First argument: Wrapped right-hand side expression of the validataclass field assignment statement + self._ctx.args[0][0], + + # Second argument: String expression, fully qualified name of the current class + self._get_call_arg_as_str(1), + + # Third argument: String expression, name of the field + self._get_call_arg_as_str(2), + + # Fourth argument: List expression with string expressions, names of base classes that define this field. + self._get_call_arg_as_list_of_str(3), + ) + + def _get_call_arg_as_str(self, arg_number: int) -> str: + """ + Parse the n-th call argument as a StrExpr and return the string value. + """ + expr = self._ctx.args[arg_number][0] + assert isinstance(expr, StrExpr) + return expr.value + + def _get_call_arg_as_list_of_str(self, arg_number: int) -> list[str]: + """ + Parse the n-th call argument as a ListExpr containing StrExprs and return a list of the string values. + """ + expr = self._ctx.args[arg_number][0] + assert isinstance(expr, ListExpr) + assert all(isinstance(item, StrExpr) for item in expr.items) + return [cast(StrExpr, item).value for item in expr.items] + + def _parse_field_definition( + self, + field_name: str, + field_rhs_expr: Expression, + base_classes: list[str], + ) -> ParsedValidataclassField: + """ + Analyze the entire field definition, starting by collecting the validator and default types of all base classes + (by retrieving the ParsedValidataclasFields from the parsed field cache), then analyzing the right-hand side + expression of the current field assignment and merging the results. + """ + # This will hold the end result that's returned at the end of the function + fully_parsed_field = ParsedValidataclassField() + + # First, retrieve the parsed field of all base classes from our cache, starting with the "oldest" class + for base_class in base_classes: + self._log_debug(f'Retrieving parsed field of base class"{base_class}" from cache') + base_parsed_field = self._parsed_field_cache.get_field(base_class, field_name) + + # If there was an error when the field in the base class was parsed, set the error flag for later, then + # ignore the base class and continue. + if base_parsed_field.error: + fully_parsed_field.error = True + else: + # Overwrite validator and default, but only if they are defined in the expression + fully_parsed_field.merge(base_parsed_field) + + # Report an error if one of the base classes had an error. We don't need a more specific error message here + # because the error should have already been reported when the base class was analyzed. We will continue parsing + # the field so that specific errors in this class are reported, but the user should know that parsing is + # incomplete because of a prior error. + if fully_parsed_field.error: + self._fail('Field cannot be fully parsed because of a prior error in one of the base classes') + + # Now, parse the right-hand side expression of the assignment in the current class and merge the result + self._log_debug('Parsing RHS of assignment', field_rhs_expr) + assignment_parsed_field = self._parse_rhs_expression(field_rhs_expr) + fully_parsed_field.merge(assignment_parsed_field) + + # Make sure that the field has a validator instance (if there was no other error yet) + if not fully_parsed_field.error and fully_parsed_field.validated_type is None: + fully_parsed_field.error = True + self._fail('No Validator found in field definition') + + return fully_parsed_field + + def _parse_rhs_expression(self, rhs_expr: Expression) -> ParsedValidataclassField: """ Analyze the right-hand side expression of a validataclass field, determining its type based on validator and default objects found in the type of the expression. @@ -131,33 +206,19 @@ def _parse_rhs_expression(self, rhs_expr: Expression) -> ParsedValidataclassFiel if isinstance(rhs_type, TupleType): # We have a tuple on the right-hand side, probably a tuple with validator and default object - self._log_debug('Field RHS is a tuple', rhs_type) - - for index, item_type in enumerate(rhs_type.items): + for item_type in rhs_type.items: item_type = get_proper_type(item_type) self._parse_rhs_item(item_type, parsed_field) else: # We have a single Instance on the right-hand side, probably a validator or default object - self._log_debug('Field RHS is a single item', rhs_type) self._parse_rhs_item(rhs_type, parsed_field) - # Stop analyzing if we have already found errors in the field definition - if parsed_field.error: - self._log_debug('Field has errors, stop checking any further') - return None - - # Make sure the field has at least a validator instance - if parsed_field.validated_type is None: - # TODO: Consider base classes here too - self._fail('No Validator found in field definition') - return None - return parsed_field def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassField) -> None: """ Analyze the type of a single item in the right-hand side of a validataclass field, i.e. an item in a tuple or - a the whole right-hand side if it's not a tuple. + the whole right-hand side if it's not a tuple. The result will be written into the given ParsedValidataclassField. May report errors, e.g. for unknown types or for another validator/default instance if the parsed field already has one. @@ -172,10 +233,12 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel if parsed_field.validated_type is not None: parsed_field.error = True self._fail('Multiple validator instances found in field definition') - else: - # Find out the output type of this validator - parsed_field.validated_type = self._get_method_return_type(item_type, 'validate') - self._log_debug(f' - Validator: {item_type.type.name} -> {parsed_field.validated_type}') + return + + # Find out the output type of this validator + # TODO: Actually store the Validator and evaluate its type later + parsed_field.validated_type = self._get_method_return_type(item_type, 'validate') + self._log_debug(f' - Validator: {item_type.type.name} -> {parsed_field.validated_type}') return # Check for default instances (base class BaseDefault) @@ -184,14 +247,19 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel if parsed_field.default_type is not None: parsed_field.error = True self._fail('Multiple default objects found in field definition') - else: - # Find out the value type of this default object - parsed_field.default_type = self._get_method_return_type(item_type, 'get_value') - self._log_debug(f' - Default object: {item_type.type.name} -> {parsed_field.default_type}') + return + + # Find out the value type of this default object + # TODO: Actually store the Default and evaluate its type later + parsed_field.default_type = self._get_method_return_type(item_type, 'get_value') + self._log_debug(f' - Default object: {item_type.type.name} -> {parsed_field.default_type}') return + # (Everything else is probably an error!) + # One easy mistake is writing a validator class name without parentheses (e.g. `field: int = IntegerValidator`). # Let's provide a more helpful error message (the default string representation would be long and confusing). + item_type_str = str(item_type) if isinstance(item_type, CallableType): callable_ret_type = get_proper_type(item_type.ret_type) item_type_str = f'Callable[..., {callable_ret_type}]' @@ -204,10 +272,8 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel f'"{callable_ret_type.type.name}()"?)' ) return - else: - item_type_str = str(item_type) - # Everything else is probably an error + # Fallback to unexpected type error parsed_field.error = True self._fail(f'Unexpected type "{item_type_str}" in field definition (expected Validator or BaseDefault)') @@ -246,10 +312,13 @@ def _get_combined_type(self, parsed_field: ParsedValidataclassField) -> ProperTy """ Returns the combined type of a field, i.e. a union of validated type and default type. """ - assert parsed_field.validated_type is not None, 'No validator. This should have been caught earlier.' + assert parsed_field.validated_type is not None or parsed_field.default_type is not None, \ + 'No validator and no default. This should have been caught earlier.' if parsed_field.default_type is None: - return parsed_field.validated_type + return parsed_field.validated_type # type: ignore[return-value] # TODO: Known issue, fix me + elif parsed_field.validated_type is None: + return parsed_field.default_type else: return make_simplified_union( [parsed_field.validated_type, parsed_field.default_type], diff --git a/tests/mypy/dataclasses/test_validataclass_subclasses.yml b/tests/mypy/dataclasses/test_validataclass_subclasses.yml new file mode 100644 index 0000000..4aeb1ca --- /dev/null +++ b/tests/mypy/dataclasses/test_validataclass_subclasses.yml @@ -0,0 +1,373 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/typeddjango/pytest-mypy-plugins/master/pytest_mypy_plugins/schema.json + +# Create a trivial subclassed validataclass without any fields +- case: empty_validataclass_subclasses + main: | + from validataclass.dataclasses import validataclass + + @validataclass + class EmptyBase: + pass + + @validataclass + class EmptySubClass(EmptyBase): + pass + + base = EmptyBase() + sub = EmptySubClass() + reveal_type(base) # N: Revealed type is "main.EmptyBase" + reveal_type(sub) # N: Revealed type is "main.EmptySubClass" + + # Test polymorphy + base_var: EmptyBase = sub # ok + sub_var: EmptySubClass = base # E: Incompatible types in assignment (expression has type "EmptyBase", variable has type "EmptySubClass") [assignment] + +# Create different subclasses of a validataclass, modifying the fields in various ways +# TODO: Missing init parameters aren't reported as errors yet. +- case: validataclass_with_subclasses + main: | + from validataclass.dataclasses import validataclass, Default, DefaultUnset, NoDefault + from validataclass.helpers import UnsetValue, UnsetValueType + from validataclass.validators import IntegerValidator, StringValidator + + # -- BASE CLASS -- + + @validataclass + class BaseClass: + required1: int = IntegerValidator() + required2: int = IntegerValidator() + optional1: int = IntegerValidator(), Default(0) + optional2: str | None = StringValidator(), Default(None) + + # -- SUB CLASS A -- + + @validataclass + class SubClassA(BaseClass): + # Don't change existing fields, but add a new one + new_field: str = StringValidator() + + # Valid initializations + valid_a1 = SubClassA( + required1=1, + required2=2, + new_field='example', + ) + valid_a2 = SubClassA( + required1=1, + required2=2, + new_field='example', + optional1=42, + optional2=None, + ) + reveal_type(valid_a1) # N: Revealed type is "main.SubClassA" + reveal_type(valid_a2) # N: Revealed type is "main.SubClassA" + reveal_type(valid_a1.required1) # N: Revealed type is "builtins.int" + reveal_type(valid_a1.new_field) # N: Revealed type is "builtins.str" + + # Invalid initializations + invalid_a1 = SubClassA() # TODO: This should be an error! + invalid_a2 = SubClassA( + required1='foo', # E: Argument "required1" to "SubClassA" has incompatible type "str"; expected "int" [arg-type] + required2=None, # E: Argument "required2" to "SubClassA" has incompatible type "None"; expected "int" [arg-type] + new_field=123, # E: Argument "new_field" to "SubClassA" has incompatible type "int"; expected "str" [arg-type] + optional1='foo', # E: Argument "optional1" to "SubClassA" has incompatible type "str"; expected "int" [arg-type] + optional2=42, # E: Argument "optional2" to "SubClassA" has incompatible type "int"; expected "str | None" [arg-type] + ) + + # -- SUB CLASS B -- + + @validataclass + class SubClassB(BaseClass): + # Make required field optional, keep the type + required1: int = Default(0) + + # Make required field optional, expanding the type: int -> int | UnsetValueType + required2: int | UnsetValueType = DefaultUnset + + # Change default of optional field, expanding the type: int -> int | None + optional1: int | None = Default(None) + + # Make optional field required, narrowing the type: str | None -> str + optional2: str = NoDefault + + # Valid initializations + valid_b1 = SubClassB( + optional1=42, + optional2='foo', + ) + valid_b2 = SubClassB( + required1=1, + required2=UnsetValue, + optional1=None, + optional2='foo', + ) + reveal_type(valid_b1) # N: Revealed type is "main.SubClassB" + reveal_type(valid_b2) # N: Revealed type is "main.SubClassB" + reveal_type(valid_b1.required1) # N: Revealed type is "builtins.int" + reveal_type(valid_b1.required2) # N: Revealed type is "builtins.int | validataclass.helpers.unset_value.UnsetValueType" + reveal_type(valid_b1.optional1) # N: Revealed type is "builtins.int | None" + reveal_type(valid_b1.optional2) # N: Revealed type is "builtins.str" + + # Invalid initializations + invalid_b1 = SubClassB() # TODO: This should be an error! + invalid_b2 = SubClassB( + required1='foo', # E: Argument "required1" to "SubClassB" has incompatible type "str"; expected "int" [arg-type] + required2='foo', # E: Argument "required2" to "SubClassB" has incompatible type "str"; expected "int | UnsetValueType" [arg-type] + optional1=UnsetValue, # E: Argument "optional1" to "SubClassB" has incompatible type "UnsetValueType"; expected "int | None" [arg-type] + optional2=None, # E: Argument "optional2" to "SubClassB" has incompatible type "None"; expected "str" [arg-type] + ) + + # -- SUB CLASS C -- + + @validataclass + class SubClassC(BaseClass): + # Change validator: same type + required1: int = IntegerValidator(min_value=0) + + # Change validator and default, expanding the type + required2: int | None = IntegerValidator(min_value=0), Default(None) + + # Change validator: different type, implicitly keep Default(0) + optional1: int | str = StringValidator() + + # Keep validator, change default to different type + optional2: str | UnsetValueType = DefaultUnset + + # Valid initializations + valid_c1 = SubClassC( + required1=1, + ) + valid_c2 = SubClassC( + required1=1, + required2=2, + optional1=3, + optional2='foo', + ) + valid_c3 = SubClassC( + required1=1, + required2=None, + optional1='foo', + optional2=UnsetValue, + ) + reveal_type(valid_c1) # N: Revealed type is "main.SubClassC" + reveal_type(valid_c2) # N: Revealed type is "main.SubClassC" + reveal_type(valid_c3) # N: Revealed type is "main.SubClassC" + reveal_type(valid_c1.required1) # N: Revealed type is "builtins.int" + reveal_type(valid_c1.required2) # N: Revealed type is "builtins.int | None" + reveal_type(valid_c1.optional1) # N: Revealed type is "builtins.int | builtins.str" + reveal_type(valid_c1.optional2) # N: Revealed type is "builtins.str | validataclass.helpers.unset_value.UnsetValueType" + + # Invalid initializations + invalid_c1 = SubClassC() # TODO: This should be an error! + invalid_c2 = SubClassC( + required1='foo', # E: Argument "required1" to "SubClassC" has incompatible type "str"; expected "int" [arg-type] + required2='foo', # E: Argument "required2" to "SubClassC" has incompatible type "str"; expected "int | None" [arg-type] + optional1=None, # E: Argument "optional1" to "SubClassC" has incompatible type "None"; expected "int | str" [arg-type] + optional2=42, # E: Argument "optional2" to "SubClassC" has incompatible type "int"; expected "str | UnsetValueType" [arg-type] + ) + +# Create a subclass of a subclass of a validataclass +# TODO: Missing init parameters aren't reported as errors yet. +- case: validataclass_with_subclasses_of_subclasses + main: | + from validataclass.dataclasses import validataclass, Default, DefaultUnset, NoDefault + from validataclass.helpers import UnsetValue, UnsetValueType + from validataclass.validators import IntegerValidator, StringValidator + + @validataclass + class A: + unmodified: int = IntegerValidator() + once_modified1: int = IntegerValidator() + once_modified2: int = IntegerValidator() + twice_modified1: int = IntegerValidator() + twice_modified2: int | None = IntegerValidator(), Default(None) + + @validataclass + class B(A): + additional: str | None = StringValidator(), Default(None) + once_modified1: int | None = Default(None) + twice_modified1: str = StringValidator() + twice_modified2: int = NoDefault + + @validataclass + class C(B): + once_modified2: int | UnsetValueType = DefaultUnset + twice_modified1: str | UnsetValueType = DefaultUnset + twice_modified2: int | UnsetValueType = DefaultUnset + + # Valid initializations + valid1 = C( + unmodified=1, + ) + valid2 = C( + unmodified=1, + additional='foo', + once_modified1=2, + once_modified2=3, + twice_modified1='foo', + twice_modified2=4, + ) + valid3 = C( + unmodified=1, + additional=None, + once_modified1=None, + once_modified2=UnsetValue, + twice_modified1=UnsetValue, + twice_modified2=UnsetValue, + ) + reveal_type(valid1) # N: Revealed type is "main.C" + reveal_type(valid2) # N: Revealed type is "main.C" + reveal_type(valid3) # N: Revealed type is "main.C" + reveal_type(valid1.unmodified) # N: Revealed type is "builtins.int" + reveal_type(valid1.additional) # N: Revealed type is "builtins.str | None" + reveal_type(valid1.once_modified1) # N: Revealed type is "builtins.int | None" + reveal_type(valid1.once_modified2) # N: Revealed type is "builtins.int | validataclass.helpers.unset_value.UnsetValueType" + reveal_type(valid1.twice_modified1) # N: Revealed type is "builtins.str | validataclass.helpers.unset_value.UnsetValueType" + reveal_type(valid1.twice_modified2) # N: Revealed type is "builtins.int | validataclass.helpers.unset_value.UnsetValueType" + + # Invalid initializations + invalid1 = C() # TODO: This should be an error! + invalid2 = C( + unmodified='foo', # E: Argument "unmodified" to "C" has incompatible type "str"; expected "int" [arg-type] + additional=42, # E: Argument "additional" to "C" has incompatible type "int"; expected "str | None" [arg-type] + once_modified1='foo', # E: Argument "once_modified1" to "C" has incompatible type "str"; expected "int | None" [arg-type] + once_modified2=None, # E: Argument "once_modified2" to "C" has incompatible type "None"; expected "int | UnsetValueType" [arg-type] + twice_modified1=None, # E: Argument "twice_modified1" to "C" has incompatible type "None"; expected "str | UnsetValueType" [arg-type] + twice_modified2=None, # E: Argument "twice_modified2" to "C" has incompatible type "None"; expected "int | UnsetValueType" [arg-type] + ) + +# Create validataclasses with multiple inheritance +# TODO: Missing init parameters aren't reported as errors yet. +- case: validataclass_with_multiple_inheritance + main: | + from validataclass.dataclasses import validataclass, Default, DefaultUnset + from validataclass.helpers import UnsetValue, UnsetValueType + from validataclass.validators import IntegerValidator, StringValidator + + @validataclass + class A: + unmodified_a: int = IntegerValidator() + unmodified_both: int | None = IntegerValidator(), Default(None) + modified_a: int = IntegerValidator() + modified_both: int = IntegerValidator(), Default(1) + + @validataclass + class B: + unmodified_b: str = StringValidator() + unmodified_both: str = StringValidator() + modified_b: str = StringValidator() + modified_both: str = StringValidator(), Default('') + + @validataclass + class C(B, A): + modified_a: int = Default(0) + modified_b: str | None = Default(None) + modified_both: str | UnsetValueType = DefaultUnset + + # TODO: Currently this requires an explicit type to be correct (otherwise it's just `str` from B). We cannot + # have an assignment without RHS though, so we need to define at least a validator or default. Should we + # handle this in the library and/or plugin somehow? + unmodified_both: str | None = StringValidator() + + # Valid initializations + valid1 = C( + unmodified_a=1, + unmodified_b='example', + ) + valid2 = C( + unmodified_a=1, + unmodified_b='example', + unmodified_both='example', + modified_a=1, + modified_b='example', + modified_both='example', + ) + valid3 = C( + unmodified_a=1, + unmodified_b='example', + unmodified_both=None, + modified_a=1, + modified_b=None, + modified_both=UnsetValue, + ) + reveal_type(valid1) # N: Revealed type is "main.C" + reveal_type(valid2) # N: Revealed type is "main.C" + reveal_type(valid3) # N: Revealed type is "main.C" + reveal_type(valid1.unmodified_a) # N: Revealed type is "builtins.int" + reveal_type(valid1.unmodified_b) # N: Revealed type is "builtins.str" + reveal_type(valid1.unmodified_both) # N: Revealed type is "builtins.str | None" + reveal_type(valid1.modified_a) # N: Revealed type is "builtins.int" + reveal_type(valid1.modified_b) # N: Revealed type is "builtins.str | None" + reveal_type(valid1.modified_both) # N: Revealed type is "builtins.str | validataclass.helpers.unset_value.UnsetValueType" + + # Invalid initializations + invalid_c1 = C() # TODO: This should be an error! + invalid_c2 = C( + unmodified_a='foo', # E: Argument "unmodified_a" to "C" has incompatible type "str"; expected "int" [arg-type] + unmodified_b=42, # E: Argument "unmodified_b" to "C" has incompatible type "int"; expected "str" [arg-type] + unmodified_both=42, # E: Argument "unmodified_both" to "C" has incompatible type "int"; expected "str | None" [arg-type] + modified_a='foo', # E: Argument "modified_a" to "C" has incompatible type "str"; expected "int" [arg-type] + modified_b=42, # E: Argument "modified_b" to "C" has incompatible type "int"; expected "str | None" [arg-type] + modified_both=42, # E: Argument "modified_both" to "C" has incompatible type "int"; expected "str | UnsetValueType" [arg-type] + ) + +# Define base class and subclasses in separate modules/files. +# This test case is important to prevent issues related to the (module-based) mypy cache. +- case: validataclass_subclasses_in_other_modules + main: | + from test_files.x_sub_class_b import SubClassB + + # Valid initializations + valid1 = SubClassB( + field1=1, + field2='test', + field3=2, + ) + valid2 = SubClassB( + field1=None, + field2=None, + field3=0, + ) + reveal_type(valid1) # N: Revealed type is "test_files.x_sub_class_b.SubClassB" + reveal_type(valid1.field1) # N: Revealed type is "builtins.int | None" + reveal_type(valid1.field2) # N: Revealed type is "builtins.str | None" + reveal_type(valid1.field3) # N: Revealed type is "builtins.int" + + # Invalid initializations + invalid1 = SubClassB( + field1='foo', # E: Argument "field1" to "SubClassB" has incompatible type "str"; expected "int | None" [arg-type] + field2=42, # E: Argument "field2" to "SubClassB" has incompatible type "int"; expected "str | None" [arg-type] + field3=None, # E: Argument "field3" to "SubClassB" has incompatible type "None"; expected "int" [arg-type] + ) + files: + # Use letter prefixes to make sure the alphabetical order is irrelevant + - path: test_files/z_base_class.py + content: | + from validataclass.dataclasses import validataclass + from validataclass.validators import IntegerValidator + + @validataclass + class Base: + field1: int = IntegerValidator() + + - path: test_files/y_sub_class_a.py + content: | + from validataclass.dataclasses import validataclass, Default + from validataclass.validators import StringValidator + from test_files.z_base_class import Base + + @validataclass + class SubClassA(Base): + field1: int | None = Default(None) + field2: str = StringValidator() + + - path: test_files/x_sub_class_b.py + content: | + from validataclass.dataclasses import validataclass, Default + from validataclass.validators import IntegerValidator + from test_files.y_sub_class_a import SubClassA + + @validataclass + class SubClassB(SubClassA): + field2: str | None = Default(None) + field3: int = IntegerValidator() diff --git a/tests/unit/dataclasses/validataclass_test.py b/tests/unit/dataclasses/validataclass_test.py index f2e8d15..68be274 100644 --- a/tests/unit/dataclasses/validataclass_test.py +++ b/tests/unit/dataclasses/validataclass_test.py @@ -138,7 +138,7 @@ class UnitTestValidatorDataclass: validated: int = IntegerValidator() # Non-init field - non_init: int = dataclasses.field(init=False, default=1) + non_init: int = dataclasses.field(init=False, default=1) # type: ignore[validataclass] # TODO: Fix me! # Get fields from dataclass fields = get_dataclass_fields(UnitTestValidatorDataclass) @@ -281,9 +281,14 @@ class SubClass(BaseClass): ] # Check type annotations - assert all(fields[field].type == int for field in ['required1', 'required2', 'optional2', 'optional4']) - assert all(fields[field].type == int | None for field in ['required3', 'optional1']) - assert all(fields[field].type == int | UnsetValueType for field in ['required4', 'optional3']) + assert fields['required1'].type == int + assert fields['required2'].type == int + assert fields['required3'].type == int | None + assert fields['required4'].type == int | UnsetValueType + assert fields['optional1'].type == int | None + assert fields['optional2'].type == int + assert fields['optional3'].type == int | UnsetValueType + assert fields['optional4'].type == int # Check validators assert all(type(field.metadata.get('validator')) is IntegerValidator for field in fields.values()) @@ -321,7 +326,7 @@ class SubClass(BaseClass): required2: str | None = StringValidator(), Default(None) # Optional fields - optional1: str = StringValidator() # No default override, so the default should still be Default(3) + optional1: str | int = StringValidator() # No default override, so the default should still be Default(3) optional2: str | None = StringValidator(), Default(None) # New fields @@ -335,8 +340,12 @@ class SubClass(BaseClass): assert list(fields.keys()) == ['required1', 'required2', 'optional1', 'optional2', 'new1', 'new2'] # Check type annotations - assert all(fields[field].type == str for field in ['required1', 'optional1', 'new1']) - assert all(fields[field].type == str | None for field in ['required2', 'optional2', 'new2']) + assert fields['required1'].type == str + assert fields['required2'].type == str | None + assert fields['optional1'].type == str | int + assert fields['optional2'].type == str | None + assert fields['new1'].type == str + assert fields['new2'].type == str | None # Check validators assert all(type(field.metadata.get('validator')) is StringValidator for field in fields.values()) @@ -361,7 +370,7 @@ class BaseClass: validated: int = IntegerValidator() # Non-init field - non_init: int = dataclasses.field(init=False, default=1) + non_init: int = dataclasses.field(init=False, default=1) # type: ignore[validataclass] # TODO: Fix me! @validataclass class SubClass(BaseClass): @@ -373,7 +382,8 @@ class SubClass(BaseClass): # Check names and types of all fields assert list(fields.keys()) == ['validated', 'non_init'] - assert all(f.type == int for f in fields.values()) + assert fields['validated'].type == int + assert fields['non_init'].type == int # Check non-init field assert fields['non_init'].init is False @@ -395,7 +405,7 @@ class BaseB: field_both: str = StringValidator() @validataclass - class SubClass(BaseB, BaseA): # type: ignore[misc] + class SubClass(BaseB, BaseA): # Override the defaults to test that the decorator recognizes all fields of both base classes. # If it does not, a "no validator for field X" error would be raised. field_a: int = Default(42) diff --git a/tests/unit/validators/integer_validator_test.py b/tests/unit/validators/integer_validator_test.py index ca2667e..f61fcbd 100644 --- a/tests/unit/validators/integer_validator_test.py +++ b/tests/unit/validators/integer_validator_test.py @@ -4,6 +4,8 @@ Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. """ +from typing import Any + import pytest from tests.test_utils import UNSET_PARAMETER @@ -265,7 +267,7 @@ def test_integer_value_range_invalid(min_value, max_value, input_data_list): validator = IntegerValidator(**validator_args) # Construct error dict with min_value and/or max_value, depending on which is specified - expected_error_dict = {'code': 'number_range_error'} + expected_error_dict: dict[str, Any] = {'code': 'number_range_error'} if min_value is not None: expected_error_dict['min_value'] = min_value if min_value is not UNSET_PARAMETER else -2147483648 if max_value is not None: From 195d2f666bbc7b321c3df692ebd6a7715fa18a10 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Tue, 3 Mar 2026 18:13:14 +0100 Subject: [PATCH 07/15] mypy plugin: Handle more edge cases, add more tests --- .coveragerc | 5 +- .../dataclasses/validataclass.py | 2 +- src/validataclass/mypy/plugin/__init__.py | 4 + src/validataclass/mypy/plugin/constants.py | 17 ++- .../plugin/{helpers.py => debug_logger.py} | 2 +- .../mypy/plugin/parsed_field_cache.py | 12 +- src/validataclass/mypy/plugin/plugin.py | 2 +- .../mypy/plugin/validataclass_transformer.py | 76 +++++++--- .../mypy/plugin/virtual_field_resolver.py | 104 +++++++++----- tests/mypy/dataclasses/test_validataclass.yml | 110 ++++++++++++++- .../test_validataclass_subclasses.yml | 131 +++++++++++++++++- tests/unit/dataclasses/validataclass_test.py | 13 +- 12 files changed, 397 insertions(+), 81 deletions(-) rename src/validataclass/mypy/plugin/{helpers.py => debug_logger.py} (96%) diff --git a/.coveragerc b/.coveragerc index 9509b63..d314f69 100644 --- a/.coveragerc +++ b/.coveragerc @@ -13,11 +13,8 @@ omit = show_missing = True skip_empty = True skip_covered = True -exclude_lines = - pragma: no ?cover +exclude_also = @abstractmethod - @overload - if TYPE_CHECKING: [html] directory = reports/coverage_html/ diff --git a/src/validataclass/dataclasses/validataclass.py b/src/validataclass/dataclasses/validataclass.py index 9c22a2b..46a2c47 100644 --- a/src/validataclass/dataclasses/validataclass.py +++ b/src/validataclass/dataclasses/validataclass.py @@ -112,7 +112,7 @@ def _prepare_dataclass_metadata(cls: type[_T]) -> None: # In case of a subclassed validataclass, get the already existing fields existing_validator_fields = _get_existing_validator_fields(cls) - # Get class annotations + # Get annotations of this class (ignores base classes) cls_annotations = get_annotations(cls) # Check for fields/attributes that have validators defined but missing a type annotation (most likely an error) diff --git a/src/validataclass/mypy/plugin/__init__.py b/src/validataclass/mypy/plugin/__init__.py index b0530a8..eb78632 100644 --- a/src/validataclass/mypy/plugin/__init__.py +++ b/src/validataclass/mypy/plugin/__init__.py @@ -5,3 +5,7 @@ """ from .plugin import plugin + +__all__ = [ + 'plugin', +] diff --git a/src/validataclass/mypy/plugin/constants.py b/src/validataclass/mypy/plugin/constants.py index 0296114..5186e7a 100644 --- a/src/validataclass/mypy/plugin/constants.py +++ b/src/validataclass/mypy/plugin/constants.py @@ -33,7 +33,20 @@ VIRTUAL_FIELD_WRAPPER_FUNC_NAME: Final = '_virtual_field_wrapper' # Custom error codes for validataclass -# TODO: Maybe add multiple error codes (as sub codes of the validataclass code). ERROR_CODE_VALIDATACLASS: Final = ErrorCode( - 'validataclass', 'Check that validataclasses are defined correctly', 'Plugin' + 'validataclass', + 'Check that validataclasses are defined correctly', + 'Plugin', +) +ERROR_CODE_VALIDATACLASS_EMPTY_TYPE: Final = ErrorCode( + 'validataclass-empty-type', + 'Check that validataclass field has a validator or default that can return a value', + 'Plugin', + sub_code_of=ERROR_CODE_VALIDATACLASS, +) +ERROR_CODE_VALIDATACLASS_NOT_IMPLEMENTED: Final = ErrorCode( + 'validataclass-not-implemented', + 'Special code for edge cases that are currently not supported by the plugin (please create a bug report)', + 'Plugin', + sub_code_of=ERROR_CODE_VALIDATACLASS, ) diff --git a/src/validataclass/mypy/plugin/helpers.py b/src/validataclass/mypy/plugin/debug_logger.py similarity index 96% rename from src/validataclass/mypy/plugin/helpers.py rename to src/validataclass/mypy/plugin/debug_logger.py index b489e19..cea5112 100644 --- a/src/validataclass/mypy/plugin/helpers.py +++ b/src/validataclass/mypy/plugin/debug_logger.py @@ -17,7 +17,7 @@ class DebugLogger: def __init__(self, debug_mode: bool): self.debug_mode = debug_mode - def log(self, level: str, context: str | None, msg: str, *objects: Any) -> None: + def log(self, level: str, context: str | None, msg: str, *objects: Any) -> None: # pragma: nocover """ Log a message on a given level, optionally with a context (e.g. file/line) and with object dumps. Debug level messages are only logged if debug mode is enabled. diff --git a/src/validataclass/mypy/plugin/parsed_field_cache.py b/src/validataclass/mypy/plugin/parsed_field_cache.py index d035920..78b0050 100644 --- a/src/validataclass/mypy/plugin/parsed_field_cache.py +++ b/src/validataclass/mypy/plugin/parsed_field_cache.py @@ -4,7 +4,7 @@ Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. """ -from mypy.types import ProperType +from mypy.types import Instance from typing_extensions import Self @@ -17,19 +17,19 @@ class ParsedValidataclassField: error: bool = False # If None, the field does not have an explicit validator/default, but could still have one from the base class. - validated_type: ProperType | None = None - default_type: ProperType | None = None + validator_type: Instance | None = None + default_type: Instance | None = None def merge(self, other: Self) -> None: """ Merge another instance of this class into this one. - Validated type and default type are replaced if the attributes in the other instance are not None. + Validator type and default object type are replaced if the attributes in the other instance are not None. The error flag is set to True if it's true for any instance. """ self.error = self.error or other.error - if other.validated_type is not None: - self.validated_type = other.validated_type + if other.validator_type is not None: + self.validator_type = other.validator_type if other.default_type is not None: self.default_type = other.default_type diff --git a/src/validataclass/mypy/plugin/plugin.py b/src/validataclass/mypy/plugin/plugin.py index 48e02e2..c8429e8 100644 --- a/src/validataclass/mypy/plugin/plugin.py +++ b/src/validataclass/mypy/plugin/plugin.py @@ -15,7 +15,7 @@ from typing_extensions import override from .constants import VALIDATACLASS_DECORATORS, VIRTUAL_FIELD_WRAPPER_FUNC -from .helpers import DebugLogger +from .debug_logger import DebugLogger from .parsed_field_cache import ParsedFieldCache from .validataclass_transformer import ValidataclassTransformer from .virtual_field_resolver import VirtualFieldResolver diff --git a/src/validataclass/mypy/plugin/validataclass_transformer.py b/src/validataclass/mypy/plugin/validataclass_transformer.py index 31d1d48..9e934f4 100644 --- a/src/validataclass/mypy/plugin/validataclass_transformer.py +++ b/src/validataclass/mypy/plugin/validataclass_transformer.py @@ -32,11 +32,12 @@ from .constants import ( ERROR_CODE_VALIDATACLASS, + ERROR_CODE_VALIDATACLASS_NOT_IMPLEMENTED, VALIDATACLASS_FIELD_FUNC, VIRTUAL_FIELD_WRAPPER_FUNC, VIRTUAL_FIELD_WRAPPER_FUNC_NAME, ) -from .helpers import DebugLogger +from .debug_logger import DebugLogger class ValidataclassField: @@ -63,7 +64,7 @@ def __init__(self, name: str): self.assignment_stmt = None @override - def __repr__(self) -> str: + def __repr__(self) -> str: # pragma: nocover return f'ValidataclassField({self.name})' @property @@ -143,26 +144,44 @@ def _get_logger_context(self, context: Context | None) -> str: context = self._decorator return f'{self._class_def.fullname}:{context.line}' - def _log_warn(self, context: Context | None, msg: str, *objects: Any) -> None: + def _log_warn(self, context: Context | None, msg: str, *objects: Any) -> None: # pragma: nocover """ - Logs a message on WARN level. - Use this to warn about unhandled expressions/types or features that are not implemented yet. + Log a message on WARN level. + + This should only be used during plugin development to warn about unhandled expressions/types. In real code, + please use `_fail` or `_fail_not_implemented` instead, which result in actual mypy errors. """ return self._logger.log('WARN', self._get_logger_context(context), msg, *objects) def _log_debug(self, context: Context | None, msg: str, *objects: Any) -> None: """ - Logs a message on DEBUG level. Only printed if debug mode is enabled. + Log a message on DEBUG level. Only printed if debug mode is enabled. Use this for better traceability and debugging of what the plugin is doing. """ return self._logger.log('DEBUG', self._get_logger_context(context), msg, *objects) def _fail(self, msg: str, context: Context, *, code: ErrorCode | None = None) -> None: """ - Reports a mypy error to the user. Default code is "validataclass". + Report a mypy error to the user. Default code is "validataclass". """ self._api.fail(msg, context, code=code or ERROR_CODE_VALIDATACLASS) + def _fail_not_implemented(self, msg: str, context: Context) -> None: # pragma: nocover + """ + Report a mypy error with error code "validataclass-not-implemented" as well as a note with more explanation. + + This is intended for edge cases that we don't support yet (e.g. because we don't have a real life example to + reproduce it). The user is requested to please create an upstream issue. + """ + error_info = self._api.msg.fail(msg, context, code=ERROR_CODE_VALIDATACLASS_NOT_IMPLEMENTED) + self._api.msg.note( + 'You found an edge case that is not supported by the validataclass mypy plugin. Please report this error ' + 'with a minimal code example to help improve the plugin: ' + 'https://github.com/binary-butterfly/validataclass/issues', + context, + parent_error=error_info, + ) + def transform(self) -> bool: """ Update the class definition of a validataclass by wrapping the right-hand side of field definitions in the @@ -207,7 +226,7 @@ def _get_assignment_statements_from_block(self, block: Block) -> Iterator[Assign if isinstance(stmt, AssignmentStmt): yield stmt else: - self._log_debug(stmt, f'Skipping statement of type {type(stmt)}') + self._log_debug(stmt, f'Skip statement of type {type(stmt)}') def _collect_fields(self) -> list[ValidataclassField] | None: """ @@ -221,12 +240,12 @@ def _collect_fields(self) -> list[ValidataclassField] | None: # Ignore base classes that are not validataclasses if 'validataclass_tag' not in base_typeinfo.metadata: - self._log_debug(None, f'Skipping base class "{base_fullname}" (no validataclass tag)') + self._log_debug(None, f'Skip base class "{base_fullname}" (no validataclass tag)') continue # Ensure base class has already been processed by this plugin, otherwise we need another pass if 'validataclass' not in base_typeinfo.metadata: - self._log_debug(None, f'Base class "{base_fullname}" not processsed yet, need another pass') + self._log_debug(None, f'Base class "{base_fullname}" not processed yet, need another pass') return None # Gather all fields previously collected by this very function when the base class was analyzed @@ -252,15 +271,32 @@ def _collect_fields(self) -> list[ValidataclassField] | None: # TODO: Unless the attribute starts with an underscore, the decorator actually checks the type of the # RHS of an annotation-less field and raises an error if it contains a validator or default (because # it means you probably forgot the annotation). It's a bit more difficult to check this here though. - self._log_debug(stmt, f'Skipping assignment for "{stmt.lvalues}" without type annotation') + self._log_debug(stmt, f'Skip assignment for "{stmt.lvalues}" without type annotation') continue - # There are more edge cases for the left-hand side that we can skip for now, like multiple assignments - # (`x, y = z`) or chained assignments (`x = y = z`). - # TODO: Check if there are edge cases that we should handle, or that we should even report an error for! + # There are more edge cases for the left-hand side expression that we can skip, like multiple assignments + # (`x, y = z`) or chained assignments (`x = y = z`). These shouldn't happen because they aren't supported + # by Python's type annotation syntax. We report this as an error and ask the user to create a bug report. lvalue = stmt.lvalues[0] if len(stmt.lvalues) == 1 else None - if lvalue is None or not isinstance(lvalue, NameExpr): - self._log_warn(stmt, 'Skipping weird edge case for assignment LHS', stmt.lvalues) + if lvalue is None or not isinstance(lvalue, NameExpr): # pragma: nocover + lvalues_str_repr = ', '.join(str(lvalue) for lvalue in stmt.lvalues) + self._fail_not_implemented( + f'Unexpected left-hand side expression type "{lvalues_str_repr}" in assignment', stmt + ) + continue + + # Edge case when a variable is defined twice in the same class: NameExpr without node. This error is handled + # by mypy (no-redef), so we can just skip it here. + if lvalue.node is None: + self._log_debug(stmt, 'Skip assignment with NameExpr without node (probably redefined)', stmt.lvalues) + continue + + # In theory, the NameExpr node can be something other than a Var (e.g. a FuncDef or TypeInfo), but I have + # no idea when this can happen. Another edge case to report to the user. + if not isinstance(lvalue.node, Var): # pragma: nocover + self._fail_not_implemented( + f'Unexpected NameExpr node type "{str(lvalue.node)}" in assignment left-hand side', stmt + ) continue # Edge case for right-hand side: We can have a type annotation statement without an assignment (`x: int`), @@ -279,7 +315,7 @@ def _collect_fields(self) -> list[ValidataclassField] | None: # Ignore fields that have been created with the regular dataclasses.field() or similar if callee_name in DATACLASS_FIELD_SPECIFIERS: # TODO: Handle these properly. - self._log_debug(stmt, f'Skipping field "{lvalue.name}" with {callee_name}') + self._log_debug(stmt, f'Skip field "{lvalue.name}" with {callee_name}') continue # Skip fields that have been created with validataclass_field() @@ -287,12 +323,12 @@ def _collect_fields(self) -> list[ValidataclassField] | None: if callee_name == VALIDATACLASS_FIELD_FUNC: # TODO: Normalize `default` - if it's NoDefault, remove the default instead? # TODO: (Can probably be ignored because we'll have to fix __init__ manually anyway.) - self._log_debug(stmt, f'Skipping field "{lvalue.name}" with validataclass_field()') + self._log_debug(stmt, f'Skip field "{lvalue.name}" with validataclass_field()') continue # Skip field if it has already been wrapped in a previous pass if callee_name == VIRTUAL_FIELD_WRAPPER_FUNC: - self._log_debug(stmt, f'Skipping already wrapped field "{lvalue.name}"') + self._log_debug(stmt, f'Skip already wrapped field "{lvalue.name}"') continue # If we're still here, we might actually have a validataclass field definition that we care about! @@ -320,7 +356,7 @@ def _transform_field_in_class(self, field: ValidataclassField) -> None: # Wrap the right-hand side of the assignment in a call to the virtual field wrapper function, changing the # assignment statement in the class body in-place. - self._log_debug(field.assignment_stmt, f'Wrapping field "{field.name}" with RHS', field.rvalue) + self._log_debug(field.assignment_stmt, f'Wrap field "{field.name}" with RHS', field.rvalue) field.assignment_stmt.rvalue = self._construct_virtual_wrapper_callexpr( field.rvalue, field.name, diff --git a/src/validataclass/mypy/plugin/virtual_field_resolver.py b/src/validataclass/mypy/plugin/virtual_field_resolver.py index 3883c7d..0a48039 100644 --- a/src/validataclass/mypy/plugin/virtual_field_resolver.py +++ b/src/validataclass/mypy/plugin/virtual_field_resolver.py @@ -19,11 +19,17 @@ TupleType, Type, TypeOfAny, + UninhabitedType, get_proper_type, ) -from .constants import ERROR_CODE_VALIDATACLASS, FIELD_DEFAULT_BASE_CLASS, VALIDATOR_BASE_CLASS -from .helpers import DebugLogger +from .constants import ( + ERROR_CODE_VALIDATACLASS, + ERROR_CODE_VALIDATACLASS_EMPTY_TYPE, + FIELD_DEFAULT_BASE_CLASS, + VALIDATOR_BASE_CLASS, +) +from .debug_logger import DebugLogger from .parsed_field_cache import ParsedFieldCache, ParsedValidataclassField @@ -56,10 +62,13 @@ def _get_logger_context(self) -> str: """ return f'{self._api.path}:{self._ctx.context.line}' - def _log_warn(self, msg: str, *objects: Any) -> None: + def _log_warn(self, msg: str, *objects: Any) -> None: # pragma: nocover """ Log a message on WARN level. - Use this to warn about unhandled expressions/types or features that are not implemented yet. + + This should only be used during plugin development to warn about unhandled expressions/types. In real code, + please use `_fail` or `_fail_not_implemented` (see ValidataclassTransformer) instead, which result in actual + mypy errors. """ return self._logger.log('WARN', self._get_logger_context(), msg, *objects) @@ -101,9 +110,24 @@ def resolve(self) -> Type: return AnyType(TypeOfAny.from_error) # Get the combined type of the field (union of validated type and default type) - combined_type = self._get_combined_type(parsed_field) - self._log_debug(' => Combined type of tuple', combined_type) - return combined_type + resolved_type = self._resolve_field_type(parsed_field) + self._log_debug(' => Resolved and combined field type', resolved_type) + + # Handle edge case where resolved type is the empty type (i.e. Never). The most likely cause for this (which is + # not "user does weird stuff") is that the field uses a RejectValidator (or similar, anything where validate() + # never returns) and does not have a default value. A validataclass with this field *can* be instantiated + # manually (if the field type is anything other than Never), but a DataclassValidator would never return. + # If we return resolved_type here, mypy will mark the class as unreachable code. Instead, we report an error + # and return an Any type. + if isinstance(resolved_type, UninhabitedType): + self._fail( + f'Dataclass can never be validated, validator and default have empty type "{resolved_type}" (did you ' + 'forget a default value?)', + code=ERROR_CODE_VALIDATACLASS_EMPTY_TYPE, + ) + return AnyType(TypeOfAny.from_error) + + return resolved_type def _get_call_args(self) -> tuple[Expression, str, str, list[str]]: """ @@ -162,7 +186,7 @@ def _parse_field_definition( # First, retrieve the parsed field of all base classes from our cache, starting with the "oldest" class for base_class in base_classes: - self._log_debug(f'Retrieving parsed field of base class"{base_class}" from cache') + self._log_debug(f'Retrieve parsed field of base class"{base_class}" from cache') base_parsed_field = self._parsed_field_cache.get_field(base_class, field_name) # If there was an error when the field in the base class was parsed, set the error flag for later, then @@ -181,12 +205,12 @@ def _parse_field_definition( self._fail('Field cannot be fully parsed because of a prior error in one of the base classes') # Now, parse the right-hand side expression of the assignment in the current class and merge the result - self._log_debug('Parsing RHS of assignment', field_rhs_expr) + self._log_debug('Parse RHS of assignment', field_rhs_expr) assignment_parsed_field = self._parse_rhs_expression(field_rhs_expr) fully_parsed_field.merge(assignment_parsed_field) # Make sure that the field has a validator instance (if there was no other error yet) - if not fully_parsed_field.error and fully_parsed_field.validated_type is None: + if not fully_parsed_field.error and fully_parsed_field.validator_type is None: fully_parsed_field.error = True self._fail('No Validator found in field definition') @@ -230,15 +254,14 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel # Check for validator instances (base class Validator) if item_type.type.has_base(VALIDATOR_BASE_CLASS): # Make sure only one validator is set - if parsed_field.validated_type is not None: + if parsed_field.validator_type is not None: parsed_field.error = True self._fail('Multiple validator instances found in field definition') return # Find out the output type of this validator - # TODO: Actually store the Validator and evaluate its type later - parsed_field.validated_type = self._get_method_return_type(item_type, 'validate') - self._log_debug(f' - Validator: {item_type.type.name} -> {parsed_field.validated_type}') + self._log_debug(f' - Validator: {item_type}') + parsed_field.validator_type = item_type return # Check for default instances (base class BaseDefault) @@ -250,9 +273,8 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel return # Find out the value type of this default object - # TODO: Actually store the Default and evaluate its type later - parsed_field.default_type = self._get_method_return_type(item_type, 'get_value') - self._log_debug(f' - Default object: {item_type.type.name} -> {parsed_field.default_type}') + self._log_debug(f' - Default object: {item_type}') + parsed_field.default_type = item_type return # (Everything else is probably an error!) @@ -277,7 +299,31 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel parsed_field.error = True self._fail(f'Unexpected type "{item_type_str}" in field definition (expected Validator or BaseDefault)') - def _get_method_return_type(self, instance_type: Instance, method_name: str) -> ProperType | None: + def _resolve_field_type(self, parsed_field: ParsedValidataclassField) -> ProperType: + """ + Resolve the type of the parsed field by determining the result type of the validator and default objects and + combining them in a type union. + """ + assert parsed_field.validator_type is not None, 'No validator. This should have been caught earlier!' + + # Determine the result type of the validator (i.e. the return type of its validate method) + validator_result_type = self._get_method_return_type(parsed_field.validator_type, 'validate') + + # Determine the result type of the default object (i.e. the return type of its get_value method) + if parsed_field.default_type is not None: + default_result_type = self._get_method_return_type(parsed_field.default_type, 'get_value') + else: + # UninhabitedType is the bottom type (Never/NoReturn), meaning there is no default value + default_result_type = UninhabitedType() + + # Combine result types of validator and default object + return make_simplified_union( + [validator_result_type, default_result_type], + line=self._ctx.context.line, + column=self._ctx.context.column, + ) + + def _get_method_return_type(self, instance_type: Instance, method_name: str) -> ProperType: """ Construct a MemberExpr for the given instance type and method name, resolve expression type and return the return type of the method. Report an error if it's not a valid callable. @@ -301,27 +347,9 @@ def _get_method_return_type(self, instance_type: Instance, method_name: str) -> # Make sure it's a callable. If it isn't, either we forgot some edge case, or the user is doing something *very* # weird (namely overriding validate or get_value to be something that's not callable, which mypy should notice - # anyway). We should report an error here, but can return None as if the validator/default didn't exist. + # anyway). We should report an error here, but can return a Never type as if the validator/default didn't exist. if not isinstance(constructed_method_type, CallableType): self._fail(f'"{instance_type.type.name}.{method_name}" is not a callable') - return None + return UninhabitedType() return get_proper_type(constructed_method_type.ret_type) - - def _get_combined_type(self, parsed_field: ParsedValidataclassField) -> ProperType: - """ - Returns the combined type of a field, i.e. a union of validated type and default type. - """ - assert parsed_field.validated_type is not None or parsed_field.default_type is not None, \ - 'No validator and no default. This should have been caught earlier.' - - if parsed_field.default_type is None: - return parsed_field.validated_type # type: ignore[return-value] # TODO: Known issue, fix me - elif parsed_field.validated_type is None: - return parsed_field.default_type - else: - return make_simplified_union( - [parsed_field.validated_type, parsed_field.default_type], - line=self._ctx.context.line, - column=self._ctx.context.column, - ) diff --git a/tests/mypy/dataclasses/test_validataclass.yml b/tests/mypy/dataclasses/test_validataclass.yml index 7929302..d203a49 100644 --- a/tests/mypy/dataclasses/test_validataclass.yml +++ b/tests/mypy/dataclasses/test_validataclass.yml @@ -40,6 +40,10 @@ reveal_type(valid1) # N: Revealed type is "main.SimpleDataclass" reveal_type(valid2) # N: Revealed type is "main.SimpleDataclass" + reveal_type(valid1.name) # N: Revealed type is "builtins.str" + reveal_type(valid1.noneable) # N: Revealed type is "builtins.int | None" + reveal_type(valid1.numbers) # N: Revealed type is "builtins.list[builtins.int]" + # Invalid initializations invalid1 = SimpleDataclass() # TODO: This should be an error! invalid2 = SimpleDataclass( # E: Unexpected keyword argument "unknown_field" for "SimpleDataclass" [call-arg] @@ -58,7 +62,7 @@ main: | from validataclass.dataclasses import validataclass, Default, DefaultFactory, DefaultUnset, NoDefault from validataclass.helpers import UnsetValue, UnsetValueType - from validataclass.validators import IntegerValidator, StringValidator + from validataclass.validators import IntegerValidator, ListValidator, RejectValidator, StringValidator @validataclass class DataclassWithDefaults: @@ -69,6 +73,9 @@ optional_none: str | None = StringValidator(), Default(None) optional_unset: str | UnsetValueType = StringValidator(), DefaultUnset optional_factory: int = IntegerValidator(), DefaultFactory(lambda: 42) + optional_list: list[int] = ListValidator(IntegerValidator()), Default([]) + + rejected_field: UnsetValueType = RejectValidator(), DefaultUnset # Valid initializations valid1 = DataclassWithDefaults( @@ -82,6 +89,7 @@ optional_none='example', optional_unset='example', optional_factory=42, + optional_list=[1, 2, 3], ) valid3 = DataclassWithDefaults( required_implicit=1, @@ -90,11 +98,21 @@ optional_none=None, optional_unset=UnsetValue, optional_factory=42, + rejected_field=UnsetValue, ) reveal_type(valid1) # N: Revealed type is "main.DataclassWithDefaults" reveal_type(valid2) # N: Revealed type is "main.DataclassWithDefaults" reveal_type(valid3) # N: Revealed type is "main.DataclassWithDefaults" + reveal_type(valid1.required_implicit) # N: Revealed type is "builtins.int" + reveal_type(valid1.required_explicit) # N: Revealed type is "builtins.int" + reveal_type(valid1.optional_str) # N: Revealed type is "builtins.str" + reveal_type(valid1.optional_none) # N: Revealed type is "builtins.str | None" + reveal_type(valid1.optional_unset) # N: Revealed type is "builtins.str | validataclass.helpers.unset_value.UnsetValueType" + reveal_type(valid1.optional_factory) # N: Revealed type is "builtins.int" + reveal_type(valid1.optional_list) # N: Revealed type is "builtins.list[builtins.int]" + reveal_type(valid1.rejected_field) # N: Revealed type is "validataclass.helpers.unset_value.UnsetValueType" + # Invalid initializations invalid1 = DataclassWithDefaults() # TODO: This should be an error! invalid2 = DataclassWithDefaults( @@ -104,14 +122,63 @@ optional_none=UnsetValue, # E: Argument "optional_none" to "DataclassWithDefaults" has incompatible type "UnsetValueType"; expected "str | None" [arg-type] optional_unset=None, # E: Argument "optional_unset" to "DataclassWithDefaults" has incompatible type "None"; expected "str | UnsetValueType" [arg-type] optional_factory='foo', # E: Argument "optional_factory" to "DataclassWithDefaults" has incompatible type "str"; expected "int" [arg-type] + optional_list=1, # E: Argument "optional_list" to "DataclassWithDefaults" has incompatible type "int"; expected "list[int]" [arg-type] ) +# Create a validataclass with more complex validators like AnyOfValidator or DictValidator +# TODO: Currently, the type inference of validators like AnyOfValidator or DictValidator fails in more complex cases, +# e.g. for a mixed type AnyOfValidator or for a DictValidator with inline specified field validators. +# We should check if we can improve this, either with more plugin magic or by modifying the validator code. +- case: validataclass_with_complex_validators + main: | + from enum import Enum + from typing import Any + from validataclass.dataclasses import validataclass, Default + from validataclass.validators import AnythingValidator, AnyOfValidator, DictValidator, DiscardValidator, \ + EnumValidator, IntegerValidator, Noneable + + class ExampleEnum(Enum): + BANANA = 'banana' + + any_of_allowed_values: list[str | int] = ['a', 1] + + @validataclass + class ExampleDataclass: + # AnythingValidator + anything: Any = AnythingValidator() + any_dict_or_none: dict[Any, Any] | None = AnythingValidator(allowed_types=[dict, None]) + any_dict_default: dict[Any, Any] | None = AnythingValidator(allowed_types=[dict]), Default(None) + + # AnyOfValidator + any_of_single_type: str = AnyOfValidator(['a', 'b']) + any_of_inline: str | int = AnyOfValidator(['a', 1]) # type: ignore[assignment] # TODO: Fix me? + any_of_external: str | int = AnyOfValidator(any_of_allowed_values) + any_of_with_default: str | None = AnyOfValidator(['a', 'b']), Default(None) + + # DictValidator + dict_field: dict[str, int] = DictValidator(field_validators={'foo': IntegerValidator()}) # type: ignore[arg-type] # TODO: Fix me? + dict_with_default: dict[str, int] = ( + DictValidator(field_validators={'foo': IntegerValidator()}), # type: ignore[arg-type] # TODO: Fix me? + Default({'foo': 0}), + ) + + # DiscardValidator + discarded_none: None = DiscardValidator() + discarded_zero: int = DiscardValidator(output_value=0) + + # EnumValidator + enum_field: ExampleEnum = EnumValidator(ExampleEnum) + + # Noneable with non-default type + none_to_str: int | str = Noneable(IntegerValidator(), default='') + none_to_str_default: int | str | None = Noneable(IntegerValidator(), default=''), Default(None) + # Invalid validataclasses with errors that the mypy plugin should detect - case: validataclass_invalid main: | from validataclass.dataclasses import validataclass, Default, DefaultFactory, DefaultUnset, NoDefault from validataclass.helpers import UnsetValue, UnsetValueType - from validataclass.validators import IntegerValidator, StringValidator, Validator + from validataclass.validators import IntegerValidator, RejectValidator, StringValidator, Validator class BadValidator(Validator[None]): """ Validator class with invalid validate method. """ @@ -127,6 +194,9 @@ # Annotated field without assignment right-hand side missing_rhs: int # E: Annotated field without assignment (missing Validator or BaseDefault on right-hand side) [validataclass] + # Field with empty type, RejectValidator can never return and field has no default + unvalidatable_field: int = RejectValidator() # E: Dataclass can never be validated, validator and default have empty type "Never" (did you forget a default value?) [validataclass-empty-type] + # Missing validator missing_validator: int = Default(42) # E: No Validator found in field definition [validataclass] @@ -138,6 +208,12 @@ missing_parentheses: int = IntegerValidator # E: Unexpected type "Callable[..., validataclass.validators.integer_validator.IntegerValidator]" in field definition (did you mean "IntegerValidator()"?) [validataclass] missing_parentheses_tuple: int = IntegerValidator(), Default # E: Unexpected type "Callable[..., validataclass.dataclasses.defaults.Default[T_Default`1]]" in field definition (expected Validator or BaseDefault) [validataclass] + # Already defined attribute + wrong_validator: int = IntegerValidator() # E: Name "wrong_validator" already defined on line 12 [no-redef] + + def wrong_default(self) -> int: # E: Name "wrong_default" already defined on line 13 [no-redef] + return 42 + # Miscellaneous nonsense nonsense1: int = 42 # E: Unexpected type "Literal[42]?" in field definition (expected Validator or BaseDefault) [validataclass] nonsense2: int = [42] # E: Unexpected type "builtins.list[builtins.int]" in field definition (expected Validator or BaseDefault) [validataclass] @@ -147,8 +223,34 @@ # Bad validator (.validate is not a function) bad_validator: None = BadValidator() # (multiple errors, see below) out: | - main:37: error: "BadValidator.validate" is not a callable [validataclass] - main:37: error: No Validator found in field definition [validataclass] + main:46: error: "BadValidator.validate" is not a callable [validataclass] + main:46: error: Dataclass can never be validated, validator and default have empty type "Never" (did you forget a default value?) [validataclass-empty-type] + +# Validataclasses with validators or defaults with invalid arguments +# TODO: It would be easier to just use the builtin IntegerValidator and Default here rather than defining custom +# classes. However, there are some problems in pytest-mypy-plugins regarding the "defined here" notes for symbols +# defined in external files. Maybe we can simplify that at some point. +- case: validataclass_invalid_call_args + main: | + from validataclass.dataclasses import validataclass, Default + from validataclass.validators import IntegerValidator, ListValidator + + class ExampleValidator(IntegerValidator): + def __init__(self, *, min_value: int | None = None): # N: "ExampleValidator" defined here + super().__init__(min_value=min_value) + + class DefaultInt(Default[int]): + def __init__(self, value: int): # N: "DefaultInt" defined here + super().__init__(value) + + @validataclass + class InvalidArgsDataclass: + invalid_validator_args1: int = ExampleValidator(nonexistant_argument=42) # E: Unexpected keyword argument "nonexistant_argument" for "ExampleValidator" [call-arg] + invalid_validator_args2: int = ExampleValidator(min_value='42') # E: Argument "min_value" to "ExampleValidator" has incompatible type "str"; expected "int | None" [arg-type] + invalid_validator_args3: list[int] = ListValidator() # E: Missing positional argument "item_validator" in call to "ListValidator" [call-arg] + invalid_default_args1: int | None = IntegerValidator(), DefaultInt() # E: Missing positional argument "value" in call to "DefaultInt" [call-arg] + invalid_default_args2: int = IntegerValidator(), DefaultInt(banana=42) # E: Unexpected keyword argument "banana" for "DefaultInt" [call-arg] + invalid_default_args3: int = IntegerValidator(), DefaultInt('not an int') # E: Argument 1 to "DefaultInt" has incompatible type "str"; expected "int" [arg-type] # Validataclass with various edge cases (that are all allowed / should be ignored) - case: validataclass_valid_edge_cases diff --git a/tests/mypy/dataclasses/test_validataclass_subclasses.yml b/tests/mypy/dataclasses/test_validataclass_subclasses.yml index 4aeb1ca..3039cd9 100644 --- a/tests/mypy/dataclasses/test_validataclass_subclasses.yml +++ b/tests/mypy/dataclasses/test_validataclass_subclasses.yml @@ -16,7 +16,7 @@ base = EmptyBase() sub = EmptySubClass() reveal_type(base) # N: Revealed type is "main.EmptyBase" - reveal_type(sub) # N: Revealed type is "main.EmptySubClass" + reveal_type(sub) # N: Revealed type is "main.EmptySubClass" # Test polymorphy base_var: EmptyBase = sub # ok @@ -371,3 +371,132 @@ class SubClassB(SubClassA): field2: str | None = Default(None) field3: int = IntegerValidator() + +# Base class with errors +- case: validataclass_base_class_with_errors + main: | + from validataclass.dataclasses import validataclass, Default + from validataclass.validators import IntegerValidator, StringValidator + + @validataclass + class BaseClassWithErrors: + # Valid definitions, incorrect typing + wrong_validator: int = StringValidator(), Default(None) # E: Incompatible types in assignment (expression has type "str | None", variable has type "int") [assignment] + + # Missing validator + missing_validator: int = Default(42) # E: No Validator found in field definition [validataclass] + + # A valid field for good measure + valid_field: str = StringValidator() + + @validataclass + class SubClass(BaseClassWithErrors): + # Modify fields + wrong_validator: int = Default(42) # E: Incompatible types in assignment (expression has type "str | int", variable has type "int") [assignment] + missing_validator: int = IntegerValidator() # E: Field cannot be fully parsed because of a prior error in one of the base classes [validataclass] + + # New field (valid) + new_field: str = StringValidator() + + # Initializations + obj1 = SubClass( + valid_field='example', + new_field='example', + ) + obj2 = SubClass( + wrong_validator=1, + missing_validator=2, + valid_field='example', + new_field='example', + ) + +# Valid base class, subclass with errors +- case: validataclass_subclass_with_errors + main: | + from validataclass.dataclasses import validataclass, Default + from validataclass.validators import IntegerValidator, StringValidator + + @validataclass + class Base: + field1: int = IntegerValidator(), Default(42) + field2: int | None = IntegerValidator(), Default(None) + not_a_method: int = IntegerValidator() + + @validataclass + class SubClassWithErrors(Base): + # Incompatibilities with base class + field1: str = StringValidator() # E: Incompatible types in assignment (expression has type "str | int", variable has type "str") [assignment] + field2: str = Default('empty') # E: Incompatible types in assignment (expression has type "int | str", variable has type "str") [assignment] + + # New fields with errors + missing_validator: int = Default(42) # E: No Validator found in field definition [validataclass] + double_validator: int = IntegerValidator(), Default(42), StringValidator() # E: Multiple validator instances found in field definition [validataclass] + double_default: int = IntegerValidator(), Default(42), Default(None) # E: Multiple default objects found in field definition [validataclass] + missing_rhs: int # E: Annotated field without assignment (missing Validator or BaseDefault on right-hand side) [validataclass] + nonsense1: int = 42 # E: Unexpected type "Literal[42]?" in field definition (expected Validator or BaseDefault) [validataclass] + + # Redefine dataclass field as method + def not_a_method(self) -> int: # (line 24, multiple errors, see below) + return 42 + out: | + main:24: error: Dataclass attribute may only be overridden by another attribute [misc] + main:24: error: Signature of "not_a_method" incompatible with supertype "Base" [override] + main:24: note: Superclass: + main:24: note: int + main:24: note: Subclass: + main:24: note: def not_a_method(self) -> int + main:24: error: Method "not_a_method" is not using @override but is overriding a method in class "main.Base" [explicit-override] + +# Validataclass with ValidataclassMixin base class +- case: validataclass_with_validataclass_mixin + main: | + from validataclass.dataclasses import Default, ValidataclassMixin, validataclass + from validataclass.validators import IntegerValidator, StringValidator + + @validataclass + class Base(ValidataclassMixin): + field1: int = IntegerValidator() + field2: str | None = StringValidator(), Default(None) + + @validataclass + class SubClass(Base): + field3: str = StringValidator() + + base = Base( + field1=42, + ) + reveal_type(base) # N: Revealed type is "main.Base" + reveal_type(base.field1) # N: Revealed type is "builtins.int" + reveal_type(base.field2) # N: Revealed type is "builtins.str | None" + reveal_type(base.to_dict()) # N: Revealed type is "builtins.dict[builtins.str, Any]" + + sub = SubClass( + field1=42, + field2=None, + field3='foo', + ) + reveal_type(sub) # N: Revealed type is "main.SubClass" + reveal_type(sub.field1) # N: Revealed type is "builtins.int" + reveal_type(sub.field2) # N: Revealed type is "builtins.str | None" + reveal_type(sub.field3) # N: Revealed type is "builtins.str" + reveal_type(sub.to_dict()) # N: Revealed type is "builtins.dict[builtins.str, Any]" + +# Validataclass with other non-validataclass base classes +- case: validataclass_with_non_validataclass_base + main: | + from validataclass.dataclasses import Default, ValidataclassMixin, validataclass + from validataclass.validators import IntegerValidator, StringValidator + + class Base: + field_validator: IntegerValidator = IntegerValidator() + field_default: Default[None] = Default(None) + + @validataclass + class ActualValidataclass(Base): + field1: int | None = Base.field_validator, Base.field_default + field2: str = Base.field_validator # E: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] + + valid = ActualValidataclass(field1=42) + reveal_type(valid) # N: Revealed type is "main.ActualValidataclass" + reveal_type(valid.field1) # N: Revealed type is "builtins.int | None" + reveal_type(valid.field2) # N: Revealed type is "builtins.str" diff --git a/tests/unit/dataclasses/validataclass_test.py b/tests/unit/dataclasses/validataclass_test.py index 68be274..89fe326 100644 --- a/tests/unit/dataclasses/validataclass_test.py +++ b/tests/unit/dataclasses/validataclass_test.py @@ -191,7 +191,8 @@ class UnitTestDataclass: @staticmethod def test_validataclass_create_objects_invalid(): """ - Create a dataclass using @validataclass and try to instantiate objects from it, but missing a required value. + Create a dataclass using @validataclass and try to instantiate objects from it, but missing a required value or + using positional arguments. """ @validataclass @@ -200,13 +201,19 @@ class UnitTestDataclass: optional_field: int = IntegerValidator(), Default(10) # Try to instantiate without the required field - with pytest.raises(TypeError, match="required keyword-only argument"): + with pytest.raises(TypeError, match="missing 1 required keyword-only argument: 'required_field'"): + # TODO: It would be good if mypy detected the missing required kwargs here too UnitTestDataclass() # Try to instantiate with the optional field, but still lacking the required field - with pytest.raises(TypeError, match="required keyword-only argument"): + with pytest.raises(TypeError, match="missing 1 required keyword-only argument: 'required_field'"): + # TODO: It would be good if mypy detected the missing required kwargs here too UnitTestDataclass(optional_field=42) + # Try to instantiate with positional arguments + with pytest.raises(TypeError, match="takes 1 positional argument but 2 were given"): + UnitTestDataclass(42) # type: ignore[misc] + @staticmethod def test_validataclass_with_mutable_defaults(): """ From 07707643d086f48c890b65103224e2ca82c2c040 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Thu, 5 Mar 2026 15:39:22 +0100 Subject: [PATCH 08/15] validataclass_field: Deprecate raw default values and dataclasses.MISSING --- docs/05-dataclasses.md | 14 +- .../dataclasses/validataclass.py | 2 +- .../dataclasses/validataclass_field.py | 41 ++--- .../validators/dataclass_validator.py | 2 +- .../dataclasses/test_validataclass_field.yml | 29 ++++ .../dataclasses/validataclass_field_test.py | 144 +++++++++--------- tests/unit/dataclasses/validataclass_test.py | 2 +- .../validators/dataclass_validator_test.py | 2 +- 8 files changed, 139 insertions(+), 97 deletions(-) create mode 100644 tests/mypy/dataclasses/test_validataclass_field.yml diff --git a/docs/05-dataclasses.md b/docs/05-dataclasses.md index b5ab029..d1f90d9 100644 --- a/docs/05-dataclasses.md +++ b/docs/05-dataclasses.md @@ -342,8 +342,10 @@ default values for fields. ### Setting defaults with `validataclass_field()` -To set a default value with `validataclass_field()`, you simply specify the `default` parameter. This parameter can be -set either to a special validataclass "default object" (which we will explain in a moment) or directly to a value. +To set a default value with `validataclass_field()`, you need to set the `default` parameter to a "default object". +These are special validataclass objects that will be explained in more detail in a moment. + +Using raw default values instead of default objects has been deprecated in version 0.12.0. **Example:** @@ -355,9 +357,11 @@ from validataclass.validators import IntegerValidator @dataclass class ExampleDataclass: - # The following fields are equivalent - field_a: int = validataclass_field(IntegerValidator(), default=42) # Specify default as direct value - field_b: int = validataclass_field(IntegerValidator(), default=Default(42)) # Specify default using a Default object + # Field with integer default + field1: int = validataclass_field(IntegerValidator(), default=Default(42)) + + # Field that defaults to None + field2: int | None = validataclass_field(IntegerValidator(), default=Default(None)) ``` diff --git a/src/validataclass/dataclasses/validataclass.py b/src/validataclass/dataclasses/validataclass.py index 46a2c47..3bb4d0c 100644 --- a/src/validataclass/dataclasses/validataclass.py +++ b/src/validataclass/dataclasses/validataclass.py @@ -75,7 +75,7 @@ class ExampleDataclass: # (Same as example_field1) example_field3: str = validataclass_field(StringValidator()) # (Same as example_field2) - example_field4: str = validataclass_field(StringValidator(), default='not set') + example_field4: str = validataclass_field(StringValidator(), default=Default('not set')) # Post-init field without validator post_init_field: int = field(init=False, default=0) ``` diff --git a/src/validataclass/dataclasses/validataclass_field.py b/src/validataclass/dataclasses/validataclass_field.py index ffb8089..16bb78a 100644 --- a/src/validataclass/dataclasses/validataclass_field.py +++ b/src/validataclass/dataclasses/validataclass_field.py @@ -5,9 +5,10 @@ """ import dataclasses +import warnings from typing import Any -from typing_extensions import TypeVar, overload +from typing_extensions import TypeVar, deprecated, overload from validataclass.validators import Validator from .defaults import BaseDefault, Default, NoDefault, _NoDefaultType @@ -18,7 +19,6 @@ T_Validated = TypeVar('T_Validated') T_Default = TypeVar('T_Default') -T_RawDefault = TypeVar('T_RawDefault') @overload @@ -32,17 +32,6 @@ def validataclass_field( ... -@overload -def validataclass_field( - validator: Validator[T_Validated], - *, - default: dataclasses._MISSING_TYPE, - metadata: dict[str, Any] | None = None, - **kwargs: Any, -) -> dataclasses.Field[T_Validated]: - ... - - @overload def validataclass_field( validator: Validator[T_Validated], @@ -55,20 +44,21 @@ def validataclass_field( @overload +@deprecated('Use default objects instead of raw defaults or dataclasses.MISSING') def validataclass_field( validator: Validator[T_Validated], *, - default: T_RawDefault, + default: Any, metadata: dict[str, Any] | None = None, **kwargs: Any, -) -> dataclasses.Field[T_Validated | T_RawDefault]: +) -> dataclasses.Field[Any]: ... def validataclass_field( validator: Validator[T_Validated], *, - default: Any = NoDefault, + default: BaseDefault[T_Default] | Any = NoDefault, metadata: dict[str, Any] | None = None, **kwargs: Any, ) -> dataclasses.Field[Any]: @@ -79,15 +69,15 @@ def validataclass_field( Additional keyword arguments will be passed to `dataclasses.field()`, with some exceptions: - - `default` is handled by this function to set metadata. It can be either a direct value or a validataclass default - object, i.e. an object of a subclass of `BaseDefault` (e.g. `Default`, `DefaultFactory`, `NoDefault`). - It is then converted to a direct value (or factory) if necessary and passed to `dataclasses.field()`. + - `default` is handled by this function to set metadata. It must be a validataclass default object (e.g. an instance + of `Default` or `DefaultFactory`, or the special value `NoDefault`). Using raw default values is **deprecated** + and won't be supported in the future. The same applies to the `dataclasses.MISSING` sentinel. - `default_factory` is not allowed. Use `default=DefaultFactory(...)` instead. - `init` is not allowed. To create a non-init field, use `dataclasses.field(init=False)` instead. Parameters: `validator`: Validator to use for validating the field (subclass of `Validator`). - `default`: Default value when the field is missing in the input data (any value or subclass of `BaseDefault`). + `default`: Default value when the field is missing (subclass of `BaseDefault`, defaults to `NoDefault`). `metadata`: Base dictionary for field metadata, gets merged with the metadata generated by this function. `**kwargs`: Additional keyword arguments that are passed to `dataclasses.field()`. """ @@ -108,10 +98,21 @@ def validataclass_field( metadata['validator'] = validator # Ensure default is a validataclass default object (any subclass of BaseDefault) + # TODO: Remove deprecated behaviour for raw defaults and dataclasses.MISSING in a future version. if default is dataclasses.MISSING: + warnings.warn( + 'Using validataclass_field() with `default=dataclasses.MISSING` is deprecated. ' + 'Please use `default=NoDefault` instead.', + DeprecationWarning + ) default = NoDefault elif not isinstance(default, BaseDefault): # Wrap value in a validataclass default object + warnings.warn( + 'Using validataclass_field() with raw default values is deprecated. ' + 'Please use default objects instead (e.g. `default=Default(...)`).', + DeprecationWarning + ) default = Default(default) if default is not NoDefault: diff --git a/src/validataclass/validators/dataclass_validator.py b/src/validataclass/validators/dataclass_validator.py index 1f12891..0d7cca3 100644 --- a/src/validataclass/validators/dataclass_validator.py +++ b/src/validataclass/validators/dataclass_validator.py @@ -56,7 +56,7 @@ class ExampleDataclass: # Equivalent definition using validataclass_field(): # example_field: str = validataclass_field(StringValidator()) - # optional_field: str = validataclass_field(StringValidator(), default='') + # optional_field: str = validataclass_field(StringValidator(), default=Default('')) # "Behind the scenes": Equivalent definition using plain dataclass fields: # example_field: str = field(metadata={'validator': StringValidator()}) diff --git a/tests/mypy/dataclasses/test_validataclass_field.yml b/tests/mypy/dataclasses/test_validataclass_field.yml new file mode 100644 index 0000000..65b7c2e --- /dev/null +++ b/tests/mypy/dataclasses/test_validataclass_field.yml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/typeddjango/pytest-mypy-plugins/master/pytest_mypy_plugins/schema.json + +# Check the inference of the return type of validataclass_field() +- case: validataclass_field_return_type + main: | + from validataclass.dataclasses import Default, NoDefault, validataclass_field + from validataclass.validators import RejectValidator, StringValidator + + reveal_type(validataclass_field(StringValidator())) # N: Revealed type is "dataclasses.Field[builtins.str]" + reveal_type(validataclass_field(StringValidator(), default=NoDefault)) # N: Revealed type is "dataclasses.Field[builtins.str]" + reveal_type(validataclass_field(StringValidator(), default=Default(None))) # N: Revealed type is "dataclasses.Field[builtins.str | None]" + reveal_type(validataclass_field(StringValidator(), default=Default(0), repr=False)) # N: Revealed type is "dataclasses.Field[builtins.str | builtins.int]" + reveal_type(validataclass_field(StringValidator(), default=Default(0), metadata={'a': 1})) # N: Revealed type is "dataclasses.Field[builtins.str | builtins.int]" + + reveal_type(validataclass_field(RejectValidator())) # N: Revealed type is "dataclasses.Field[Never]" + reveal_type(validataclass_field(RejectValidator(), default=Default(None))) # N: Revealed type is "dataclasses.Field[None]" + +# Check that using validataclass_field() with raw default values or dataclasses.MISSING is reported as deprecated. +- case: validataclass_field_with_raw_defaults_deprecated + # Use regex matching here because the error messages are sooo long + regex: true + main: | + import dataclasses + from validataclass.dataclasses import validataclass_field + from validataclass.validators import StringValidator + + validataclass_field(StringValidator(), default='') # E: overload .* is deprecated: Use default objects instead of raw defaults or dataclasses\.MISSING \[deprecated\] + validataclass_field(StringValidator(), default=None) # E: overload .* is deprecated: Use default objects instead of raw defaults or dataclasses\.MISSING \[deprecated\] + validataclass_field(StringValidator(), default=dataclasses.MISSING) # E: overload .* is deprecated: Use default objects instead of raw defaults or dataclasses\.MISSING \[deprecated\] diff --git a/tests/unit/dataclasses/validataclass_field_test.py b/tests/unit/dataclasses/validataclass_field_test.py index 9a85f04..ce36b04 100644 --- a/tests/unit/dataclasses/validataclass_field_test.py +++ b/tests/unit/dataclasses/validataclass_field_test.py @@ -9,7 +9,6 @@ import pytest from typing_extensions import override -from tests.test_utils import UNSET_PARAMETER from tests.unit.dataclasses._helpers import assert_field_default, assert_field_no_default, get_dataclass_fields from validataclass.dataclasses import BaseDefault, Default, DefaultFactory, DefaultUnset, NoDefault, validataclass_field from validataclass.helpers import UnsetValue @@ -20,22 +19,14 @@ class ValidataclassFieldTest: """ Tests for the validataclass_field() helper method. """ @staticmethod - @pytest.mark.parametrize( - 'param_default', - [ - # Parameter is not set at all - UNSET_PARAMETER, - - # Parameter is set explicitly to these sentinel values that mean "no default" - dataclasses.MISSING, - NoDefault, - ], - ) - def test_validataclass_field_without_default(param_default): + @pytest.mark.parametrize('explicit_no_default', [True, False]) + def test_validataclass_field_without_default(explicit_no_default): """ Test validataclass_field function on its own, without a default value (implicitly and explicitly). """ # Create field - params = {} if param_default is UNSET_PARAMETER else {'default': param_default} - field = validataclass_field(IntegerValidator(), **params) + if explicit_no_default: + field = validataclass_field(IntegerValidator(), default=NoDefault) + else: + field = validataclass_field(IntegerValidator()) # Check field metadata assert type(field.metadata.get('validator')) is IntegerValidator @@ -47,51 +38,23 @@ def test_validataclass_field_without_default(param_default): @staticmethod @pytest.mark.parametrize( - 'param_default, expected_default', + 'param_default, expected_default, needs_factory', [ - # Explicit Default objects - (Default(42), 42), - (Default(None), None), - (Default(UnsetValue), UnsetValue), - (DefaultUnset, UnsetValue), - - # Regular values (automatically converted to Default objects) - (42, 42), - (None, None), - (UnsetValue, UnsetValue), - ], - ) - def test_validataclass_field_with_default(param_default, expected_default): - """ Test validataclass_field function on its own, with various static default values. """ - # Create field - field = validataclass_field(IntegerValidator(), default=param_default) - - # Check field metadata - metadata_validator = field.metadata.get('validator') - metadata_default = field.metadata.get('validator_default') - - assert type(metadata_validator) is IntegerValidator - assert type(metadata_default) is Default - assert metadata_default.get_value() == expected_default - assert metadata_default.needs_factory() is False - - # Check field default and default_factory - assert field.default == expected_default - assert field.default_factory is dataclasses.MISSING + # Static default values that don't need a factory + (Default(42), 42, False), + (Default(None), None, False), + (Default(UnsetValue), UnsetValue, False), + (DefaultUnset, UnsetValue, False), - @staticmethod - @pytest.mark.parametrize( - 'param_default, expected_default, expected_default_cls', - [ - # Default object with mutable value (should result in a default_factory) - (Default([]), [], Default), + # Default object with mutable value (should result in a default factory) + (Default([]), [], True), # DefaultFactory object - (DefaultFactory(lambda: 3), 3, DefaultFactory), + (DefaultFactory(lambda: 3), 3, True), ], ) - def test_validataclass_field_with_default_factory(param_default, expected_default, expected_default_cls): - """ Test validataclass_field function on its own, with default objects that require a default_factory. """ + def test_validataclass_field_with_default(param_default, expected_default, needs_factory): + """ Test validataclass_field function on its own, with various static default values and default factories. """ # Create field field = validataclass_field(IntegerValidator(), default=param_default) @@ -100,14 +63,19 @@ def test_validataclass_field_with_default_factory(param_default, expected_defaul metadata_default = field.metadata.get('validator_default') assert type(metadata_validator) is IntegerValidator - assert type(metadata_default) is expected_default_cls + assert metadata_default is not None + assert type(metadata_default) is type(param_default) assert metadata_default.get_value() == expected_default - assert metadata_default.needs_factory() is True + assert metadata_default.needs_factory() == needs_factory # Check field default and default_factory - assert field.default is dataclasses.MISSING - assert field.default_factory is not dataclasses.MISSING - assert field.default_factory() == expected_default + if needs_factory: + assert field.default is dataclasses.MISSING + assert callable(field.default_factory) + assert field.default_factory() == expected_default + else: + assert field.default == expected_default + assert field.default_factory is dataclasses.MISSING @staticmethod def test_validataclass_field_with_custom_default_class(): @@ -174,24 +142,29 @@ def test_validataclass_fields_in_dataclass(): @dataclasses.dataclass class UnitTestDataclass: - foo: int = validataclass_field(IntegerValidator()) - bar: int = validataclass_field(IntegerValidator(), default=Default(1)) - baz: int = validataclass_field(IntegerValidator(), default=42) + no_default_implicit: int = validataclass_field(IntegerValidator()) + no_default_explicit: int = validataclass_field(IntegerValidator(), default=NoDefault) + default_int: int = validataclass_field(IntegerValidator(), default=Default(1)) + default_none: int | None = validataclass_field(IntegerValidator(), default=Default(None)) # Get fields from dataclass fields = get_dataclass_fields(UnitTestDataclass) - # Check names and types of all fields - assert list(fields.keys()) == ['foo', 'bar', 'baz'] - assert all(f.type is int for f in fields.values()) + # Check types of all fields + assert len(fields.keys()) == 4 + assert fields['no_default_implicit'].type is int + assert fields['no_default_explicit'].type is int + assert fields['default_int'].type is int + assert fields['default_none'].type == int | None # Check that all fields have an IntegerValidator object as validator assert all(type(f.metadata.get('validator')) is IntegerValidator for f in fields.values()) # Check field defaults - assert_field_no_default(fields['foo']) - assert_field_default(fields['bar'], default_value=1) - assert_field_default(fields['baz'], default_value=42) + assert_field_no_default(fields['no_default_implicit']) + assert_field_no_default(fields['no_default_explicit']) + assert_field_default(fields['default_int'], default_value=1) + assert_field_default(fields['default_none'], default_value=None) @staticmethod def test_validataclass_field_with_init_kwarg_raises_exception(): @@ -212,3 +185,38 @@ def test_validataclass_field_with_default_factory_kwarg_raises_exception(): == 'Keyword argument "default_factory" is not allowed in validator field (use default=DefaultFactory(...) ' 'instead).' ) + + @staticmethod + @pytest.mark.parametrize('param_default', [42, '', None]) + def test_validataclass_field_with_raw_defaults_is_deprecated(param_default: int | str | None) -> None: + """ Test that validataclass_field() with raw default values is deprecated. """ + # Create field and test that it generates a deprecation warning + with pytest.deprecated_call(match='Please use default objects instead'): + field = validataclass_field(IntegerValidator(), default=param_default) # type: ignore[deprecated] + + # Check field metadata + metadata_validator = field.metadata.get('validator') + metadata_default = field.metadata.get('validator_default') + + assert type(metadata_validator) is IntegerValidator + assert type(metadata_default) is Default + assert metadata_default.get_value() == param_default + + # Check field default and default_factory + assert field.default == param_default + assert field.default_factory is dataclasses.MISSING + + @staticmethod + def test_validataclass_field_with_dataclasses_missing_is_deprecated(): + """ Test that validataclass_field() with `default=dataclasses.MISSING` is deprecated. """ + # Create field and test that it generates a deprecation warning + with pytest.deprecated_call(match='Please use `default=NoDefault` instead'): + field = validataclass_field(IntegerValidator(), default=dataclasses.MISSING) # type: ignore[deprecated] + + # Check field metadata + assert type(field.metadata.get('validator')) is IntegerValidator + assert 'validator_default' not in field.metadata + + # Check field default + assert field.default is dataclasses.MISSING + assert field.default_factory is dataclasses.MISSING diff --git a/tests/unit/dataclasses/validataclass_test.py b/tests/unit/dataclasses/validataclass_test.py index 89fe326..1ee0cca 100644 --- a/tests/unit/dataclasses/validataclass_test.py +++ b/tests/unit/dataclasses/validataclass_test.py @@ -50,7 +50,7 @@ def test_validataclass_without_kwargs(): class UnitTestValidatorDataclass: foo: int = IntegerValidator() bar: int = validataclass_field(IntegerValidator(), default=Default(0)) - baz: str | None = validataclass_field(Noneable(StringValidator()), default=None) + baz: str | None = validataclass_field(Noneable(StringValidator()), default=Default(None)) # Check that @validataclass actually created a dataclass (i.e. used @dataclass on the class) assert dataclasses.is_dataclass(UnitTestValidatorDataclass) diff --git a/tests/unit/validators/dataclass_validator_test.py b/tests/unit/validators/dataclass_validator_test.py index 16d9964..a510a1d 100644 --- a/tests/unit/validators/dataclass_validator_test.py +++ b/tests/unit/validators/dataclass_validator_test.py @@ -56,7 +56,7 @@ class UnitTestNestedDataclass: name: str = StringValidator() test_fruit: UnitTestDataclass = DataclassValidator(UnitTestDataclass) test_vegetable: UnitTestDataclass | None = \ - validataclass_field(DataclassValidator(UnitTestDataclass), default=None) + validataclass_field(DataclassValidator(UnitTestDataclass), default=Default(None)) # Dataclass with non-init field and __post_init__() method From 6b5cbf917f0199c96b01ad7b956ced21590228d0 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Thu, 5 Mar 2026 20:31:54 +0100 Subject: [PATCH 09/15] mypy plugin: Handle validataclass_field() and dataclasses.field() --- .../dataclasses/validataclass_field.py | 20 ++-- .../mypy/plugin/validataclass_transformer.py | 26 ++--- .../mypy/plugin/virtual_field_resolver.py | 100 +++++++++++++++++- tests/mypy/dataclasses/test_validataclass.yml | 90 ++++++++++++++++ .../dataclasses/test_validataclass_field.yml | 19 ++-- .../test_validataclass_subclasses.yml | 53 ++++++++++ .../dataclasses/validataclass_field_test.py | 38 +++++-- tests/unit/dataclasses/validataclass_test.py | 4 +- 8 files changed, 300 insertions(+), 50 deletions(-) diff --git a/src/validataclass/dataclasses/validataclass_field.py b/src/validataclass/dataclasses/validataclass_field.py index 16bb78a..48e2ada 100644 --- a/src/validataclass/dataclasses/validataclass_field.py +++ b/src/validataclass/dataclasses/validataclass_field.py @@ -21,6 +21,11 @@ T_Default = TypeVar('T_Default') +# NOTE: Actual return type of this function is `Field[T_Validated | T_Default]`. However, we will pretend here that it +# returns `T_Validated | T_Default` instead, to help type checkers understand what happens within the context of a +# dataclass. This is the same thing that typeshed does. +# (Using this function outside the context of a dataclass usually doesn't make a lot of sense.) + @overload def validataclass_field( validator: Validator[T_Validated], @@ -28,7 +33,7 @@ def validataclass_field( default: _NoDefaultType = NoDefault, metadata: dict[str, Any] | None = None, **kwargs: Any, -) -> dataclasses.Field[T_Validated]: +) -> T_Validated: ... @@ -39,7 +44,7 @@ def validataclass_field( default: BaseDefault[T_Default], metadata: dict[str, Any] | None = None, **kwargs: Any, -) -> dataclasses.Field[T_Validated | T_Default]: +) -> T_Validated | T_Default: ... @@ -51,7 +56,7 @@ def validataclass_field( default: Any, metadata: dict[str, Any] | None = None, **kwargs: Any, -) -> dataclasses.Field[Any]: +) -> Any: ... @@ -61,7 +66,7 @@ def validataclass_field( default: BaseDefault[T_Default] | Any = NoDefault, metadata: dict[str, Any] | None = None, **kwargs: Any, -) -> dataclasses.Field[Any]: +) -> Any: """ Defines a dataclass field compatible with DataclassValidator. @@ -87,10 +92,10 @@ def validataclass_field( # Check for incompatible keyword arguments if 'init' in kwargs: - raise ValueError('Keyword argument "init" is not allowed in validator field.') + raise ValueError('Keyword argument "init" is not allowed in validataclass_field.') if 'default_factory' in kwargs: raise ValueError( - 'Keyword argument "default_factory" is not allowed in validator field (use default=DefaultFactory(...) ' + 'Keyword argument "default_factory" is not allowed in validataclass_field (use default=DefaultFactory(...) ' 'instead).' ) @@ -126,5 +131,4 @@ def validataclass_field( kwargs['default'] = default.get_value() # Create a dataclass field with our metadata - # NOTE: typeshed types the field() function as 'T' rather than 'Field[T]', but the actual return type is a Field. - return dataclasses.field(metadata=metadata, **kwargs) # type: ignore[no-any-return] + return dataclasses.field(metadata=metadata, **kwargs) diff --git a/src/validataclass/mypy/plugin/validataclass_transformer.py b/src/validataclass/mypy/plugin/validataclass_transformer.py index 9e934f4..fa49292 100644 --- a/src/validataclass/mypy/plugin/validataclass_transformer.py +++ b/src/validataclass/mypy/plugin/validataclass_transformer.py @@ -20,6 +20,7 @@ GDEF, ListExpr, NameExpr, + RefExpr, StrExpr, TempNode, Var, @@ -33,7 +34,6 @@ from .constants import ( ERROR_CODE_VALIDATACLASS, ERROR_CODE_VALIDATACLASS_NOT_IMPLEMENTED, - VALIDATACLASS_FIELD_FUNC, VIRTUAL_FIELD_WRAPPER_FUNC, VIRTUAL_FIELD_WRAPPER_FUNC_NAME, ) @@ -308,29 +308,21 @@ def _collect_fields(self) -> list[ValidataclassField] | None: ) continue - # Handle fields that are already wrapped or created with field specifier functions - if isinstance(stmt.rvalue, CallExpr) and isinstance(stmt.rvalue.callee, NameExpr): + # Handle fields that are already wrapped or created with special functions. + # (Fields created with validataclass_field() are wrapped normally.) + if isinstance(stmt.rvalue, CallExpr) and isinstance(stmt.rvalue.callee, RefExpr): callee_name = stmt.rvalue.callee.fullname + # Skip field if it has already been wrapped in a previous pass (this probably shouldn't happen though) + if callee_name == VIRTUAL_FIELD_WRAPPER_FUNC: + self._log_debug(stmt, f'Skip already wrapped field "{lvalue.name}"') + continue + # Ignore fields that have been created with the regular dataclasses.field() or similar if callee_name in DATACLASS_FIELD_SPECIFIERS: - # TODO: Handle these properly. self._log_debug(stmt, f'Skip field "{lvalue.name}" with {callee_name}') continue - # Skip fields that have been created with validataclass_field() - # TODO: I think we need to wrap these too actually :/ - if callee_name == VALIDATACLASS_FIELD_FUNC: - # TODO: Normalize `default` - if it's NoDefault, remove the default instead? - # TODO: (Can probably be ignored because we'll have to fix __init__ manually anyway.) - self._log_debug(stmt, f'Skip field "{lvalue.name}" with validataclass_field()') - continue - - # Skip field if it has already been wrapped in a previous pass - if callee_name == VIRTUAL_FIELD_WRAPPER_FUNC: - self._log_debug(stmt, f'Skip already wrapped field "{lvalue.name}"') - continue - # If we're still here, we might actually have a validataclass field definition that we care about! if lvalue.name not in current_fields: current_fields[lvalue.name] = ValidataclassField(lvalue.name) diff --git a/src/validataclass/mypy/plugin/virtual_field_resolver.py b/src/validataclass/mypy/plugin/virtual_field_resolver.py index 0a48039..fcdaa3e 100644 --- a/src/validataclass/mypy/plugin/virtual_field_resolver.py +++ b/src/validataclass/mypy/plugin/virtual_field_resolver.py @@ -7,7 +7,7 @@ from typing import Any, cast from mypy.errorcodes import ErrorCode -from mypy.nodes import Context, Expression, ListExpr, MemberExpr, StrExpr, TempNode +from mypy.nodes import ArgKind, CallExpr, Context, Expression, ListExpr, MemberExpr, StrExpr, TempNode, RefExpr from mypy.plugin import CheckerPluginInterface, FunctionContext from mypy.typeops import make_simplified_union from mypy.types import ( @@ -27,6 +27,7 @@ ERROR_CODE_VALIDATACLASS, ERROR_CODE_VALIDATACLASS_EMPTY_TYPE, FIELD_DEFAULT_BASE_CLASS, + VALIDATACLASS_FIELD_FUNC, VALIDATOR_BASE_CLASS, ) from .debug_logger import DebugLogger @@ -181,6 +182,15 @@ def _parse_field_definition( (by retrieving the ParsedValidataclasFields from the parsed field cache), then analyzing the right-hand side expression of the current field assignment and merging the results. """ + # First of all, check if the field was created explicitly using validataclass_field(), i.e. the right-hand side + # is a call to that function. Since this functions are not dependent on base classes (they always replace the + # entire field definition), we can shortcut the analysis here. + # (Fields created with dataclasses.field() are skipped by the ValidataclassTransformer.) + if isinstance(field_rhs_expr, CallExpr) and isinstance(field_rhs_expr.callee, RefExpr): + # Handle fields created explicitly using validataclass_field() + if field_rhs_expr.callee.fullname == VALIDATACLASS_FIELD_FUNC: + return self._parse_validataclass_field_callexpr(field_rhs_expr) + # This will hold the end result that's returned at the end of the function fully_parsed_field = ParsedValidataclassField() @@ -224,8 +234,6 @@ def _parse_rhs_expression(self, rhs_expr: Expression) -> ParsedValidataclassFiel # Get type of field assignment RHS rhs_type = get_proper_type(self._api.get_expression_type(rhs_expr)) - # TODO: Handle validataclass_field() calls properly. - parsed_field = ParsedValidataclassField() if isinstance(rhs_type, TupleType): @@ -259,7 +267,7 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel self._fail('Multiple validator instances found in field definition') return - # Find out the output type of this validator + # Store type of validator for later self._log_debug(f' - Validator: {item_type}') parsed_field.validator_type = item_type return @@ -272,7 +280,7 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel self._fail('Multiple default objects found in field definition') return - # Find out the value type of this default object + # Store type of default object for later self._log_debug(f' - Default object: {item_type}') parsed_field.default_type = item_type return @@ -299,6 +307,88 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel parsed_field.error = True self._fail(f'Unexpected type "{item_type_str}" in field definition (expected Validator or BaseDefault)') + def _parse_validataclass_field_callexpr(self, call_expr: CallExpr) -> ParsedValidataclassField: + """ + Analyze a `validataclass_field()` call expression to find validator and default objects. + """ + parsed_field = ParsedValidataclassField() + + # Iterate over call arguments, differentiate positional and named arguments, find validator and default + for i in range(len(call_expr.args)): + # Get argument expression, kind (positional/named) and name (None for positional arguments) + arg_expr = call_expr.args[i] + arg_kind = call_expr.arg_kinds[i] + arg_name = call_expr.arg_names[i] + + # Get type of argument expression + arg_type = get_proper_type(self._api.get_expression_type(arg_expr)) + + # NOTE: We can ignore a lot of possible errors here that are already covered by mypy itself, e.g. too many + # arguments, duplicate arguments, incorrect argument types, etc. We still have to check for these errors + # (because it means the call is invalid and can't be parsed correctly), but we don't need to report them. + + # The function has only a single positional argument, which is the validator. We can also assume that all + # positional arguments come first (otherwise it's a syntax error and we don't even reach this code). + if (arg_kind == ArgKind.ARG_POS and i == 0) or arg_name == 'validator': + # There can only be one validator (mypy will report an error for this) + if parsed_field.validator_type is not None: + parsed_field.error = True + break + + # Ensure that argument is a Validator instance (mypy will report an error otherwise) + if not isinstance(arg_type, Instance) or not arg_type.type.has_base(VALIDATOR_BASE_CLASS): + parsed_field.error = True + break + + # Store type of validator for later + parsed_field.validator_type = arg_type + continue + + # Check for named arguments (unknown arguments can be ignored here) + match arg_name: + # Check for field default argument + case 'default': + # There can only be one default (mypy will report an error for this) + if parsed_field.default_type is not None: + parsed_field.error = True + break + + # Ensure that argument is a BaseDefault instance (mypy will report an error otherwise) + # NOTE: Using raw default values and dataclasses.MISSING here has been deprecated, so we don't need to + # support them. mypy will report a deprecation warning, we can just treat it as an error and return Any. + if not isinstance(arg_type, Instance) or not arg_type.type.has_base(FIELD_DEFAULT_BASE_CLASS): + parsed_field.error = True + break + + # Store type of default object for later + parsed_field.default_type = arg_type + + # Validataclass fields are required to be init fields, so the argument isn't allowed + case 'init': + # Report error but continue parsing, because this argument wouldn't influence the field type + self._fail('Keyword argument "init" is not allowed in validataclass_field') + + # Validataclass fields should use a DefaultFactory instead of the default_factory argument + case 'default_factory': + # Report error and stop parsing, because the argument related to the field default + self._fail( + 'Keyword argument "default_factory" is not allowed in validataclass_field; use ' + 'default=DefaultFactory(...) instead' + ) + parsed_field.error = True + break + + # Stop checking if there were errors parsing the arguments + if parsed_field.error: + return parsed_field + + # Ensure that the field has a validator + if parsed_field.validator_type is None: + parsed_field.error = True + self._fail('No validator found in validataclass_field() call') + + return parsed_field + def _resolve_field_type(self, parsed_field: ParsedValidataclassField) -> ProperType: """ Resolve the type of the parsed field by determining the result type of the validator and default objects and diff --git a/tests/mypy/dataclasses/test_validataclass.yml b/tests/mypy/dataclasses/test_validataclass.yml index d203a49..a0cf078 100644 --- a/tests/mypy/dataclasses/test_validataclass.yml +++ b/tests/mypy/dataclasses/test_validataclass.yml @@ -325,3 +325,93 @@ @staticmethod def some_static_method() -> None: pass + +# Validataclasses with fields created using validataclass_field() or dataclasses.field() +- case: validataclass_with_explicit_field_function + main: | + import dataclasses + from validataclass.dataclasses import Default, NoDefault, validataclass, validataclass_field + from validataclass.validators import IntegerValidator, StringValidator + + @validataclass + class GoodValidataclass: + # Fields created with validataclass_field + field1: int = validataclass_field(IntegerValidator()) + field2: int = validataclass_field(IntegerValidator(), default=NoDefault) + field3: int | str = validataclass_field(IntegerValidator(), default=Default('')) + field4: int | None = validataclass_field(IntegerValidator(), default=Default(None), metadata={'foo': 1}) + field5: int | str = validataclass_field(default=Default(''), validator=IntegerValidator(), repr=False) + + # Fields created with dataclasses.field (these are ignored by the plugin, they're annotated as returning the + # type of their default value - which is good enough here) + non_init1: int = dataclasses.field(init=False) + non_init2: int = dataclasses.field(init=False, default=42) + very_manual_field: int = dataclasses.field(default=42, metadata={'validator': IntegerValidator()}) + + @validataclass + class BadValidataclass: + # Fields created with validataclass_field with incompatible field types + invalid_type1: int = validataclass_field(StringValidator()) # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] + invalid_type2: int = validataclass_field(IntegerValidator(), default=Default('')) # E: Incompatible types in assignment (expression has type "int | str", variable has type "int") [assignment] + + # Unsupported arguments for validataclass_field + invalid_init: int = validataclass_field(IntegerValidator(), init=False) # E: Keyword argument "init" is not allowed in validataclass_field [validataclass] + invalid_default_factory: int = validataclass_field(IntegerValidator(), default_factory=lambda: 42) # E: Keyword argument "default_factory" is not allowed in validataclass_field; use default=DefaultFactory(...) instead [validataclass] + + # Fields created with dataclasses.field (field type is default type) + non_init_invalid_type: str = dataclasses.field(init=False, default=42) # E: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] + very_manual_field: str = dataclasses.field(default=42, metadata={'validator': IntegerValidator()}) # E: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] + +# Validataclasses using validataclass_fields() with invalid arguments (types, number of arguments, duplicate arguments) +- case: validataclass_with_explicit_field_function_invalid_args + main: | + from validataclass.dataclasses import Default, validataclass, validataclass_field + from validataclass.validators import IntegerValidator + + @validataclass + class BadValidataclass: + # Missing required argument (validator) + invalid_args1: int = validataclass_field() # (line 7, multiple errors) + + # Only positional argument should be a validator + invalid_args2: int = validataclass_field(Default(0)) # (line 10, multiple errors) + + # Too many positional arguments + invalid_args3: int = validataclass_field(IntegerValidator(), Default(0)) # (line 13, multiple errors) + + # Duplicate arguments + invalid_args4: int = validataclass_field(IntegerValidator(), validator=IntegerValidator()) # (line 16, multiple errors) + invalid_args5: int = validataclass_field(IntegerValidator(), default=Default(''), default=Default(0)) # (line 17, multiple errors) + + # Incorrect type for "default" argument (raw default values are deprecated) + invalid_args6: int = validataclass_field(IntegerValidator(), default=42) # (line 20, multiple errors) + # Use regex matching here because mypy is very verbose about argument mismatches for overloaded functions + regex: true + out: | + main:7: error: All overload variants of "validataclass_field" require at least one argument \[call-overload\] + main:7: note: Possible overload variants: + main:7: note: def .* + main:7: note: def .* + main:7: note: def .* + main:7: error: No validator found in validataclass_field\(\) call \[validataclass\] + main:10: error: No overload variant of "validataclass_field" matches argument type "Default\[int\]" \[call-overload\] + main:10: note: Possible overload variants: + main:10: note: def .* + main:10: note: def .* + main:10: note: def .* + main:13: error: No overload variant of "validataclass_field" matches argument types "IntegerValidator", "Default\[int\]" \[call-overload\] + main:13: note: Possible overload variants: + main:13: note: def .* + main:13: note: def .* + main:13: note: def .* + main:16: error: No overload variant of "validataclass_field" matches argument types "IntegerValidator", "IntegerValidator" \[call-overload\] + main:16: note: Possible overload variants: + main:16: note: def .* + main:16: note: def .* + main:16: note: def .* + main:17: error: No overload variant of "validataclass_field" matches argument types "IntegerValidator", "Default\[str\]", "Default\[int\]" \[call-overload\] + main:17: note: Possible overload variants: + main:17: note: def .* + main:17: note: def .* + main:17: note: def .* + main:20: error: overload .* is deprecated: Use default objects instead of raw defaults or dataclasses\.MISSING \[deprecated\] diff --git a/tests/mypy/dataclasses/test_validataclass_field.yml b/tests/mypy/dataclasses/test_validataclass_field.yml index 65b7c2e..19b0ff4 100644 --- a/tests/mypy/dataclasses/test_validataclass_field.yml +++ b/tests/mypy/dataclasses/test_validataclass_field.yml @@ -1,19 +1,24 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/typeddjango/pytest-mypy-plugins/master/pytest_mypy_plugins/schema.json # Check the inference of the return type of validataclass_field() +# NOTE: The annotated return type of this function is just T rather than dataclasses.Field[T], for reasons (see code). - case: validataclass_field_return_type main: | from validataclass.dataclasses import Default, NoDefault, validataclass_field from validataclass.validators import RejectValidator, StringValidator - reveal_type(validataclass_field(StringValidator())) # N: Revealed type is "dataclasses.Field[builtins.str]" - reveal_type(validataclass_field(StringValidator(), default=NoDefault)) # N: Revealed type is "dataclasses.Field[builtins.str]" - reveal_type(validataclass_field(StringValidator(), default=Default(None))) # N: Revealed type is "dataclasses.Field[builtins.str | None]" - reveal_type(validataclass_field(StringValidator(), default=Default(0), repr=False)) # N: Revealed type is "dataclasses.Field[builtins.str | builtins.int]" - reveal_type(validataclass_field(StringValidator(), default=Default(0), metadata={'a': 1})) # N: Revealed type is "dataclasses.Field[builtins.str | builtins.int]" + reveal_type(validataclass_field(StringValidator())) # N: Revealed type is "builtins.str" + reveal_type(validataclass_field(StringValidator(), default=NoDefault)) # N: Revealed type is "builtins.str" + reveal_type(validataclass_field(StringValidator(), default=Default(None))) # N: Revealed type is "builtins.str | None" + reveal_type(validataclass_field(StringValidator(), default=Default(0), repr=False)) # N: Revealed type is "builtins.str | builtins.int" + reveal_type(validataclass_field(StringValidator(), default=Default(0), metadata={'a': 1})) # N: Revealed type is "builtins.str | builtins.int" - reveal_type(validataclass_field(RejectValidator())) # N: Revealed type is "dataclasses.Field[Never]" - reveal_type(validataclass_field(RejectValidator(), default=Default(None))) # N: Revealed type is "dataclasses.Field[None]" + # Validator with Never type (but the result is not Never) + reveal_type(validataclass_field(RejectValidator(), default=Default(None))) # N: Revealed type is "None" + + # Resulting type is Never (this needs to be the last line or wrapped in a try-except block because mypy considers + # all code after it as unreachable) + reveal_type(validataclass_field(RejectValidator())) # N: Revealed type is "Never" # Check that using validataclass_field() with raw default values or dataclasses.MISSING is reported as deprecated. - case: validataclass_field_with_raw_defaults_deprecated diff --git a/tests/mypy/dataclasses/test_validataclass_subclasses.yml b/tests/mypy/dataclasses/test_validataclass_subclasses.yml index 3039cd9..2e4403e 100644 --- a/tests/mypy/dataclasses/test_validataclass_subclasses.yml +++ b/tests/mypy/dataclasses/test_validataclass_subclasses.yml @@ -500,3 +500,56 @@ reveal_type(valid) # N: Revealed type is "main.ActualValidataclass" reveal_type(valid.field1) # N: Revealed type is "builtins.int | None" reveal_type(valid.field2) # N: Revealed type is "builtins.str" + +# Validataclasses and subclasses with fields created using validataclass_field() or dataclasses.field() +- case: validataclass_subclasses_with_explicit_field_function + main: | + import dataclasses + from validataclass.dataclasses import Default, validataclass, validataclass_field + from validataclass.validators import IntegerValidator, StringValidator + + @validataclass + class Base: + # Fields created explicitly with validataclass_field + explicit1: int = validataclass_field(IntegerValidator()) + explicit2: int | None = validataclass_field(IntegerValidator(), default=Default(None)) + explicit3: int = validataclass_field(IntegerValidator()) + explicit4: int | None = validataclass_field(IntegerValidator(), default=Default(None)) + + # Fields created with tuples + tuple1: int = IntegerValidator() + tuple2: int | None = IntegerValidator(), Default(None) + + @validataclass + class GoodSubClass(Base): + # Override fields created with validataclass_field using validataclass_field + # (Should override the default even when not specified) + explicit1: str | None = validataclass_field(StringValidator(), default=Default(None)) + explicit2: str = validataclass_field(StringValidator()) + + # Override fields created with validataclass_field using tuples + # (Should take base class into account) + explicit3: int | None = Default(None) + explicit4: str | None = StringValidator() + + # Override fields created with tuples using validataclass_field + # (Should override the default even when not specified) + tuple1: str | None = validataclass_field(StringValidator(), default=Default(None)) + tuple2: str = validataclass_field(StringValidator()) + + @validataclass + class BadSubClass(Base): + # Override fields created with validataclass_field using validataclass_field + # (Should override the default even when not specified) + explicit1: int = validataclass_field(StringValidator(), default=Default(None)) # E: Incompatible types in assignment (expression has type "str | None", variable has type "int") [assignment] + explicit2: int = validataclass_field(StringValidator()) # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] + + # Override fields created with validataclass_field using tuples + # (Should take base class into account) + explicit3: None = Default(None) # E: Incompatible types in assignment (expression has type "int | None", variable has type "None") [assignment] + explicit4: str = StringValidator() # E: Incompatible types in assignment (expression has type "str | None", variable has type "str") [assignment] + + # Override fields created with tuples using validataclass_field + # (Should override the default even when not specified) + tuple1: str = validataclass_field(StringValidator(), default=Default(None)) # E: Incompatible types in assignment (expression has type "str | None", variable has type "str") [assignment] + tuple2: int = validataclass_field(StringValidator()) # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] diff --git a/tests/unit/dataclasses/validataclass_field_test.py b/tests/unit/dataclasses/validataclass_field_test.py index ce36b04..7b6fd3a 100644 --- a/tests/unit/dataclasses/validataclass_field_test.py +++ b/tests/unit/dataclasses/validataclass_field_test.py @@ -5,6 +5,7 @@ """ import dataclasses +from typing import Any import pytest from typing_extensions import override @@ -12,7 +13,22 @@ from tests.unit.dataclasses._helpers import assert_field_default, assert_field_no_default, get_dataclass_fields from validataclass.dataclasses import BaseDefault, Default, DefaultFactory, DefaultUnset, NoDefault, validataclass_field from validataclass.helpers import UnsetValue -from validataclass.validators import IntegerValidator +from validataclass.validators import IntegerValidator, Validator + + +# Here we are testing the validataclass_field() function on its own. It usually doesn't make sense to use this function +# outside the context of a dataclass, which is why the annotated return type doesn't match what it actually returns +# (see note in validataclass_field module). +# However, for testing, we need the actual return type (otherwise type checkers will be very unhappy about this file), +# so we're actually using a wrapper here that fixes the type. I know it's ugly, but it's just for testing. +# (We'll keep the typing as simple as possible and skip the overloads, though.) +def typed_validataclass_field( + validator: Validator[Any], + *, + default: Any = NoDefault, + **kwargs: Any, +) -> dataclasses.Field[Any]: + return validataclass_field(validator, default=default, **kwargs) # type: ignore[no-any-return] class ValidataclassFieldTest: @@ -24,9 +40,9 @@ def test_validataclass_field_without_default(explicit_no_default): """ Test validataclass_field function on its own, without a default value (implicitly and explicitly). """ # Create field if explicit_no_default: - field = validataclass_field(IntegerValidator(), default=NoDefault) + field = typed_validataclass_field(IntegerValidator(), default=NoDefault) else: - field = validataclass_field(IntegerValidator()) + field = typed_validataclass_field(IntegerValidator()) # Check field metadata assert type(field.metadata.get('validator')) is IntegerValidator @@ -56,7 +72,7 @@ def test_validataclass_field_without_default(explicit_no_default): def test_validataclass_field_with_default(param_default, expected_default, needs_factory): """ Test validataclass_field function on its own, with various static default values and default factories. """ # Create field - field = validataclass_field(IntegerValidator(), default=param_default) + field = typed_validataclass_field(IntegerValidator(), default=param_default) # Check field metadata metadata_validator = field.metadata.get('validator') @@ -95,7 +111,7 @@ def needs_factory(self) -> bool: return True # Create field - field = validataclass_field(IntegerValidator(), default=CustomDefault()) + field = typed_validataclass_field(IntegerValidator(), default=CustomDefault()) # Check field metadata metadata_validator = field.metadata.get('validator') @@ -117,7 +133,7 @@ def needs_factory(self) -> bool: def test_validataclass_field_with_metadata(): """ Test validataclass_field function on its own, with custom metadata. """ # Create field with custom metadata (validataclass metadata will be overwritten by the function) - field = validataclass_field( + field = typed_validataclass_field( IntegerValidator(), default=Default(42), metadata={ @@ -172,7 +188,7 @@ def test_validataclass_field_with_init_kwarg_raises_exception(): with pytest.raises(ValueError) as exception_info: validataclass_field(IntegerValidator(), init=False) - assert str(exception_info.value) == 'Keyword argument "init" is not allowed in validator field.' + assert str(exception_info.value) == 'Keyword argument "init" is not allowed in validataclass_field.' @staticmethod def test_validataclass_field_with_default_factory_kwarg_raises_exception(): @@ -182,8 +198,8 @@ def test_validataclass_field_with_default_factory_kwarg_raises_exception(): assert ( str(exception_info.value) - == 'Keyword argument "default_factory" is not allowed in validator field (use default=DefaultFactory(...) ' - 'instead).' + == 'Keyword argument "default_factory" is not allowed in validataclass_field (use ' + 'default=DefaultFactory(...) instead).' ) @staticmethod @@ -192,7 +208,7 @@ def test_validataclass_field_with_raw_defaults_is_deprecated(param_default: int """ Test that validataclass_field() with raw default values is deprecated. """ # Create field and test that it generates a deprecation warning with pytest.deprecated_call(match='Please use default objects instead'): - field = validataclass_field(IntegerValidator(), default=param_default) # type: ignore[deprecated] + field = typed_validataclass_field(IntegerValidator(), default=param_default) # Check field metadata metadata_validator = field.metadata.get('validator') @@ -211,7 +227,7 @@ def test_validataclass_field_with_dataclasses_missing_is_deprecated(): """ Test that validataclass_field() with `default=dataclasses.MISSING` is deprecated. """ # Create field and test that it generates a deprecation warning with pytest.deprecated_call(match='Please use `default=NoDefault` instead'): - field = validataclass_field(IntegerValidator(), default=dataclasses.MISSING) # type: ignore[deprecated] + field = typed_validataclass_field(IntegerValidator(), default=dataclasses.MISSING) # Check field metadata assert type(field.metadata.get('validator')) is IntegerValidator diff --git a/tests/unit/dataclasses/validataclass_test.py b/tests/unit/dataclasses/validataclass_test.py index 1ee0cca..66dcf49 100644 --- a/tests/unit/dataclasses/validataclass_test.py +++ b/tests/unit/dataclasses/validataclass_test.py @@ -138,7 +138,7 @@ class UnitTestValidatorDataclass: validated: int = IntegerValidator() # Non-init field - non_init: int = dataclasses.field(init=False, default=1) # type: ignore[validataclass] # TODO: Fix me! + non_init: int = dataclasses.field(init=False, default=1) # Get fields from dataclass fields = get_dataclass_fields(UnitTestValidatorDataclass) @@ -377,7 +377,7 @@ class BaseClass: validated: int = IntegerValidator() # Non-init field - non_init: int = dataclasses.field(init=False, default=1) # type: ignore[validataclass] # TODO: Fix me! + non_init: int = dataclasses.field(init=False, default=1) @validataclass class SubClass(BaseClass): From 492bc23c7fb1b37808df60b04dc669238d5cd308 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Mon, 9 Mar 2026 13:43:07 +0100 Subject: [PATCH 10/15] mypy plugin: Make sure decorator hook is idempotent --- .../mypy/plugin/validataclass_transformer.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/validataclass/mypy/plugin/validataclass_transformer.py b/src/validataclass/mypy/plugin/validataclass_transformer.py index fa49292..974d2ba 100644 --- a/src/validataclass/mypy/plugin/validataclass_transformer.py +++ b/src/validataclass/mypy/plugin/validataclass_transformer.py @@ -192,9 +192,21 @@ def transform(self) -> bool: Called for every class that is decorated with `@validataclass` (or an equivalent decorator). """ + # Plugin hooks may be called several times, so we need to check if we have already processed this class. + # TODO: I'm not sure how to test this. There is a comment in the mypy.plugin module that recommends using a + # forward reference to a class which should force the module to be processed multiple times, but this doesn't + # seem to work. It should be fine though, we're just skipping the class if we've already processed it. + if 'validataclass' in self._class_def.info.metadata: # pragma: nocover + # If we've processed this class, the dataclass plugin probably has too. But we can't be sure, so let the + # dataclass plugin handle this for itself. + return dataclass_class_maker_callback(self._ctx) + # Collect all validataclass fields in this class, including fields defined in base classes current_fields = self._collect_fields() - if current_fields is None: + + # If collecting the fields failed, we need another pass + # TODO: No idea how to enforce this in a test case. Add a test if you ever find out. + if current_fields is None: # pragma: nocover return False # Iterate over all fields and modify the assignment statements @@ -244,7 +256,8 @@ def _collect_fields(self) -> list[ValidataclassField] | None: continue # Ensure base class has already been processed by this plugin, otherwise we need another pass - if 'validataclass' not in base_typeinfo.metadata: + # TODO: No idea how to enforce this in a test case. Add a test if you ever find out. + if 'validataclass' not in base_typeinfo.metadata: # pragma: nocover self._log_debug(None, f'Base class "{base_fullname}" not processed yet, need another pass') return None @@ -313,8 +326,10 @@ def _collect_fields(self) -> list[ValidataclassField] | None: if isinstance(stmt.rvalue, CallExpr) and isinstance(stmt.rvalue.callee, RefExpr): callee_name = stmt.rvalue.callee.fullname - # Skip field if it has already been wrapped in a previous pass (this probably shouldn't happen though) - if callee_name == VIRTUAL_FIELD_WRAPPER_FUNC: + # Skip field if it has already been wrapped in a previous pass + # (This probably should never happen because it's avoided by the initial check whether the class has + # already been processed, but we keep this check to be safe.) + if callee_name == VIRTUAL_FIELD_WRAPPER_FUNC: # pragma: nocover self._log_debug(stmt, f'Skip already wrapped field "{lvalue.name}"') continue From 3c9d920a44559dbf167a354075c390ec9424502c Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Mon, 9 Mar 2026 14:28:16 +0100 Subject: [PATCH 11/15] mypy plugin: Clean up, refactoring, renaming --- src/validataclass/mypy/plugin/constants.py | 31 +++---------------- src/validataclass/mypy/plugin/error_codes.py | 30 ++++++++++++++++++ ...eld_resolver.py => field_type_resolver.py} | 26 ++++++---------- .../mypy/plugin/parsed_field_cache.py | 2 +- src/validataclass/mypy/plugin/plugin.py | 6 ++-- .../mypy/plugin/validataclass_transformer.py | 8 ++--- 6 files changed, 50 insertions(+), 53 deletions(-) create mode 100644 src/validataclass/mypy/plugin/error_codes.py rename src/validataclass/mypy/plugin/{virtual_field_resolver.py => field_type_resolver.py} (96%) diff --git a/src/validataclass/mypy/plugin/constants.py b/src/validataclass/mypy/plugin/constants.py index 5186e7a..0192c8c 100644 --- a/src/validataclass/mypy/plugin/constants.py +++ b/src/validataclass/mypy/plugin/constants.py @@ -6,17 +6,16 @@ from typing import Final -from mypy.errorcodes import ErrorCode - # Decorators that turn a class to a validataclass # (Not Final so it can be modified by other plugins that add their own validataclass-style decorators.) VALIDATACLASS_DECORATORS = { 'validataclass.dataclasses.validataclass.validataclass', } -# Full and short name of the validataclass_field() -VALIDATACLASS_FIELD_FUNC: Final = 'validataclass.dataclasses.validataclass_field.validataclass_field' -VALIDATACLASS_FIELD_FUNC_NAME: Final = 'validataclass_field' +# Functions that create validataclass fields (must have similar signature as validataclass_field) +VALIDATACLASS_FIELD_FUNCS = { + 'validataclass.dataclasses.validataclass_field.validataclass_field', +} # Full name of base class for validators VALIDATOR_BASE_CLASS: Final = 'validataclass.validators.validator.Validator' @@ -24,29 +23,7 @@ # Full name of base class for default objects FIELD_DEFAULT_BASE_CLASS: Final = 'validataclass.dataclasses.defaults.BaseDefault' -# Full name of the type of the special NoDefault object -FIELD_NO_DEFAULT_CLASS: Final = 'validataclass.dataclasses.defaults._NoDefaultType' - # Full and short name of the virtual field wrapper function # NOTE: This function does not actually exist in the code, it only exists for mypy. VIRTUAL_FIELD_WRAPPER_FUNC: Final = 'validataclass.mypy._virtual_field_wrapper' VIRTUAL_FIELD_WRAPPER_FUNC_NAME: Final = '_virtual_field_wrapper' - -# Custom error codes for validataclass -ERROR_CODE_VALIDATACLASS: Final = ErrorCode( - 'validataclass', - 'Check that validataclasses are defined correctly', - 'Plugin', -) -ERROR_CODE_VALIDATACLASS_EMPTY_TYPE: Final = ErrorCode( - 'validataclass-empty-type', - 'Check that validataclass field has a validator or default that can return a value', - 'Plugin', - sub_code_of=ERROR_CODE_VALIDATACLASS, -) -ERROR_CODE_VALIDATACLASS_NOT_IMPLEMENTED: Final = ErrorCode( - 'validataclass-not-implemented', - 'Special code for edge cases that are currently not supported by the plugin (please create a bug report)', - 'Plugin', - sub_code_of=ERROR_CODE_VALIDATACLASS, -) diff --git a/src/validataclass/mypy/plugin/error_codes.py b/src/validataclass/mypy/plugin/error_codes.py new file mode 100644 index 0000000..0d4511c --- /dev/null +++ b/src/validataclass/mypy/plugin/error_codes.py @@ -0,0 +1,30 @@ +""" +validataclass +Copyright (c) 2026, binary butterfly GmbH and contributors +Use of this source code is governed by an MIT-style license that can be found in the LICENSE file. +""" + +from typing import Final + +from mypy.errorcodes import ErrorCode + +# Custom error codes for validataclass +ERROR_CODE_VALIDATACLASS: Final = ErrorCode( + 'validataclass', + 'Check that validataclasses are defined correctly', + 'Plugin', +) + +ERROR_CODE_VALIDATACLASS_EMPTY_TYPE: Final = ErrorCode( + 'validataclass-empty-type', + 'Check that validataclass field has a validator or default that can return a value', + 'Plugin', + sub_code_of=ERROR_CODE_VALIDATACLASS, +) + +ERROR_CODE_VALIDATACLASS_NOT_IMPLEMENTED: Final = ErrorCode( + 'validataclass-not-implemented', + 'Special code for edge cases that are currently not supported by the plugin (please create a bug report)', + 'Plugin', + sub_code_of=ERROR_CODE_VALIDATACLASS, +) diff --git a/src/validataclass/mypy/plugin/virtual_field_resolver.py b/src/validataclass/mypy/plugin/field_type_resolver.py similarity index 96% rename from src/validataclass/mypy/plugin/virtual_field_resolver.py rename to src/validataclass/mypy/plugin/field_type_resolver.py index fcdaa3e..aec5fb1 100644 --- a/src/validataclass/mypy/plugin/virtual_field_resolver.py +++ b/src/validataclass/mypy/plugin/field_type_resolver.py @@ -23,18 +23,13 @@ get_proper_type, ) -from .constants import ( - ERROR_CODE_VALIDATACLASS, - ERROR_CODE_VALIDATACLASS_EMPTY_TYPE, - FIELD_DEFAULT_BASE_CLASS, - VALIDATACLASS_FIELD_FUNC, - VALIDATOR_BASE_CLASS, -) +from .constants import FIELD_DEFAULT_BASE_CLASS, VALIDATACLASS_FIELD_FUNCS, VALIDATOR_BASE_CLASS +from .error_codes import ERROR_CODE_VALIDATACLASS, ERROR_CODE_VALIDATACLASS_EMPTY_TYPE from .debug_logger import DebugLogger from .parsed_field_cache import ParsedFieldCache, ParsedValidataclassField -class VirtualFieldResolver: +class FieldTypeResolver: """ Handler for function hook to analyse the wrapped expression in virtual field wrapper calls, i.e. the right-hand side expressions of assignments in a validataclass. @@ -112,7 +107,7 @@ def resolve(self) -> Type: # Get the combined type of the field (union of validated type and default type) resolved_type = self._resolve_field_type(parsed_field) - self._log_debug(' => Resolved and combined field type', resolved_type) + self._log_debug('Resolved type of field "{field_name}"', resolved_type) # Handle edge case where resolved type is the empty type (i.e. Never). The most likely cause for this (which is # not "user does weird stuff") is that the field uses a RejectValidator (or similar, anything where validate() @@ -188,7 +183,7 @@ def _parse_field_definition( # (Fields created with dataclasses.field() are skipped by the ValidataclassTransformer.) if isinstance(field_rhs_expr, CallExpr) and isinstance(field_rhs_expr.callee, RefExpr): # Handle fields created explicitly using validataclass_field() - if field_rhs_expr.callee.fullname == VALIDATACLASS_FIELD_FUNC: + if field_rhs_expr.callee.fullname in VALIDATACLASS_FIELD_FUNCS: return self._parse_validataclass_field_callexpr(field_rhs_expr) # This will hold the end result that's returned at the end of the function @@ -196,7 +191,6 @@ def _parse_field_definition( # First, retrieve the parsed field of all base classes from our cache, starting with the "oldest" class for base_class in base_classes: - self._log_debug(f'Retrieve parsed field of base class"{base_class}" from cache') base_parsed_field = self._parsed_field_cache.get_field(base_class, field_name) # If there was an error when the field in the base class was parsed, set the error flag for later, then @@ -215,7 +209,7 @@ def _parse_field_definition( self._fail('Field cannot be fully parsed because of a prior error in one of the base classes') # Now, parse the right-hand side expression of the assignment in the current class and merge the result - self._log_debug('Parse RHS of assignment', field_rhs_expr) + self._log_debug(f'Parse field "{field_name}" with RHS expression', field_rhs_expr) assignment_parsed_field = self._parse_rhs_expression(field_rhs_expr) fully_parsed_field.merge(assignment_parsed_field) @@ -268,7 +262,7 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel return # Store type of validator for later - self._log_debug(f' - Validator: {item_type}') + self._log_debug(f' - Validator type: {item_type}') parsed_field.validator_type = item_type return @@ -281,7 +275,7 @@ def _parse_rhs_item(self, item_type: Type, parsed_field: ParsedValidataclassFiel return # Store type of default object for later - self._log_debug(f' - Default object: {item_type}') + self._log_debug(f' - Default type: {item_type}') parsed_field.default_type = item_type return @@ -354,8 +348,8 @@ def _parse_validataclass_field_callexpr(self, call_expr: CallExpr) -> ParsedVali break # Ensure that argument is a BaseDefault instance (mypy will report an error otherwise) - # NOTE: Using raw default values and dataclasses.MISSING here has been deprecated, so we don't need to - # support them. mypy will report a deprecation warning, we can just treat it as an error and return Any. + # NOTE: Using raw defaults and dataclasses.MISSING here has been deprecated, so we don't need to + # support them. mypy will report a deprecation warning, we just treat it as an error and return Any. if not isinstance(arg_type, Instance) or not arg_type.type.has_base(FIELD_DEFAULT_BASE_CLASS): parsed_field.error = True break diff --git a/src/validataclass/mypy/plugin/parsed_field_cache.py b/src/validataclass/mypy/plugin/parsed_field_cache.py index 78b0050..cdb33d9 100644 --- a/src/validataclass/mypy/plugin/parsed_field_cache.py +++ b/src/validataclass/mypy/plugin/parsed_field_cache.py @@ -38,7 +38,7 @@ class ParsedFieldCache: """ Cache for ParsedValidataclassFields, i.e. the parsed validator and default types of a validataclass field. - This cache is globally shared for the entire plugin, meaning that all VirtualFieldResolver instances can access it. + This cache is globally shared for the entire plugin, meaning that all FieldTypeResolver instances can access it. It's used to store the result of the field resolver, so that later instances can access the parsed types of a field in a base class. diff --git a/src/validataclass/mypy/plugin/plugin.py b/src/validataclass/mypy/plugin/plugin.py index c8429e8..bd0ffc8 100644 --- a/src/validataclass/mypy/plugin/plugin.py +++ b/src/validataclass/mypy/plugin/plugin.py @@ -16,9 +16,9 @@ from .constants import VALIDATACLASS_DECORATORS, VIRTUAL_FIELD_WRAPPER_FUNC from .debug_logger import DebugLogger +from .field_type_resolver import FieldTypeResolver from .parsed_field_cache import ParsedFieldCache from .validataclass_transformer import ValidataclassTransformer -from .virtual_field_resolver import VirtualFieldResolver def _is_debug_mode() -> bool: @@ -33,7 +33,7 @@ class ValidataclassPlugin(Plugin): # Logger class for easier debugging _logger: DebugLogger - # Internal cache for parsed validataclass fields, shared between all instances of VirtualFieldResolver + # Internal cache for parsed validataclass fields, shared between all instances of FieldTypeResolver _parsed_field_cache: ParsedFieldCache def __init__(self, options: Options): @@ -170,7 +170,7 @@ def _virtual_field_wrapper_callback(self, ctx: FunctionContext) -> Type: Analyze type of field definition and adjust function return type. """ - resolver = VirtualFieldResolver(ctx, self._logger, self._parsed_field_cache) + resolver = FieldTypeResolver(ctx, self._logger, self._parsed_field_cache) return resolver.resolve() diff --git a/src/validataclass/mypy/plugin/validataclass_transformer.py b/src/validataclass/mypy/plugin/validataclass_transformer.py index 974d2ba..b98f02b 100644 --- a/src/validataclass/mypy/plugin/validataclass_transformer.py +++ b/src/validataclass/mypy/plugin/validataclass_transformer.py @@ -31,12 +31,8 @@ from mypy.types import AnyType, CallableType, TypeOfAny, UnboundType from typing_extensions import override -from .constants import ( - ERROR_CODE_VALIDATACLASS, - ERROR_CODE_VALIDATACLASS_NOT_IMPLEMENTED, - VIRTUAL_FIELD_WRAPPER_FUNC, - VIRTUAL_FIELD_WRAPPER_FUNC_NAME, -) +from .constants import VIRTUAL_FIELD_WRAPPER_FUNC, VIRTUAL_FIELD_WRAPPER_FUNC_NAME +from .error_codes import ERROR_CODE_VALIDATACLASS, ERROR_CODE_VALIDATACLASS_NOT_IMPLEMENTED from .debug_logger import DebugLogger From 6846407cd093e9bca1fc01e612712c0c7374f4c8 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Mon, 9 Mar 2026 15:44:35 +0100 Subject: [PATCH 12/15] mypy: Enable mutable-override rule --- pyproject.toml | 3 +-- tests/unit/validators/dict_validator_test.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e50cb8b..dcd5b13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,8 +35,7 @@ enable_error_code = [ "deprecated", "explicit-override", "ignore-without-code", - # TODO: Maybe enable this is in the future (when we have the mypy plugin) - # "mutable-override", + "mutable-override", "possibly-undefined", "redundant-expr", "redundant-self", diff --git a/tests/unit/validators/dict_validator_test.py b/tests/unit/validators/dict_validator_test.py index 5eb91e9..6fedd98 100644 --- a/tests/unit/validators/dict_validator_test.py +++ b/tests/unit/validators/dict_validator_test.py @@ -625,7 +625,7 @@ def test_subclassed_dict_with_default_validator_valid(): """ Create a subclassed DictValidator that sets default_validator and test it with valid data. """ class UnitTestDefaultDictValidator(DictValidator[Decimal]): - default_validator = DecimalValidator() + default_validator = DecimalValidator() # type: ignore[mutable-override] validator = UnitTestDefaultDictValidator() @@ -646,7 +646,7 @@ def test_subclassed_dict_with_default_validator_invalid(): """ Create a subclassed DictValidator that sets default_validator and test it with invalid data. """ class UnitTestDefaultDictValidator(DictValidator[Decimal]): - default_validator = DecimalValidator() + default_validator = DecimalValidator() # type: ignore[mutable-override] validator = UnitTestDefaultDictValidator() From 40925b9ed1dfa2b8cf9e8abbfb6f8c614fdb1f81 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Mon, 9 Mar 2026 15:46:01 +0100 Subject: [PATCH 13/15] Update pytest-mypy-plugins testing requirement --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 579f39d..6e26494 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,7 @@ where = src testing = pytest ~= 9.0 pytest-cov ~= 7.0 - pytest-mypy-plugins ~= 3.2 + pytest-mypy-plugins ~= 3.3 coverage ~= 7.13 flake8 ~= 7.3 mypy ~= 1.19 From 16129b0c6e1afbe2109cc7c84d5fa61b1ac3ce0a Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Mon, 9 Mar 2026 15:50:50 +0100 Subject: [PATCH 14/15] GitHub Actions: Temporarily add dev-mypy branch to tests workflow --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a6a9c61..c1ad4aa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,13 +2,16 @@ name: Unit tests +# TODO: Remove dev-mypy after merging it into main. on: push: branches: - main + - dev-mypy pull_request: branches: - main + - dev-mypy jobs: test: From d7a1e7d6d1bebdde54df829b07b255f563d399a9 Mon Sep 17 00:00:00 2001 From: Lexi Stelter Date: Mon, 9 Mar 2026 15:57:31 +0100 Subject: [PATCH 15/15] Fix tiny linting issue --- tests/unit/dataclasses/validataclass_field_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/dataclasses/validataclass_field_test.py b/tests/unit/dataclasses/validataclass_field_test.py index 7b6fd3a..6cdd538 100644 --- a/tests/unit/dataclasses/validataclass_field_test.py +++ b/tests/unit/dataclasses/validataclass_field_test.py @@ -199,7 +199,7 @@ def test_validataclass_field_with_default_factory_kwarg_raises_exception(): assert ( str(exception_info.value) == 'Keyword argument "default_factory" is not allowed in validataclass_field (use ' - 'default=DefaultFactory(...) instead).' + 'default=DefaultFactory(...) instead).' ) @staticmethod