diff --git a/.tool-versions b/.tool-versions index 97da906..e111eee 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -python 3.11.14 -poetry 2.3.3 +python 3.12.12 +poetry 2.4.1 java liberica-1.8.0 diff --git a/poetry.lock b/poetry.lock index 9f84732..102e9a7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "argcomplete" @@ -2118,8 +2118,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -2506,8 +2506,8 @@ astroid = ">=3.3.8,<=3.4.0.dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.2", markers = "python_version < \"3.11\""}, - {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, {version = ">=0.3.6", markers = "python_version == \"3.11\""}, + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, ] isort = ">=4.2.5,<5.13 || >5.13,<7" mccabe = ">=0.6,<0.8" diff --git a/src/dve/core_engine/backends/base/reader.py b/src/dve/core_engine/backends/base/reader.py index e9ed9e8..ae0e99f 100644 --- a/src/dve/core_engine/backends/base/reader.py +++ b/src/dve/core_engine/backends/base/reader.py @@ -119,10 +119,7 @@ def read_to_entity_type( """ if entity_name == Iterator[dict[str, Any]]: return self.read_to_py_iterator( - resource, - entity_name, - schema, # type: ignore - all_model_fields + resource, entity_name, schema, all_model_fields # type: ignore ) self.raise_if_not_sensible_file(resource, entity_name) @@ -133,11 +130,7 @@ def read_to_entity_type( raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err return reader_func( - self, - resource, - entity_name, - schema, - all_model_fields=all_model_fields # type: ignore + self, resource, entity_name, schema, all_model_fields=all_model_fields # type: ignore ) def add_record_index(self, entity: EntityType, **kwargs) -> EntityType: diff --git a/src/dve/core_engine/backends/exceptions.py b/src/dve/core_engine/backends/exceptions.py index c72f252..bb58516 100644 --- a/src/dve/core_engine/backends/exceptions.py +++ b/src/dve/core_engine/backends/exceptions.py @@ -37,10 +37,7 @@ class UnableToParseCSVError(MessageBearingError): """An error raised when unable to parse a CSV file""" def __init__( - self, - entity_name: str, - field_check_error_message: str, - field_check_error_code: str + self, entity_name: str, field_check_error_message: str, field_check_error_code: str ): super().__init__( messages=[ diff --git a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py index 786ef8f..65e20af 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py +++ b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py @@ -276,7 +276,9 @@ def _ddb_filter_contract_errors( "Entity": "STRING", }, ) - .filter(f"FailureType == 'record' AND Status != 'informational' AND Entity = '{entity_name}'") # pylint: disable=C0301 + .filter( + f"FailureType == 'record' AND Status != 'informational' AND Entity = '{entity_name}'" + ) # pylint: disable=C0301 .select("RecordIndex") .distinct() .order("RecordIndex asc") @@ -286,9 +288,7 @@ def _ddb_filter_contract_errors( return entity filtered_entity = entity.join( - relevant_record_rejection_codes_rel, - condition="__record_index__ == RecordIndex", - how="anti" + relevant_record_rejection_codes_rel, condition="__record_index__ == RecordIndex", how="anti" ) return filtered_entity diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py index 717ced2..3cd931d 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py @@ -71,7 +71,7 @@ def __init__( quote_char=quotechar, field_check=field_check, field_check_error_code=field_check_error_code, - field_check_error_message=field_check_error_message + field_check_error_message=field_check_error_message, ) def read_to_py_iterator( @@ -254,7 +254,7 @@ def read_to_relation( # pylint: disable=unused-argument resource=resource, entity_name=entity_name, schema=schema, - all_model_fields=all_model_fields + all_model_fields=all_model_fields, ) entity = entity.select(StarExpression(exclude=[RECORD_INDEX_COLUMN_NAME])).distinct() no_records = entity.shape[0] diff --git a/src/dve/core_engine/backends/implementations/spark/readers/csv.py b/src/dve/core_engine/backends/implementations/spark/readers/csv.py index c983cdf..2df30c5 100644 --- a/src/dve/core_engine/backends/implementations/spark/readers/csv.py +++ b/src/dve/core_engine/backends/implementations/spark/readers/csv.py @@ -9,13 +9,13 @@ from pyspark.sql.types import StructType from dve.core_engine.backends.base.reader import read_function -from dve.core_engine.backends.readers.csv import CSVFileReader from dve.core_engine.backends.exceptions import EmptyFileError from dve.core_engine.backends.implementations.spark.spark_helpers import ( get_type_from_annotation, spark_record_index, spark_write_parquet, ) +from dve.core_engine.backends.readers.csv import CSVFileReader from dve.core_engine.type_hints import URI, EntityName from dve.parser.file_handling import get_content_length diff --git a/src/dve/core_engine/backends/implementations/spark/rules.py b/src/dve/core_engine/backends/implementations/spark/rules.py index 66564ee..825ee15 100644 --- a/src/dve/core_engine/backends/implementations/spark/rules.py +++ b/src/dve/core_engine/backends/implementations/spark/rules.py @@ -14,10 +14,10 @@ create_udf, get_all_registered_udfs, object_to_spark_literal, + spark_filter_contract_errors, spark_read_parquet, spark_record_index, spark_write_parquet, - spark_filter_contract_errors, ) from dve.core_engine.backends.implementations.spark.types import ( Joined, diff --git a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py index 2c2fde4..c8e949d 100644 --- a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py +++ b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py @@ -26,8 +26,8 @@ from pyspark.sql.types import LongType, StructField, StructType from typing_extensions import Annotated, Protocol, TypedDict, get_args, get_origin, get_type_hints -from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type from dve.common.error_utils import get_feedback_errors_uri +from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME from dve.core_engine.type_hints import URI, EntityName @@ -380,12 +380,14 @@ def _spark_filter_contract_errors( relevant_record_rejections_codes_df = ( self.spark_session.read.json( path=contract_error_location, - schema=st.StructType([ - st.StructField("RecordIndex", st.IntegerType()), - st.StructField("FailureType", st.StringType()), - st.StructField("Status", st.StringType()), - st.StructField("Entity", st.StringType()), - ]), + schema=st.StructType( + [ + st.StructField("RecordIndex", st.IntegerType()), + st.StructField("FailureType", st.StringType()), + st.StructField("Status", st.StringType()), + st.StructField("Entity", st.StringType()), + ] + ), ) .filter( (sf.col("FailureType") == sf.lit("record")) diff --git a/src/dve/core_engine/backends/metadata/contract.py b/src/dve/core_engine/backends/metadata/contract.py index 234a1e9..b233e92 100644 --- a/src/dve/core_engine/backends/metadata/contract.py +++ b/src/dve/core_engine/backends/metadata/contract.py @@ -44,7 +44,7 @@ def schemas(self) -> dict[EntityName, type[BaseModel]]: """The per-entity schemas, as pydantic models.""" if not self._schemas: for entity_name, validator in self.validators.items(): - self._schemas[entity_name] = validator.model # type: ignore # pylint: disable=E1137 + self._schemas[entity_name] = validator.model # type: ignore # pylint: disable=E1137 return self._schemas.copy() # pylint: disable=E1101 @root_validator(allow_reuse=True) diff --git a/src/dve/core_engine/backends/readers/csv.py b/src/dve/core_engine/backends/readers/csv.py index 9d37b06..dbace5b 100644 --- a/src/dve/core_engine/backends/readers/csv.py +++ b/src/dve/core_engine/backends/readers/csv.py @@ -16,11 +16,11 @@ MissingHeaderError, ) from dve.core_engine.backends.readers.utilities import ( - raise_message_bearing_error_on_header_differences + get_all_model_fields, + raise_message_bearing_error_on_header_differences, ) from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME -from dve.core_engine.backends.readers.utilities import get_all_model_fields from dve.core_engine.type_hints import EntityName from dve.parser.file_handling import get_content_length, open_stream from dve.parser.file_handling.implementations.file import file_uri_to_local_path diff --git a/src/dve/core_engine/backends/readers/utilities.py b/src/dve/core_engine/backends/readers/utilities.py index c46cac7..00086bd 100644 --- a/src/dve/core_engine/backends/readers/utilities.py +++ b/src/dve/core_engine/backends/readers/utilities.py @@ -44,16 +44,16 @@ def raise_message_bearing_error_on_header_differences( header or vice versa. """ missing, additional = check_csv_header_expected( - resource, - expected_schema, - all_model_fields, - delimiter, - quote_char + resource, expected_schema, all_model_fields, delimiter, quote_char ) if missing or additional: - record_details_missing = f"missing fields: {', '.join(sorted(missing))};" if missing else "" # pylint: disable=C0301 - record_details_additional = f"additional fields: {', '.join(sorted(additional))};" if additional else "" # pylint: disable=C0301 + record_details_missing = ( + f"missing fields: {', '.join(sorted(missing))};" if missing else "" + ) # pylint: disable=C0301 + record_details_additional = ( + f"additional fields: {', '.join(sorted(additional))};" if additional else "" + ) # pylint: disable=C0301 raise MessageBearingError( "The CSV header doesn't match what is expected", messages=[ diff --git a/src/dve/core_engine/configuration/v1/__init__.py b/src/dve/core_engine/configuration/v1/__init__.py index 04d5e07..6221957 100644 --- a/src/dve/core_engine/configuration/v1/__init__.py +++ b/src/dve/core_engine/configuration/v1/__init__.py @@ -281,7 +281,9 @@ def _load_rules_and_vars(self) -> tuple[list[Rule], list[TemplateVariables]]: rules, local_variable_list = [], [] added_rules: set[RuleName] = set() - for index, complex_rule_config in enumerate(self.transformations.complex_rules): # pylint: disable=E1101 + for index, complex_rule_config in enumerate( + self.transformations.complex_rules + ): # pylint: disable=E1101 rule, local_params, deps = self._resolve_business_rule(complex_rule_config) missing_rules = deps - added_rules if missing_rules: diff --git a/src/dve/core_engine/engine.py b/src/dve/core_engine/engine.py index b2ec9a7..838bfb5 100644 --- a/src/dve/core_engine/engine.py +++ b/src/dve/core_engine/engine.py @@ -170,7 +170,9 @@ def __exit__( exc_value: Optional[Exception], traceback: Optional[TracebackType], ) -> None: - self.main_log.info(f"Exiting pipeline context, clearing {self.cache_prefix!r}") # pylint: disable=E1101 + self.main_log.info( + f"Exiting pipeline context, clearing {self.cache_prefix!r}" + ) # pylint: disable=E1101 cache_dir = self._cache_dir self._cache_dir = None @@ -198,7 +200,9 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities: """ output_entities = {} - self.main_log.info(f"Writing entities to the output location: {self.output_prefix_uri}") # pylint: disable=E1101 + self.main_log.info( + f"Writing entities to the output location: {self.output_prefix_uri}" + ) # pylint: disable=E1101 for entity_name, entity in entities.items(): entity = entity.drop(RECORD_INDEX_COLUMN_NAME) @@ -206,9 +210,13 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities: output_uri = joinuri(self.output_prefix_uri, entity_name) if get_resource_exists(output_uri): - self.main_log.info(f"{output_uri} already exists - will be overwritten") # pylint: disable=E1101 + self.main_log.info( + f"{output_uri} already exists - will be overwritten" + ) # pylint: disable=E1101 - self.main_log.info(f"+ Writing parquet output to {output_uri!r}") # pylint: disable=E1101 + self.main_log.info( + f"+ Writing parquet output to {output_uri!r}" + ) # pylint: disable=E1101 entity.write.mode("overwrite").parquet(output_uri) spark_session = SparkSession.builder.getOrCreate() output_entities[entity_name] = spark_session.read.format("parquet").load( diff --git a/src/dve/core_engine/message.py b/src/dve/core_engine/message.py index 627ae3a..626a710 100644 --- a/src/dve/core_engine/message.py +++ b/src/dve/core_engine/message.py @@ -36,6 +36,8 @@ class DataContractErrorDetail(BaseModel): """Define custom error codes for validation issues raised during the data contract phase""" error_code: str + error_level: Optional[FailureType] = "record" + is_informational: Optional[bool] = False error_message: Optional[str] = None reporting_entity: Optional[str] = None @@ -247,26 +249,14 @@ def from_pydantic_error( messages: Messages = [] for error_dict in error.errors(): error_type = error_dict["type"] + # TODO - review in pydantic v2 - how handles null vs not provided values if "none.not_allowed" in error_type or "value_error.missing" in error_type: category = "Blank" else: category = "Bad value" - error_code = error_type - if "." in error_code: - error_code = error_code.split(".", 1)[-1] - - if error_code in INTEGRITY_ERROR_CODES: - failure_type: FailureType = "integrity" - elif error_code in SUBMISSION_ERROR_CODES: - failure_type = "submission" - else: - failure_type = "record" error_field = ".".join([idx for idx in error_dict["loc"] if not isinstance(idx, int)]) - is_informational = False - if error_code.endswith("warning"): - is_informational = True error_detail: DataContractErrorDetail = error_details.get( # type: ignore error_field, DEFAULT_ERROR_DETAIL ).get(category) @@ -276,8 +266,8 @@ def from_pydantic_error( entity=error_detail.reporting_entity or entity, original_entity=entity, record=record, - failure_type=failure_type, - is_informational=is_informational, + failure_type=error_detail.error_level, # type: ignore + is_informational=error_detail.is_informational, # type: ignore error_type=error_type, error_location=error_dict["loc"], # type: ignore error_message=error_detail.template_message(record, error_dict["loc"]), diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py index 1c32e87..eb66fc8 100644 --- a/src/dve/pipeline/pipeline.py +++ b/src/dve/pipeline/pipeline.py @@ -223,7 +223,7 @@ def write_file_to_parquet( submission_file_uri, model_name, stringify_model(model), # type: ignore - get_all_model_fields(models.values()) # type: ignore + get_all_model_fields(models.values()), # type: ignore ), f"{out}{model_name}", ) diff --git a/src/dve/reporting/excel_report.py b/src/dve/reporting/excel_report.py index 9471c83..55e65bd 100644 --- a/src/dve/reporting/excel_report.py +++ b/src/dve/reporting/excel_report.py @@ -141,11 +141,17 @@ def _add_submission_info(self, status: str, summary: Worksheet): for key, value in self.summary_dict.items(): summary.append(["", _key_renames.get(key, key), str(value)]) - summary.append([ - "", - "Total Number of Records Processed", - self.submission_status.number_of_records if self.submission_status.number_of_records else 0 # pylint: disable=C0301 - ]) + summary.append( + [ + "", + "Total Number of Records Processed", + ( + self.submission_status.number_of_records + if self.submission_status.number_of_records + else 0 + ), # pylint: disable=C0301 + ] + ) summary.append(["", ""]) diff --git a/tests/features/movies.feature b/tests/features/movies.feature index 6916a4e..750975e 100644 --- a/tests/features/movies.feature +++ b/tests/features/movies.feature @@ -19,16 +19,18 @@ Feature: Pipeline tests using the movies dataset Then the movies entity is stored as a parquet after the file_transformation phase And the latest audit record for the submission is marked with processing status data_contract When I run the data contract phase - Then there are 3 record rejections from the data_contract phase + Then there is 1 submission rejection from the data_contract phase + And there are 3 record rejections from the data_contract phase And there are errors with the following details and associated error_count from the data_contract phase - | Entity | ErrorCode | ErrorMessage | RecordIndex | error_count | - | movies | BLANKYEAR | year not provided | 2 | 1 | - | movies_rename_test | DODGYYEAR | year value (NOT_A_NUMBER) is invalid | 1 | 1 | - | movies | DODGYDATE | date_joined value is not valid: daft_date | 1 | 1 | + | Entity | ErrorCode | ErrorMessage | RecordIndex | error_count | + | movies | BLANKYEAR | year not provided | 2 | 1 | + | movies_rename_test | DODGYYEAR | year value (NOT_A_NUMBER) is invalid | 1 | 1 | + | movies | DODGYDATE | date_joined value is not valid: daft_date | 1 | 1 | + | movies | BLANKTITLE | title should not be blank | 4 | 1 | And the movies entity is stored as a parquet after the data_contract phase And the latest audit record for the submission is marked with processing status business_rules When I run the business rules phase - Then The rules restrict "movies" to 2 qualifying records + Then The rules restrict "movies" to 3 qualifying records And there are errors with the following details and associated error_count from the business_rules phase | ErrorCode | ErrorMessage | RecordIndex | error_count | | LIMITED_RATINGS | Movie has too few ratings ([6.5]) | 4 | 1 | @@ -37,10 +39,11 @@ Feature: Pipeline tests using the movies dataset When I run the error report phase Then An error report is produced And The statistics entry for the submission shows the following information - | parameter | value | - | record_count | 5 | - | number_record_rejections | 4 | - | number_warnings | 1 | + | parameter | value | + | record_count | 5 | + | number_submission_rejections | 1 | + | number_record_rejections | 3 | + | number_warnings | 2 | And the error aggregates are persisted Scenario: Validate and filter movies (duckdb) @@ -55,16 +58,18 @@ Feature: Pipeline tests using the movies dataset Then the movies entity is stored as a parquet after the file_transformation phase And the latest audit record for the submission is marked with processing status data_contract When I run the data contract phase - Then there are 3 record rejections from the data_contract phase + Then there is 1 submission rejection from the data_contract phase + And there are 3 record rejections from the data_contract phase And there are errors with the following details and associated error_count from the data_contract phase | Entity | ErrorCode | ErrorMessage | RecordIndex | error_count | | movies | BLANKYEAR | year not provided | 2 | 1 | | movies_rename_test | DODGYYEAR | year value (NOT_A_NUMBER) is invalid | 1 | 1 | | movies | DODGYDATE | date_joined value is not valid: daft_date | 1 | 1 | + | movies | BLANKTITLE | title should not be blank | 4 | 1 | And the movies entity is stored as a parquet after the data_contract phase And the latest audit record for the submission is marked with processing status business_rules When I run the business rules phase - Then The rules restrict "movies" to 2 qualifying records + Then The rules restrict "movies" to 3 qualifying records And there are errors with the following details and associated error_count from the business_rules phase | ErrorCode | ErrorMessage | RecordIndex | error_count | | LIMITED_RATINGS | Movie has too few ratings ([6.5]) | 4 | 1 | @@ -73,9 +78,10 @@ Feature: Pipeline tests using the movies dataset When I run the error report phase Then An error report is produced And The statistics entry for the submission shows the following information - | parameter | value | - | record_count | 5 | - | number_record_rejections | 4 | - | number_warnings | 1 | + | parameter | value | + | record_count | 5 | + | number_submission_rejections | 1 | + | number_record_rejections | 3 | + | number_warnings | 2 | And the error aggregates are persisted diff --git a/tests/features/planets.feature b/tests/features/planets.feature index c469a92..b37b60b 100644 --- a/tests/features/planets.feature +++ b/tests/features/planets.feature @@ -18,6 +18,7 @@ Feature: Pipeline tests using the planets dataset And the latest audit record for the submission is marked with processing status data_contract When I run the data contract phase Then there is 1 record rejection from the data_contract phase + And there are no submission rejections from the data_contract phase And the planets entity is stored as a parquet after the data_contract phase And the latest audit record for the submission is marked with processing status business_rules When I run the business rules phase diff --git a/tests/features/steps/steps_pipeline.py b/tests/features/steps/steps_pipeline.py index 061bfe1..c72c873 100644 --- a/tests/features/steps/steps_pipeline.py +++ b/tests/features/steps/steps_pipeline.py @@ -155,13 +155,13 @@ def create_error_report(context: Context): -@then("there are {expected_num_errors:d} record rejections from the {service} phase") -@then("there is {expected_num_errors:d} record rejection from the {service} phase") -@then("there are no record rejections from the {service} phase") -def get_record_rejects_from_service(context: Context, service: str, expected_num_errors: int = 0): +@then("there are {expected_num_errors:d} {error_type} rejections from the {service} phase") +@then("there is {expected_num_errors:d} {error_type} rejection from the {service} phase") +@then("there are no {error_type} rejections from the {service} phase") +def get_record_rejects_from_service(context: Context, service: str, error_type: str, expected_num_errors: int = 0): processing_path = ctxt.get_processing_location(context) message_df = load_errors_from_service(processing_path, service) - num_rejections = message_df.filter(pl.col("FailureType").eq("record")).shape[0] + num_rejections = message_df.filter(pl.col("FailureType").eq(error_type)).shape[0] assert num_rejections == expected_num_errors, f"Got {num_rejections} actual rejections" diff --git a/tests/test_core_engine/test_message.py b/tests/test_core_engine/test_message.py index ccb6736..4c092f1 100644 --- a/tests/test_core_engine/test_message.py +++ b/tests/test_core_engine/test_message.py @@ -183,9 +183,11 @@ class TestModel(BaseModel): custom_error_details: str = """ {"idx": {"Blank": {"error_code": "IDBLANKERRCODE", - "error_message": "idx is a mandatory field"}, + "error_message": "idx is a mandatory field", + "is_informational": true}, "Bad value": {"error_code": "IDDODGYVALCODE", - "error_message": "idx value is dodgy: {{idx}}"}}, + "error_message": "idx value is dodgy: {{idx}}", + "error_level": "submission"}}, "date_field": {"Bad value": {"error_code": "DATEDODGYVALCODE", "error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}} """ @@ -216,10 +218,16 @@ class TestModel(BaseModel): assert len(msgs_bad) == 3 assert msgs_bad[0].error_code == error_details.get("date_field").get("Bad value").error_code assert msgs_bad[0].error_message == error_details.get("date_field").get("Bad value").template_message(_bad_value_data) + assert msgs_bad[0].failure_type == "record" + assert not msgs_bad[0].is_informational assert msgs_bad[1].error_code == error_details.get("idx").get("Bad value").error_code assert msgs_bad[1].error_message == error_details.get("idx").get("Bad value").template_message(_bad_value_data) + assert msgs_bad[1].failure_type == "submission" + assert not msgs_bad[1].is_informational assert msgs_bad[2].error_code == bad_val_default.error_code assert msgs_bad[2].error_message == bad_val_default.error_message + assert msgs_bad[2].failure_type == "record" + assert not msgs_bad[2].is_informational msgs_blank = FeedbackMessage.from_pydantic_error(entity="test_entity", record = _blank_value_data, @@ -232,6 +240,7 @@ class TestModel(BaseModel): assert len(msgs_blank) == 2 assert msgs_blank[0].error_code == error_details.get("idx").get("Blank").error_code assert msgs_blank[0].error_message == error_details.get("idx").get("Blank").template_message(_blank_value_data) + assert msgs_blank[0].is_informational assert msgs_blank[1].error_code == blank_default.error_code assert msgs_blank[1].error_message == blank_default.error_message @@ -281,4 +290,5 @@ class TestModel(BaseModel): msg = msg[0] assert msg.error_code == "DATEDODGYVALCODE" assert msg.error_message == "date_field value is dodgy: a_field: test, date_field: Barry" + diff --git a/tests/testdata/movies/movies.json b/tests/testdata/movies/movies.json index afa606d..db6029f 100644 --- a/tests/testdata/movies/movies.json +++ b/tests/testdata/movies/movies.json @@ -32,7 +32,6 @@ ] }, { - "title": "One with a cat and a dog", "year": 2020, "genre": ["Fantasy", "Family"], "duration_minutes": 110, diff --git a/tests/testdata/movies/movies_contract_error_details.json b/tests/testdata/movies/movies_contract_error_details.json index f8cd934..260ee96 100644 --- a/tests/testdata/movies/movies_contract_error_details.json +++ b/tests/testdata/movies/movies_contract_error_details.json @@ -2,13 +2,15 @@ "title": { "Blank": { "error_code": "BLANKTITLE", - "error_message": "title should not be blank" + "error_message": "title should not be blank", + "error_level": "submission" } }, "year": { "Blank": { "error_code": "BLANKYEAR", - "error_message": "year not provided" + "error_message": "year not provided", + "is_informational": true }, "Bad value": { "error_code": "DODGYYEAR",