diff --git a/python/lib/sift_client/_tests/util/test_test_results_utils.py b/python/lib/sift_client/_tests/util/test_test_results_utils.py index cd7c953e6..61d08a8d2 100644 --- a/python/lib/sift_client/_tests/util/test_test_results_utils.py +++ b/python/lib/sift_client/_tests/util/test_test_results_utils.py @@ -23,6 +23,31 @@ def test_link_run_to_report(self, report_context, nostromo_run): report_context.report.update({"run_id": nostromo_run.id_}) assert report_context.report.run_id == nostromo_run.id_ + def test_docstring_description_setup(self, step): + """Test that the description of a step is set to the docstring of the test function. + + Args: + step: The step to test. + """ + expected_description = self.test_docstring_description_setup.__doc__ + assert step.current_step.description == expected_description + + def helper_function(_step: NewStep): + """Helper function description.""" + with _step.substep("Helper Substep") as helper_substep: + # This test is more of an example to indicate that only top level functions collected by pytest receive function's docstring. + assert helper_substep.current_step.description == None + + helper_function(step) + + def test_docstring_description_override(self, step): + """This description can still be overridden.""" + current_desc = self.test_docstring_description_override.__doc__ + assert step.current_step.description == current_desc + new_desc = "Manually updated description." + step.current_step.update({"description": new_desc}) + assert step.current_step.description == new_desc + def test_new_step(self, report_context): initial_end_time = report_context.report.end_time first_step_path = report_context.get_next_step_path() @@ -145,6 +170,7 @@ def test_bad_assert(self, report_context, step): "Nested Substep", "Has a bad assert" ) as nested_substep_context: nested_substep = nested_substep_context.current_step + nested_substep_context.force_result = True assert False == True with substep_context.substep( "Sibling Substep", "Should pass" @@ -207,6 +233,9 @@ def test_assign_value_to_measurement(self): assign_value_to_measurement(measurement, True) assert measurement.boolean_value == True + with pytest.raises(ValueError, match="Invalid value type: "): + assign_value_to_measurement(measurement, None) + def test_evaluate_measurement_bounds(self): measurement = TestMeasurementUpdate( measurement_type=TestMeasurementType.DOUBLE, diff --git a/python/lib/sift_client/util/test_results/bounds.py b/python/lib/sift_client/util/test_results/bounds.py index f1bf42920..a52d1cc8d 100644 --- a/python/lib/sift_client/util/test_results/bounds.py +++ b/python/lib/sift_client/util/test_results/bounds.py @@ -29,7 +29,7 @@ def assign_value_to_measurement( measurement.string_value = value measurement.measurement_type = TestMeasurementType.STRING else: - raise ValueError("Invalid value type") + raise ValueError(f"Invalid value type: {type(value)}") def evaluate_measurement_bounds( diff --git a/python/lib/sift_client/util/test_results/context_manager.py b/python/lib/sift_client/util/test_results/context_manager.py index 6a4a391d6..38feeb65e 100644 --- a/python/lib/sift_client/util/test_results/context_manager.py +++ b/python/lib/sift_client/util/test_results/context_manager.py @@ -162,6 +162,7 @@ def resolve_and_propagate_step_result( # Update the parent step results if this step failed (true by default so no need to do anything if we didn't fail). if not result: self.any_failures = True + self.open_step_results[step.step_path] = False path_parts = step.step_path.split(".") if len(path_parts) > 1: parent_step_path = ".".join(path_parts[:-1]) @@ -217,13 +218,15 @@ def update_step_from_result( exc: type[Exception] | None, exc_value: Exception | None, tb: traceback.TracebackException | None, - ): + ) -> bool: """Update the step based on its substeps and if there was an exception while executing the step. Args: exc: The class of Exception that was raised. exc_value: The exception value. tb: The traceback object. + + returns: The false if step failed or errored, true otherwise. """ error_info = None if exc: @@ -256,19 +259,25 @@ def update_step_from_result( } ) + return result + def __exit__(self, exc, exc_value, tb): - self.update_step_from_result(exc, exc_value, tb) + result = self.update_step_from_result(exc, exc_value, tb) # Now that the step is updated. Let the report context handle removing it from the stack and updating the report context. self.report_context.exit_step(self.current_step) - return True + # Test only attribute (hence not public class variable) + if hasattr(self, "force_result"): + result = self.force_result + + return result def measure( self, *, name: str, - value: float | str | bool, + value: float | str | bool | int, bounds: dict[str, float] | NumericBounds | str | None = None, timestamp: datetime | None = None, unit: str | None = None, diff --git a/python/lib/sift_client/util/test_results/pytest_util.py b/python/lib/sift_client/util/test_results/pytest_util.py index d92869e8f..90a30ced3 100644 --- a/python/lib/sift_client/util/test_results/pytest_util.py +++ b/python/lib/sift_client/util/test_results/pytest_util.py @@ -62,7 +62,8 @@ def _step_impl( report_context: ReportContext, request: pytest.FixtureRequest ) -> Generator[NewStep | None, None, None]: name = str(request.node.name) - with report_context.new_step(name=name) as new_step: + existing_docstring = request.node.obj.__doc__ or None + with report_context.new_step(name=name, description=existing_docstring) as new_step: yield new_step if hasattr(request.node, "rep_call") and request.node.rep_call.excinfo: new_step.update_step_from_result(