Test results error helpers

python/lib/sift_client/_tests/util/test_test_results_utils.py

@@ -23,6 +23,31 @@ def test_link_run_to_report(self, report_context, nostromo_run):
         report_context.report.update({"run_id": nostromo_run.id_})
         assert report_context.report.run_id == nostromo_run.id_
 
+    def test_docstring_description_setup(self, step):
+        """Test that the description of a step is set to the docstring of the test function.
+
+        Args:
+            step: The step to test.
+        """
+        expected_description = self.test_docstring_description_setup.__doc__
+        assert step.current_step.description == expected_description
+
+        def helper_function(_step: NewStep):
+            """Helper function description."""
+            with _step.substep("Helper Substep") as helper_substep:
+                # This test is more of an example to indicate that only top level functions collected by pytest receive function's docstring.
+                assert helper_substep.current_step.description == None
+
+        helper_function(step)
+
+    def test_docstring_description_override(self, step):
+        """This description can still be overridden."""
+        current_desc = self.test_docstring_description_override.__doc__
+        assert step.current_step.description == current_desc
+        new_desc = "Manually updated description."
+        step.current_step.update({"description": new_desc})
+        assert step.current_step.description == new_desc
+
     def test_new_step(self, report_context):
         initial_end_time = report_context.report.end_time
         first_step_path = report_context.get_next_step_path()
@@ -145,6 +170,7 @@ def test_bad_assert(self, report_context, step):
                     "Nested Substep", "Has a bad assert"
                 ) as nested_substep_context:
                     nested_substep = nested_substep_context.current_step
+                    nested_substep_context.force_result = True
                     assert False == True
                 with substep_context.substep(
                     "Sibling Substep", "Should pass"
@@ -207,6 +233,9 @@ def test_assign_value_to_measurement(self):
         assign_value_to_measurement(measurement, True)
         assert measurement.boolean_value == True
 
+        with pytest.raises(ValueError, match="Invalid value type: <class 'NoneType'>"):
+            assign_value_to_measurement(measurement, None)
+
     def test_evaluate_measurement_bounds(self):
         measurement = TestMeasurementUpdate(
             measurement_type=TestMeasurementType.DOUBLE,

python/lib/sift_client/util/test_results/bounds.py

@@ -29,7 +29,7 @@ def assign_value_to_measurement(
         measurement.string_value = value
         measurement.measurement_type = TestMeasurementType.STRING
     else:
-        raise ValueError("Invalid value type")
+        raise ValueError(f"Invalid value type: {type(value)}")
 
 
 def evaluate_measurement_bounds(

python/lib/sift_client/util/test_results/context_manager.py

@@ -162,6 +162,7 @@ def resolve_and_propagate_step_result(
         # Update the parent step results if this step failed (true by default so no need to do anything if we didn't fail).
         if not result:
             self.any_failures = True
+            self.open_step_results[step.step_path] = False
             path_parts = step.step_path.split(".")
             if len(path_parts) > 1:
                 parent_step_path = ".".join(path_parts[:-1])
@@ -217,13 +218,15 @@ def update_step_from_result(
         exc: type[Exception] | None,
         exc_value: Exception | None,
         tb: traceback.TracebackException | None,
-    ):
+    ) -> bool:
         """Update the step based on its substeps and if there was an exception while executing the step.
 
         Args:
             exc: The class of Exception that was raised.
             exc_value: The exception value.
             tb: The traceback object.
+
+        returns: The false if step failed or errored, true otherwise.
         """
         error_info = None
         if exc:
@@ -256,19 +259,25 @@ def update_step_from_result(
             }
         )
 
+        return result
+
     def __exit__(self, exc, exc_value, tb):
-        self.update_step_from_result(exc, exc_value, tb)
+        result = self.update_step_from_result(exc, exc_value, tb)
 
         # Now that the step is updated. Let the report context handle removing it from the stack and updating the report context.
         self.report_context.exit_step(self.current_step)
 
-        return True
+        # Test only attribute (hence not public class variable)
+        if hasattr(self, "force_result"):
+            result = self.force_result
+
+        return result
 
     def measure(
         self,
         *,
         name: str,
-        value: float | str | bool,
+        value: float | str | bool | int,
         bounds: dict[str, float] | NumericBounds | str | None = None,
         timestamp: datetime | None = None,
         unit: str | None = None,

python/lib/sift_client/util/test_results/pytest_util.py

@@ -62,7 +62,8 @@ def _step_impl(
     report_context: ReportContext, request: pytest.FixtureRequest
 ) -> Generator[NewStep | None, None, None]:
     name = str(request.node.name)
-    with report_context.new_step(name=name) as new_step:
+    existing_docstring = request.node.obj.__doc__ or None
+    with report_context.new_step(name=name, description=existing_docstring) as new_step:
         yield new_step
         if hasattr(request.node, "rep_call") and request.node.rep_call.excinfo:
             new_step.update_step_from_result(