diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..bc915c1ff0 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,7 @@ +RELEASE_TYPE: patch + +This release improves shrinking for a very specific category of generator: +If you have a primitive strategy such as :func:`~hypothesis.strategies.text()` +and write ``my_primitive_strategy | some_more_complicated_strategy``, values +produced by the second strategy can now be shrunk as if they had come +from the first strategy. diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index ff2d8d521c..cb1f97dd1c 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -138,6 +138,13 @@ def structural_coverage(label: int) -> StructuralCoverageTag: # performance. We lose scan resistance, but that's probably fine here. POOLED_CONSTRAINTS_CACHE: LRUCache[tuple[Any, ...], ChoiceConstraintsT] = LRUCache(4096) +# The Python types corresponding to our choice-node primitive types. A strategy +# whose generated value has one of these exact types gets recorded on its span +# as the generated primitive value. We use a tuple rather than a set because +# ``in`` on a tuple relies only on equality, so it still works if the generated +# value's class has an unhashable metaclass. +_PRIMITIVE_CHOICE_TYPES: tuple[type, ...] = (int, bool, str, bytes, float) + class Span: """A span tracks the hierarchical structure of choices within a single test run. @@ -239,6 +246,15 @@ def children(self) -> "list[Span]": order.""" return [self.owner[i] for i in self.owner.children[self.index]] + @property + def generated_primitive_value(self) -> Any: + """The primitive value the corresponding strategy produced, or + ``None`` if there is none. Spans produced by a strategy generating a + primitive value (one of the types we have a choice node for - + ``int``, ``bool``, ``str``, ``bytes``, or ``float``) record the + generated value here.""" + return self.owner.generated_primitive_values.get(self.index) + class SpanProperty: """There are many properties of spans that we calculate by @@ -321,6 +337,9 @@ def __init__(self) -> None: self.__index_of_labels: dict[int, int] | None = {} self.trail = IntList() self.nodes: list[ChoiceNode] = [] + self.generated_primitive_values: dict[int, Any] = {} + self.__open_spans: list[int] = [] + self.__span_count = 0 def freeze(self) -> None: self.__index_of_labels = None @@ -336,12 +355,24 @@ def start_span(self, label: int) -> None: i = self.__index_of_labels.setdefault(label, len(self.labels)) self.labels.append(label) self.trail.append(TrailType.CHOICE + 1 + i) + self.__open_spans.append(self.__span_count) + self.__span_count += 1 def stop_span(self, *, discard: bool) -> None: if discard: self.trail.append(TrailType.STOP_SPAN_DISCARD) else: self.trail.append(TrailType.STOP_SPAN_NO_DISCARD) + self.__open_spans.pop() + + def record_value_for_span(self, value: Any) -> None: + # Record ``value`` against the most recently started span, but only if + # it is one of the primitive choice-node types. Called by + # ConjectureData.draw to capture the value a strategy produced. + if type(value) not in _PRIMITIVE_CHOICE_TYPES: + return + assert self.__open_spans, "Cannot record a value without an open span" + self.generated_primitive_values[self.__open_spans[-1]] = value class _starts_and_ends(SpanProperty): @@ -442,6 +473,9 @@ class Spans: def __init__(self, record: SpanRecord) -> None: self.trail = record.trail self.labels = record.labels + self.generated_primitive_values: dict[int, Any] = ( + record.generated_primitive_values + ) self.__length = self.trail.count( TrailType.STOP_SPAN_DISCARD ) + record.trail.count(TrailType.STOP_SPAN_NO_DISCARD) @@ -997,6 +1031,33 @@ def draw_boolean( constraints: BooleanConstraints = self._pooled_constraints("boolean", {"p": p}) return self._draw("boolean", constraints, observe=observe, forced=forced) + def add_choice_node_for(self, value: Any) -> None: + """Record ``value`` in the choice sequence as a forced choice. + + Strategies like :func:`just` and :func:`sampled_from` produce a value + without consulting the choice sequence. This method places a forced + choice so that the span is non-empty and visible to span-level + machinery (the recorded generated primitive value on the span, + shrinking widenings). + + For primitive values, the forced choice is of the corresponding type. + For non-primitive values it is a forced boolean ``False`` - the + simplest choice we can add, so the span doesn't look any more + complex than an empty one would. + """ + if type(value) is bool: + self.draw_boolean(forced=value) + elif type(value) is int: + self.draw_integer(forced=value) + elif type(value) is float: + self.draw_float(forced=value) + elif type(value) is str: + self.draw_string(IntervalSet.from_string(value), forced=value) + elif type(value) is bytes: + self.draw_bytes(max_size=len(value), forced=value) + else: + self.draw_boolean(forced=False) + @overload def _pooled_constraints( self, choice_type: Literal["integer"], constraints: IntegerConstraints @@ -1202,26 +1263,32 @@ def draw( self.start_span(label=label) try: if not at_top_level: - return unwrapped.do_draw(self) - assert start_time is not None - key = observe_as or f"generate:unlabeled_{len(self.draw_times)}" - try: + v = unwrapped.do_draw(self) + else: + assert start_time is not None + key = observe_as or f"generate:unlabeled_{len(self.draw_times)}" try: - v = unwrapped.do_draw(self) - finally: - # Subtract the time spent in GC to avoid overcounting, as it is - # accounted for at the overall example level. - in_gctime = gc_cumulative_time() - gc_start_time - self.draw_times[key] = time.perf_counter() - start_time - in_gctime - except Exception as err: - add_note( - err, - f"while generating {key.removeprefix('generate:')!r} from {strategy!r}", - ) - raise - if observability_enabled(): - avoid = self.provider.avoid_realization - self._observability_args[key] = to_jsonable(v, avoid_realization=avoid) + try: + v = unwrapped.do_draw(self) + finally: + # Subtract the time spent in GC to avoid overcounting, as + # it is accounted for at the overall example level. + in_gctime = gc_cumulative_time() - gc_start_time + self.draw_times[key] = ( + time.perf_counter() - start_time - in_gctime + ) + except Exception as err: + add_note( + err, + f"while generating {key.removeprefix('generate:')!r} from {strategy!r}", + ) + raise + if observability_enabled(): + avoid = self.provider.avoid_realization + self._observability_args[key] = to_jsonable( + v, avoid_realization=avoid + ) + self.__span_record.record_value_for_span(v) return v finally: self.stop_span() diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py index d8212f41c3..baa893e835 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py @@ -55,7 +55,8 @@ prefix_selection_order, random_selection_order, ) -from hypothesis.internal.floats import MAX_PRECISE_INTEGER +from hypothesis.internal.floats import MAX_PRECISE_INTEGER, SMALLEST_SUBNORMAL +from hypothesis.internal.intervalsets import IntervalSet if TYPE_CHECKING: from random import Random @@ -89,6 +90,60 @@ def sort_key(nodes: Sequence[ChoiceNode]) -> tuple[int, tuple[int, ...]]: ) +def _choice_node_for_value(value: ChoiceT) -> ChoiceNode: + """Return a ``ChoiceNode`` wrapping a primitive value, with permissive + constraints that accept the value. Used by + ``widen_to_span_with_generated_primitive_value`` to synthesise a single + choice from a span's recorded generated primitive value.""" + if type(value) is bool: + return ChoiceNode( + type="boolean", value=value, constraints={"p": 0.5}, was_forced=False + ) + if type(value) is int: + return ChoiceNode( + type="integer", + value=value, + constraints={ + "min_value": None, + "max_value": None, + "weights": None, + "shrink_towards": 0, + }, + was_forced=False, + ) + if type(value) is float: + return ChoiceNode( + type="float", + value=value, + constraints={ + "min_value": -math.inf, + "max_value": math.inf, + "allow_nan": True, + "smallest_nonzero_magnitude": SMALLEST_SUBNORMAL, + }, + was_forced=False, + ) + if type(value) is str: + return ChoiceNode( + type="string", + value=value, + constraints={ + "intervals": IntervalSet.from_string(value), + "min_size": 0, + "max_size": len(value), + }, + was_forced=False, + ) + if type(value) is bytes: + return ChoiceNode( + type="bytes", + value=value, + constraints={"min_size": 0, "max_size": len(value)}, + was_forced=False, + ) + raise AssertionError(f"non-primitive value {value!r} of type {type(value)}") + + @dataclass(slots=True, frozen=False) class ShrinkPass: function: Any @@ -321,6 +376,7 @@ def __init__( ShrinkPass(self.redistribute_numeric_pairs), ShrinkPass(self.lower_integers_together), ShrinkPass(self.lower_duplicated_characters), + ShrinkPass(self.widen_to_span_with_generated_primitive_value), ] # Because the shrinker is also used to `pareto_optimise` in the target phase, @@ -501,6 +557,12 @@ def explain(self) -> None: ): continue + # Skip slices with no non-forced nodes - there's nothing we can + # vary, so the "or any other generated value" note would be + # misleading (the value is in fact fully determined). + if all(nodes[i].was_forced for i in range(start, end)): + continue # pragma: no cover # only reachable via explain phase + # Run our experiments n_same_failures = 0 note = "or any other generated value" @@ -1508,6 +1570,51 @@ def copy_node(node, n): ), ) + def widen_to_span_with_generated_primitive_value(self, chooser): + """Try to navigate away from a specific ``one_of`` alternative into + an earlier one by using the span's recorded generated primitive value. + + If we have an integer choice with ``min_value == 0`` currently set to + a non-zero value, and it is immediately followed by a span whose + corresponding strategy produced a primitive value, we replace the + integer with ``0`` and the span's choices with a single choice + holding that primitive value. The engine then re-runs the test + against the earlier alternative with that value. + + This is useful for ``basic_strategy | specific_strategy``, where + the specific branch produced a primitive that the basic branch could + also have produced: we slip the primitive across into the basic + branch so that normal shrinking can take it the rest of the way. + """ + node = chooser.choose( + self.nodes, + lambda n: ( + n.type == "integer" + and not n.was_forced + and n.constraints["min_value"] == 0 + and n.value != 0 + ), + ) + + following = node.index + 1 + if following >= len(self.spans_starting_at): + return + + candidate_spans = self.spans_starting_at[following] + span_idx = chooser.choose( + candidate_spans, + lambda i: self.spans[i].generated_primitive_value is not None, + ) + span = self.spans[span_idx] + replacement = _choice_node_for_value(span.generated_primitive_value) + + self.consider_new_nodes( + self.nodes[: node.index] + + (node.copy(with_value=0),) + + (replacement,) + + self.nodes[span.end :] + ) + def minimize_nodes(self, nodes): choice_type = nodes[0].type value = nodes[0].value diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/misc.py b/hypothesis-python/src/hypothesis/strategies/_internal/misc.py index cbcfa32445..7a6c1af27e 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/misc.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/misc.py @@ -33,12 +33,6 @@ class JustStrategy(SampledFromStrategy[Ex]): It's implemented as a length-one SampledFromStrategy so that all our special-case logic for filtering and sets applies also to just(x). - - The important difference from a SampledFromStrategy with only one - element to choose is that JustStrategy *never* touches the underlying - choice sequence, i.e. drawing neither reads from nor writes to `data`. - This is a reasonably important optimisation (or semantic distinction!) - for both JustStrategy and SampledFromStrategy. """ @property @@ -60,7 +54,8 @@ def calc_is_cacheable(self, recur: RecurT) -> bool: def do_filtered_draw(self, data: ConjectureData) -> Ex | UniqueIdentifier: # The parent class's `do_draw` implementation delegates directly to # `do_filtered_draw`, which we can greatly simplify in this case since - # we have exactly one value. (This also avoids drawing any data.) + # we have exactly one value. The parent's ``do_draw`` will record the + # resulting value with ``data.add_choice_node_for``. return self._transform(self.value) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py index a9bf739386..0dbab92158 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py @@ -654,7 +654,7 @@ def calc_label(self) -> int: # The worst case performance of this scheme is # itertools.chain(range(2**100), [st.none()]), where it degrades to # hashing every int in the range. - (elements_is_hashable, hash_value) = _is_hashable(self.elements) + elements_is_hashable, hash_value = _is_hashable(self.elements) if isinstance(self.elements, range) or ( elements_is_hashable and not any(isinstance(e, SearchStrategy) for e in self.elements) @@ -719,6 +719,10 @@ def do_draw(self, data: ConjectureData) -> Ex: if result is filter_not_satisfied: data.mark_invalid(f"Aborted test because unable to satisfy {self!r}") assert not isinstance(result, UniqueIdentifier) + # Record the generated value as a forced choice so the span is + # non-empty and (for primitive values) annotated with the generated + # value for the shrinker's widening pass. + data.add_choice_node_for(result) return result def get_element(self, i: int) -> Ex | UniqueIdentifier: @@ -843,12 +847,30 @@ def calc_label(self) -> int: ) def do_draw(self, data: ConjectureData) -> Ex: - strategy = data.draw( - SampledFromStrategy(self.element_strategies).filter( - lambda s: not s.is_currently_empty(data) + strategies = self.element_strategies + n = len(strategies) + # Draw an index to pick an alternative, retrying a few times if the + # chosen strategy is currently empty (e.g. an empty Bundle in stateful + # testing). + for attempt in range(3): + i = data.draw_integer(0, n - 1) + if not strategies[i].is_currently_empty(data): + return data.draw(strategies[i]) + if attempt == 0: + data.events[ + f"Retried draw from {self!r} to avoid empty alternative" + ] = "" + + # If the retries all landed on empty alternatives, fall back to + # exhaustively picking a non-empty one. + allowed = [i for i in range(n) if not strategies[i].is_currently_empty(data)] + if not allowed: + data.mark_invalid( + f"Aborted test because all alternatives of {self!r} were empty" ) - ) - return data.draw(strategy) + i = data.choice(allowed) + data.draw_integer(0, n - 1, forced=i) + return data.draw(strategies[i]) def __repr__(self) -> str: return "one_of({})".format(", ".join(map(repr, self.original_strategies))) diff --git a/hypothesis-python/tests/conjecture/test_test_data.py b/hypothesis-python/tests/conjecture/test_test_data.py index faca8a87c2..a5309c7dc8 100644 --- a/hypothesis-python/tests/conjecture/test_test_data.py +++ b/hypothesis-python/tests/conjecture/test_test_data.py @@ -9,6 +9,7 @@ # obtain one at https://mozilla.org/MPL/2.0/. import itertools +from random import Random import pytest @@ -162,7 +163,7 @@ def test_example_depth_marking(): def test_has_examples_even_when_empty(): - d = ConjectureData.for_choices([]) + d = ConjectureData.for_choices([False]) d.draw(st.just(False)) d.freeze() assert d.spans @@ -413,3 +414,98 @@ def test_overruns_at_exactly_max_length(): except StopTest: pass assert data.status is Status.OVERRUN + + +@pytest.mark.parametrize( + "strategy, expected", + [ + (st.just(0), 0), + (st.just(True), True), + (st.just("hello"), "hello"), + (st.just(b"abc"), b"abc"), + (st.just(1.5), 1.5), + ], +) +def test_primitive_strategy_spans_record_generated_primitive_value(strategy, expected): + d = ConjectureData(random=Random(0)) + value = d.draw(strategy) + d.freeze() + assert value == expected + strategy_span = next(ex for ex in d.spans if ex.depth == 1) + assert strategy_span.generated_primitive_value == expected + + +def test_non_primitive_strategy_spans_do_not_record_a_value(): + d = ConjectureData(random=Random(0)) + d.draw(st.just((1, "x"))) + d.freeze() + # A just() returning a tuple is not a primitive value - no value recorded. + outer = next(ex for ex in d.spans if ex.depth == 1) + assert outer.generated_primitive_value is None + + +def test_top_level_span_does_not_record_a_value(): + d = ConjectureData(random=Random(0)) + d.draw(st.just(0)) + d.freeze() + # The top-level span wraps the whole test run, not a single strategy. + assert d.spans[0].generated_primitive_value is None + + +def test_manual_record_value_for_span(): + # Test the low-level record_value_for_span API directly. + d = ConjectureData(random=Random(0)) + d.start_span(1) + d._ConjectureData__span_record.record_value_for_span("outer") + d.start_span(2) + d._ConjectureData__span_record.record_value_for_span(42) + d.stop_span() + d.stop_span() + d.freeze() + values = { + ex.label: ex.generated_primitive_value for ex in d.spans if ex.label in (1, 2) + } + assert values == {1: "outer", 2: 42} + + +def test_span_without_value_returns_none(): + d = ConjectureData(random=Random(0)) + d.start_span(1) + d.stop_span() + d.freeze() + span = next(ex for ex in d.spans if ex.label == 1) + assert span.generated_primitive_value is None + + +def test_record_value_for_span_ignores_non_primitive(): + # record_value_for_span only records primitive choice-node types. + d = ConjectureData(random=Random(0)) + d.start_span(1) + d._ConjectureData__span_record.record_value_for_span([1, 2, 3]) + d.stop_span() + d.freeze() + span = next(ex for ex in d.spans if ex.label == 1) + assert span.generated_primitive_value is None + + +@pytest.mark.parametrize( + "value", + [True, 42, 1.5, "hello", b"hello"], +) +def test_choice_node_for_value_round_trips_primitives(value): + # The shrinker's widening pass synthesises a ChoiceNode from a primitive + # value. Exercise each primitive type. + from hypothesis.internal.conjecture.choice import choice_permitted + from hypothesis.internal.conjecture.shrinker import _choice_node_for_value + + node = _choice_node_for_value(value) + assert node.value == value + assert not node.was_forced + assert choice_permitted(node.value, node.constraints) + + +def test_choice_node_for_value_rejects_non_primitive(): + from hypothesis.internal.conjecture.shrinker import _choice_node_for_value + + with pytest.raises(AssertionError, match="non-primitive"): + _choice_node_for_value([1, 2, 3]) diff --git a/hypothesis-python/tests/cover/test_sampled_from.py b/hypothesis-python/tests/cover/test_sampled_from.py index e6a4f7ef51..9012d190f7 100644 --- a/hypothesis-python/tests/cover/test_sampled_from.py +++ b/hypothesis-python/tests/cover/test_sampled_from.py @@ -160,7 +160,7 @@ def test_unsatisfiable_explicit_filteredstrategy_just(x): def test_transformed_just_strategy(): - data = ConjectureData.for_choices([]) + data = ConjectureData.for_choices([2]) s = JustStrategy([1]).map(lambda x: x * 2) assert s.do_draw(data) == 2 sf = s.filter(lambda x: False) diff --git a/hypothesis-python/tests/cover/test_searchstrategy.py b/hypothesis-python/tests/cover/test_searchstrategy.py index 269da9898c..025c4de386 100644 --- a/hypothesis-python/tests/cover/test_searchstrategy.py +++ b/hypothesis-python/tests/cover/test_searchstrategy.py @@ -52,18 +52,6 @@ def __repr__(self): assert repr(st.just(WeirdRepr())) == f"just({WeirdRepr()!r})" -def test_just_strategy_does_not_draw(): - data = ConjectureData.for_choices([]) - s = st.just("hello") - assert s.do_draw(data) == "hello" - - -def test_none_strategy_does_not_draw(): - data = ConjectureData.for_choices([]) - s = st.none() - assert s.do_draw(data) is None - - def test_can_map(): s = st.integers().map(pack=lambda t: "foo") assert_simple_property(s, lambda v: v == "foo") @@ -227,3 +215,13 @@ def test_to_jsonable_handles_reference_cycles(obj, value): def test_deferred_strategy_draw(): strategy = st.deferred(lambda: st.integers()) assert strategy.do_draw(ConjectureData.for_choices([0])) == 0 + + +def test_lazy_strategy_draw(): + # Directly exercise LazyStrategy.do_draw (normally data.draw unwraps + # lazy strategies, so this is the only path that reaches do_draw). + from hypothesis.strategies._internal.lazy import LazyStrategy + + strategy = st.integers() # integers() returns a LazyStrategy wrapper + assert isinstance(strategy, LazyStrategy) + assert strategy.do_draw(ConjectureData.for_choices([0])) == 0 diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py index d5f75aee84..943373d4c3 100644 --- a/hypothesis-python/tests/cover/test_stateful.py +++ b/hypothesis-python/tests/cover/test_stateful.py @@ -731,16 +731,13 @@ def rule_1(self): run_state_machine_as_test(BadInvariant) result = "\n".join(err.value.__notes__) - assert ( - result - == """ + assert result == """ Falsifying example: state = BadInvariant() state.initialize_1() state.invariant_1() state.teardown() """.strip() - ) def test_invariant_present_in_falsifying_example(): @@ -945,16 +942,13 @@ def fail_fast(self, param): run_state_machine_as_test(WithInitializeBundleRules) result = "\n".join(err.value.__notes__) - assert ( - result - == """ + assert result == """ Falsifying example: state = WithInitializeBundleRules() a_0 = state.initialize_a(dep='dep') state.fail_fast(param=a_0) state.teardown() """.strip() - ) def test_initialize_rule_dont_mix_with_precondition(): @@ -1076,9 +1070,7 @@ def fail_eventually(self): run_state_machine_as_test(StateMachine) result = "\n".join(err.value.__notes__) - assert ( - result - == """ + assert result == """ Falsifying example: state = StateMachine() state.initialize() @@ -1086,7 +1078,6 @@ def fail_eventually(self): state.fail_eventually() state.teardown() """.strip() - ) def test_steps_printed_despite_pytest_fail(): @@ -1099,14 +1090,11 @@ def oops(self): with pytest.raises(Failed) as err: run_state_machine_as_test(RaisesProblem) - assert ( - "\n".join(err.value.__notes__).strip() - == """ + assert "\n".join(err.value.__notes__).strip() == """ Falsifying example: state = RaisesProblem() state.oops() state.teardown()""".strip() - ) def test_steps_not_printed_with_pytest_skip(capsys): @@ -1159,7 +1147,7 @@ def oops(self): def test_reproduce_failure_works(): - @reproduce_failure(__version__, encode_failure([False, 0, True])) + @reproduce_failure(__version__, encode_failure([False, 0, False, False])) class TrivialMachine(RuleBasedStateMachine): @rule() def oops(self): @@ -1170,7 +1158,7 @@ def oops(self): def test_reproduce_failure_fails_if_no_error(): - @reproduce_failure(__version__, encode_failure([False, 0, True])) + @reproduce_failure(__version__, encode_failure([False, 0, False, False])) class TrivialMachine(RuleBasedStateMachine): @rule() def ok(self): @@ -1316,16 +1304,13 @@ def fail_fast(self, param): run_state_machine_as_test(Machine) result = "\n".join(err.value.__notes__) - assert ( - result - == """ + assert result == """ Falsifying example: state = Machine() a_0, a_1, a_2 = state.initialize() state.fail_fast(param=a_2) state.teardown() """.strip() - ) @pytest.mark.parametrize( @@ -1368,16 +1353,13 @@ def fail_fast(self): run_state_machine_as_test(Machine) result = "\n".join(err.value.__notes__) - assert ( - result - == f""" + assert result == f""" Falsifying example: state = Machine() {repr_} state.fail_fast() state.teardown() """.strip() - ) def test_multiple_targets(): @@ -1405,9 +1387,7 @@ def fail_fast(self, a1, a2, a3, b1, b2, b3): run_state_machine_as_test(Machine) result = "\n".join(err.value.__notes__) - assert ( - result - == """ + assert result == """ Falsifying example: state = Machine() a_0, a_1, a_2 = state.initialize() @@ -1415,7 +1395,6 @@ def fail_fast(self, a1, a2, a3, b1, b2, b3): state.fail_fast(a1=a_2, a2=a_1, a3=a_0, b1=b_2, b2=b_1, b3=b_0) state.teardown() """.strip() - ) def test_multiple_common_targets(): @@ -1446,9 +1425,7 @@ def fail_fast(self, a1, a2, a3, a4, a5, a6, b1, b2, b3): run_state_machine_as_test(Machine) result = "\n".join(err.value.__notes__) - assert ( - result - == """ + assert result == """ Falsifying example: state = Machine() a_0, a_1, a_2 = state.initialize() @@ -1457,7 +1434,6 @@ def fail_fast(self, a1, a2, a3, a4, a5, a6, b1, b2, b3): state.fail_fast(a1=a_5, a2=a_4, a3=a_3, a4=a_2, a5=a_1, a6=a_0, b1=b_2, b2=b_1, b3=b_0) state.teardown() """.strip() - ) class LotsOfEntropyPerStepMachine(RuleBasedStateMachine): diff --git a/hypothesis-python/tests/quality/test_widening_shrinks.py b/hypothesis-python/tests/quality/test_widening_shrinks.py new file mode 100644 index 0000000000..764e97fd10 --- /dev/null +++ b/hypothesis-python/tests/quality/test_widening_shrinks.py @@ -0,0 +1,244 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +"""Quality tests for ``basic_strategy | specific_strategy`` combinations. + +Each predicate below is chosen so that random draws from the basic (open) +strategy satisfy it with probability well under 1%. That ensures the +initial failing example essentially always comes from the specific +(complicated) branch, so the test genuinely exercises the shrinker's +ability to widen out of the specific branch into the open one. +""" + +import hypothesis.strategies as st +from hypothesis.strategies import ( + integers, + just, + lists, + sampled_from, + text, +) + +from tests.common.debug import minimal + + +def test_widen_text_with_just_long_string(): + assert ( + minimal( + text() | just("I am the very model of a modern major generator"), + lambda s: "very model" in s, + ) + == "very model" + ) + + +def test_widen_text_with_just_short_substring(): + assert minimal(text() | just("xyzzy-plover"), lambda s: "xyzzy" in s) == "xyzzy" + + +def test_widen_text_with_sampled_from(): + assert ( + minimal( + text() + | sampled_from(["quetzalcoatl-1", "quetzalcoatl-2", "quetzalcoatl-3"]), + lambda s: "quetzalcoatl" in s, + ) + == "quetzalcoatl" + ) + + +def test_widen_integers_with_just(): + # A narrow window around the just value - random ``integers()`` draws + # hit it much less than 1% of the time. + assert ( + minimal( + integers() | just(5000), + lambda n: 4990 <= n <= 5010, + ) + == 4990 + ) + + +def test_widen_integers_with_sampled_from(): + assert ( + minimal( + integers() | sampled_from([4991, 4999, 5001, 5009]), + lambda n: 4990 <= n <= 5010, + ) + == 4990 + ) + + +def test_widen_bounded_integers_with_just(): + assert ( + minimal( + integers(0, 10_000) | just(5000), + lambda n: 4990 <= n <= 5010, + ) + == 4990 + ) + + +def test_widen_binary_with_just(): + # ``b"abc"`` in random bytes is vanishingly rare (~L/256**3 per draw). + assert ( + minimal( + st.binary() | just(b"xyz-abcdefghijk-specific-bytes"), + lambda b: b"abc" in b, + ) + == b"abc" + ) + + +def test_widen_floats_with_just(): + # A narrow window well away from the typical shrink targets. + assert ( + minimal( + st.floats() | just(11.0), + lambda x: 10.0 <= x <= 12.0, + ) + == 10.0 + ) + + +def test_widen_text_in_a_list(): + result = minimal( + lists(text() | just("marker-specific-string"), min_size=1), + lambda xs: any("marker" in x for x in xs), + ) + assert result == ["marker"] + + +def test_widen_integers_in_a_list(): + result = minimal( + lists(integers() | just(500), min_size=1), + lambda xs: any(490 <= x <= 510 for x in xs), + ) + assert result == [490] + + +def test_widen_three_way_one_of(): + # ``quux`` is a distinctive 4-letter sequence - essentially never + # generated by random ``text()`` - so the initial failure comes from + # one of the two ``just()`` branches. + s = text() | just("quux-kangaroo") | just("wallaby-quux-kangaroo") + assert minimal(s, lambda v: "quux" in v) == "quux" + + +def test_widen_text_with_builds(): + compound = st.builds( + lambda a, b: a + " middle " + b, + sampled_from(["alpha", "beta"]), + sampled_from(["gamma", "delta"]), + ) + assert minimal(text() | compound, lambda s: "middle" in s) == "middle" + + +# The tests below show that widening still reaches the minimum when the +# open strategy carries restrictions, as long as the specific branch's value +# respects those restrictions. + + +def test_widen_text_with_alphabet(): + # Random ``text(alphabet="abcd")`` draws hit the 8-character target + # under 1% of the time. + assert ( + minimal( + text(alphabet="abcd") | just("aabbccdd"), + lambda s: "aabbccdd" in s, + ) + == "aabbccdd" + ) + + +def test_widen_text_with_min_size(): + # ``"hello"`` in random ``text(min_size=3)`` is extremely rare. + assert ( + minimal( + text(min_size=3) | just("helloworld"), + lambda s: "hello" in s, + ) + == "hello" + ) + + +def test_widen_text_with_max_size(): + # A 2-char distinctive sequence inside a 5-char cap - very rare. + assert minimal(text(max_size=5) | just("xyz"), lambda s: "xy" in s) == "xy" + + +def test_widen_integers_with_min_value(): + assert ( + minimal( + integers(min_value=10) | just(5000), + lambda n: 4990 <= n <= 5010, + ) + == 4990 + ) + + +def test_widen_integers_with_max_value(): + assert ( + minimal( + integers(max_value=100) | just(-5000), + lambda n: -5010 <= n <= -4990, + ) + == -4990 + ) + + +def test_widen_integers_with_narrow_range(): + assert ( + minimal( + integers(0, 1000) | just(500), + lambda n: 490 <= n <= 510, + ) + == 490 + ) + + +def test_widen_floats_with_range(): + assert ( + minimal( + st.floats(0.0, 100.0) | just(11.0), + lambda x: 10.0 <= x <= 12.0, + ) + == 10.0 + ) + + +def test_widen_floats_no_nan(): + assert ( + minimal( + st.floats(allow_nan=False) | just(11.0), + lambda x: 10.0 <= x <= 12.0, + ) + == 10.0 + ) + + +def test_widen_binary_with_min_size(): + assert ( + minimal( + st.binary(min_size=3) | just(b"helloworld"), + lambda b: b"hello" in b, + ) + == b"hello" + ) + + +def test_widen_binary_with_bounded_size(): + assert ( + minimal( + st.binary(min_size=3, max_size=5) | just(b"abcde"), + lambda b: b"abc" in b, + ) + == b"abc" + )