diff --git a/hypothesis-python/tests/common/utils.py b/hypothesis-python/tests/common/utils.py index 6b70b254ff..9557c98e90 100644 --- a/hypothesis-python/tests/common/utils.py +++ b/hypothesis-python/tests/common/utils.py @@ -9,6 +9,7 @@ # obtain one at https://mozilla.org/MPL/2.0/. import contextlib +import enum import sys import warnings from io import StringIO @@ -249,3 +250,36 @@ def capture_observations(): # config option, so *linking against* something built this way can break us. # Everything is terrible PYTHON_FTZ = next_down(sys.float_info.min) == 0.0 + + +class Why(enum.Enum): + # Use an enum here so it's easier to find and/or exclude some cases later + + # things we want to fix + flaky_replay = "Inconsistent results from replaying a failing test..." + symbolic_outside_context = "CrosshairInternal error (using value outside context)" + floats = "crosshair doesn't reason about signed zero (and other edge cases?)" + no_unsatisfiable = "doesn't raise Unsatisfiable for some reason" + + # things that are basically fine to leave alone + + # nested_given: https://github.com/pschanely/hypothesis-crosshair/issues/11 + nested_given = "nested @given decorators don't work with crosshair" + other = "reasons not elsewhere categorized" + + +def xfail_on_crosshair(why: Why, /, *, strict=True, as_marks=False): + try: + import pytest + except ImportError: + return lambda fn: fn + + current_backend = settings.get_profile(settings._current_profile).backend + kw = { + "strict": strict, + "reason": f"Expected failure due to: {why.value}", + "condition": current_backend == "crosshair", + } + if as_marks: # for use with pytest.param(..., marks=xfail_on_crosshair()) + return (pytest.mark.xf_crosshair, pytest.mark.xfail(**kw)) + return lambda fn: pytest.mark.xf_crosshair(pytest.mark.xfail(**kw)(fn)) diff --git a/hypothesis-python/tests/conftest.py b/hypothesis-python/tests/conftest.py index 9b2c7ee5c3..3fe577a5a0 100644 --- a/hypothesis-python/tests/conftest.py +++ b/hypothesis-python/tests/conftest.py @@ -44,6 +44,7 @@ def pytest_configure(config): "markers", "xp_min_version(api_version): run when greater or equal to api_version", ) + config.addinivalue_line("markers", "xf_crosshair: selection for xfailing symbolics") def pytest_addoption(parser): diff --git a/hypothesis-python/tests/cover/test_cache_implementation.py b/hypothesis-python/tests/cover/test_cache_implementation.py index c5282ae84b..db4b531986 100644 --- a/hypothesis-python/tests/cover/test_cache_implementation.py +++ b/hypothesis-python/tests/cover/test_cache_implementation.py @@ -26,7 +26,7 @@ from hypothesis.errors import InvalidArgument from hypothesis.internal.cache import GenericCache, LRUCache, LRUReusedCache -from tests.common.utils import skipif_emscripten +from tests.common.utils import Why, skipif_emscripten, xfail_on_crosshair class LRUCacheAlternative(GenericCache): @@ -116,6 +116,7 @@ def test_behaves_like_a_dict_with_losses(implementation, writes, size): assert len(target) <= min(len(model), size) +@xfail_on_crosshair(Why.symbolic_outside_context) @settings( suppress_health_check={HealthCheck.too_slow} | set(settings.get_profile(settings._current_profile).suppress_health_check), diff --git a/hypothesis-python/tests/cover/test_core.py b/hypothesis-python/tests/cover/test_core.py index ecc9cd8170..8be19cc614 100644 --- a/hypothesis-python/tests/cover/test_core.py +++ b/hypothesis-python/tests/cover/test_core.py @@ -17,7 +17,10 @@ from hypothesis.database import InMemoryExampleDatabase from hypothesis.errors import InvalidArgument, NoSuchExample, Unsatisfiable +from tests.common.utils import Why, xfail_on_crosshair + +@xfail_on_crosshair(Why.other) def test_stops_after_max_examples_if_satisfying(): tracker = [] @@ -33,6 +36,7 @@ def track(x): assert len(tracker) == max_examples +@xfail_on_crosshair(Why.symbolic_outside_context) def test_stops_after_ten_times_max_examples_if_not_satisfying(): count = [0] diff --git a/hypothesis-python/tests/cover/test_filter_rewriting.py b/hypothesis-python/tests/cover/test_filter_rewriting.py index f9a2c46172..33a419142b 100644 --- a/hypothesis-python/tests/cover/test_filter_rewriting.py +++ b/hypothesis-python/tests/cover/test_filter_rewriting.py @@ -31,7 +31,7 @@ from hypothesis.strategies._internal.strings import BytesStrategy, TextStrategy from tests.common.debug import check_can_generate_examples -from tests.common.utils import fails_with +from tests.common.utils import Why, fails_with, xfail_on_crosshair A_FEW = 15 # speed up massively-parametrized tests @@ -374,9 +374,11 @@ def test_isidentifier_filter_properly_rewritten(al, data): assert example.isidentifier() -@pytest.mark.parametrize("al", ["¥¦§©"]) -def test_isidentifer_filter_unsatisfiable(al): - fs = st.text(alphabet=al).filter(str.isidentifier) +@xfail_on_crosshair(Why.no_unsatisfiable) # maybe a bug? +def test_isidentifer_filter_unsatisfiable(): + alphabet = "¥¦§©" + assert not any(f"_{c}".isidentifier() for c in alphabet) + fs = st.text(alphabet=alphabet).filter(str.isidentifier) with pytest.raises(Unsatisfiable): check_can_generate_examples(fs) @@ -571,6 +573,7 @@ def test_filter_rewriting_lambda_len_unique_elements( assert predicate(value) +@xfail_on_crosshair(Why.no_unsatisfiable) @pytest.mark.parametrize( "predicate", [ diff --git a/hypothesis-python/tests/cover/test_find.py b/hypothesis-python/tests/cover/test_find.py index 9eeddfb546..14d0ad3d20 100644 --- a/hypothesis-python/tests/cover/test_find.py +++ b/hypothesis-python/tests/cover/test_find.py @@ -12,7 +12,10 @@ from hypothesis import Phase, find, settings, strategies as st +from tests.common.utils import Why, xfail_on_crosshair + +@xfail_on_crosshair(Why.symbolic_outside_context) def test_find_uses_provided_random(): prev = None diff --git a/hypothesis-python/tests/cover/test_flakiness.py b/hypothesis-python/tests/cover/test_flakiness.py index 5a360ba025..046e94b624 100644 --- a/hypothesis-python/tests/cover/test_flakiness.py +++ b/hypothesis-python/tests/cover/test_flakiness.py @@ -17,13 +17,14 @@ from hypothesis.internal.scrutineer import Tracer from hypothesis.strategies import booleans, composite, integers, lists, random_module -from tests.common.utils import no_shrink +from tests.common.utils import Why, no_shrink, xfail_on_crosshair class Nope(Exception): pass +@xfail_on_crosshair(Why.symbolic_outside_context) def test_fails_only_once_is_flaky(): first_call = [True] @@ -40,6 +41,7 @@ def rude(x): assert isinstance(exceptions[0], Nope) +@xfail_on_crosshair(Why.symbolic_outside_context) def test_gives_flaky_error_if_assumption_is_flaky(): seen = set() @@ -78,6 +80,7 @@ def test(x): assert isinstance(exceptions[0], ZeroDivisionError) +@xfail_on_crosshair(Why.symbolic_outside_context) def test_does_not_attempt_to_shrink_flaky_errors(): values = [] @@ -109,6 +112,7 @@ def single_bool_lists(draw): return result +@xfail_on_crosshair(Why.nested_given) @example([True, False, False, False], [3], None) @example([False, True, False, False], [3], None) @example([False, False, True, False], [3], None) diff --git a/hypothesis-python/tests/cover/test_float_nastiness.py b/hypothesis-python/tests/cover/test_float_nastiness.py index 3f798c156f..5d7c2cc39b 100644 --- a/hypothesis-python/tests/cover/test_float_nastiness.py +++ b/hypothesis-python/tests/cover/test_float_nastiness.py @@ -26,6 +26,7 @@ ) from tests.common.debug import find_any, minimal +from tests.common.utils import Why, xfail_on_crosshair try: import numpy @@ -66,6 +67,7 @@ def test_does_not_generate_negative_if_right_boundary_is_positive(x): assert math.copysign(1, x) == 1 +@xfail_on_crosshair(Why.floats) @given(st.floats(-1.0, -0.0)) def test_does_not_generate_positive_if_right_boundary_is_negative(x): assert math.copysign(1, x) == -1 @@ -76,6 +78,7 @@ def test_half_bounded_generates_zero(): find_any(st.floats(max_value=1.0), lambda x: x == 0.0) +@xfail_on_crosshair(Why.floats) @given(st.floats(max_value=-0.0)) def test_half_bounded_respects_sign_of_upper_bound(x): assert math.copysign(1, x) == -1 diff --git a/hypothesis-python/tests/cover/test_given_error_conditions.py b/hypothesis-python/tests/cover/test_given_error_conditions.py index e417a6fc1a..0abe51a2bf 100644 --- a/hypothesis-python/tests/cover/test_given_error_conditions.py +++ b/hypothesis-python/tests/cover/test_given_error_conditions.py @@ -14,16 +14,14 @@ from hypothesis.errors import InvalidArgument, Unsatisfiable from hypothesis.strategies import booleans, integers, nothing -from tests.common.utils import fails_with +from tests.common.utils import Why, fails_with, xfail_on_crosshair -def test_raises_unsatisfiable_if_all_false_in_finite_set(): - @given(booleans()) - def test_assume_false(x): - reject() - - with pytest.raises(Unsatisfiable): - test_assume_false() +@xfail_on_crosshair(Why.no_unsatisfiable) +@fails_with(Unsatisfiable) +@given(booleans()) +def test_raises_unsatisfiable_if_all_false_in_finite_set(x): + reject() def test_does_not_raise_unsatisfiable_if_some_false_in_finite_set(): diff --git a/hypothesis-python/tests/cover/test_interactive_example.py b/hypothesis-python/tests/cover/test_interactive_example.py index 6a2f0daf8d..81c0a8b856 100644 --- a/hypothesis-python/tests/cover/test_interactive_example.py +++ b/hypothesis-python/tests/cover/test_interactive_example.py @@ -24,7 +24,7 @@ from hypothesis.internal.compat import WINDOWS from tests.common.debug import find_any -from tests.common.utils import fails_with, skipif_emscripten +from tests.common.utils import Why, fails_with, skipif_emscripten, xfail_on_crosshair pytest_plugins = "pytester" @@ -45,6 +45,7 @@ def test_exception_in_compare_can_still_have_example(): st.one_of(st.none().map(lambda n: Decimal("snan")), st.just(Decimal(0))).example() +@xfail_on_crosshair(Why.symbolic_outside_context) def test_does_not_always_give_the_same_example(): s = st.integers() assert len({s.example() for _ in range(100)}) >= 10 diff --git a/hypothesis-python/tests/cover/test_lookup.py b/hypothesis-python/tests/cover/test_lookup.py index 493f7fff84..096481f4b5 100644 --- a/hypothesis-python/tests/cover/test_lookup.py +++ b/hypothesis-python/tests/cover/test_lookup.py @@ -52,7 +52,7 @@ find_any, minimal, ) -from tests.common.utils import fails_with, temp_registered +from tests.common.utils import Why, fails_with, temp_registered, xfail_on_crosshair sentinel = object() BUILTIN_TYPES = tuple( diff --git a/hypothesis-python/tests/cover/test_reproduce_failure.py b/hypothesis-python/tests/cover/test_reproduce_failure.py index db3942f02d..18e5f85894 100644 --- a/hypothesis-python/tests/cover/test_reproduce_failure.py +++ b/hypothesis-python/tests/cover/test_reproduce_failure.py @@ -27,7 +27,7 @@ from hypothesis.core import decode_failure, encode_failure from hypothesis.errors import DidNotReproduce, InvalidArgument, UnsatisfiedAssumption -from tests.common.utils import capture_out, no_shrink +from tests.common.utils import Why, capture_out, no_shrink, xfail_on_crosshair @example(bytes(20)) # shorter compressed @@ -118,6 +118,7 @@ def test(x): test() +@xfail_on_crosshair(Why.symbolic_outside_context) def test_prints_reproduction_if_requested(): failing_example = [None] diff --git a/hypothesis-python/tests/cover/test_sampled_from.py b/hypothesis-python/tests/cover/test_sampled_from.py index ea852c2095..11450ba7e4 100644 --- a/hypothesis-python/tests/cover/test_sampled_from.py +++ b/hypothesis-python/tests/cover/test_sampled_from.py @@ -34,7 +34,7 @@ assert_simple_property, check_can_generate_examples, ) -from tests.common.utils import fails_with +from tests.common.utils import Why, fails_with, xfail_on_crosshair an_enum = enum.Enum("A", "a b c") a_flag = enum.Flag("A", "a b c") @@ -69,6 +69,7 @@ def test_unsat_filtered_sampling(x): raise AssertionError +@xfail_on_crosshair(Why.no_unsatisfiable) @fails_with(Unsatisfiable) @settings(suppress_health_check=[]) @given(sampled_from(range(2)).filter(lambda x: x < 0)) @@ -144,12 +145,14 @@ def test_efficient_sets_of_samples_with_chained_transformations_slow_path(x): assert x == {x * 2 for x in range(20) if x % 3} +@xfail_on_crosshair(Why.no_unsatisfiable) @fails_with(Unsatisfiable) @given(FilteredStrategy(st.sampled_from([None, False, ""]), conditions=(bool,))) def test_unsatisfiable_explicit_filteredstrategy_sampled(x): raise AssertionError("Unreachable because there are no valid examples") +@xfail_on_crosshair(Why.no_unsatisfiable) @fails_with(Unsatisfiable) @given(FilteredStrategy(st.none(), conditions=(bool,))) def test_unsatisfiable_explicit_filteredstrategy_just(x): diff --git a/hypothesis-python/tests/cover/test_searchstrategy.py b/hypothesis-python/tests/cover/test_searchstrategy.py index e545cb185a..719f92c1c6 100644 --- a/hypothesis-python/tests/cover/test_searchstrategy.py +++ b/hypothesis-python/tests/cover/test_searchstrategy.py @@ -25,7 +25,7 @@ from hypothesis.strategies._internal.utils import to_jsonable from tests.common.debug import assert_simple_property, check_can_generate_examples -from tests.common.utils import checks_deprecated_behaviour +from tests.common.utils import Why, checks_deprecated_behaviour, xfail_on_crosshair def test_or_errors_when_given_non_strategy(): @@ -69,6 +69,7 @@ def test_can_map(): assert_simple_property(s, lambda v: v == "foo") +@xfail_on_crosshair(Why.no_unsatisfiable) def test_example_raises_unsatisfiable_when_too_filtered(): with pytest.raises(Unsatisfiable): check_can_generate_examples(integers().filter(lambda x: False)) diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py index d85ced33cf..6631fbd74e 100644 --- a/hypothesis-python/tests/cover/test_settings.py +++ b/hypothesis-python/tests/cover/test_settings.py @@ -35,11 +35,13 @@ from hypothesis.utils.conventions import not_set from tests.common.utils import ( + Why, checks_deprecated_behaviour, counts_calls, fails_with, skipif_emscripten, validate_deprecation, + xfail_on_crosshair, ) @@ -297,6 +299,7 @@ def test_database_is_reference_preserved(): assert s.database is s.database +@xfail_on_crosshair(Why.other) @settings(verbosity=Verbosity.verbose) @example(x=99) @given(st.integers()) diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py index a8d02e9bc7..b02fcb1f90 100644 --- a/hypothesis-python/tests/cover/test_stateful.py +++ b/hypothesis-python/tests/cover/test_stateful.py @@ -42,7 +42,12 @@ ) from hypothesis.strategies import binary, data, integers, just, lists -from tests.common.utils import capture_out, validate_deprecation +from tests.common.utils import ( + Why, + capture_out, + validate_deprecation, + xfail_on_crosshair, +) from tests.nocover.test_stateful import DepthMachine NO_BLOB_SETTINGS = Settings(print_blob=False, phases=tuple(Phase)[:-1]) @@ -1175,6 +1180,8 @@ def teardown(self): assert self.a >= 2 +# Replay overruns after we trigger a crosshair.util.IgnoreAttempt exception for n=3 +@xfail_on_crosshair(Why.other) def test_min_steps_argument(): # You must pass a non-negative integer... for n_steps in (-1, "nan", 5.0): diff --git a/hypothesis-python/tests/cover/test_testdecorators.py b/hypothesis-python/tests/cover/test_testdecorators.py index 0cb9cd3c21..f225f78f1a 100644 --- a/hypothesis-python/tests/cover/test_testdecorators.py +++ b/hypothesis-python/tests/cover/test_testdecorators.py @@ -30,6 +30,7 @@ ) from tests.common.utils import ( + Why, assert_falsifying_output, capture_out, fails, @@ -37,6 +38,7 @@ no_shrink, raises, skipif_emscripten, + xfail_on_crosshair, ) # This particular test file is run under both pytest and nose, so it can't @@ -304,6 +306,7 @@ def test_has_ascii(x): assert any(c in ascii_characters for c in x) +@xfail_on_crosshair(Why.symbolic_outside_context, strict=False) def test_can_derandomize(): values = [] @@ -393,6 +396,7 @@ def test_mixed_text(x): assert set(x).issubset(set("abcdefg")) +@xfail_on_crosshair(Why.other, strict=False) # runs ~five failing examples def test_when_set_to_no_simplifies_runs_failing_example_twice(): failing = [] @@ -478,6 +482,7 @@ def test_empty_lists(xs): assert xs == [] +@xfail_on_crosshair(Why.other, strict=False) def test_given_usable_inline_on_lambdas(): xs = [] given(booleans())(lambda x: xs.append(x))() diff --git a/hypothesis-python/tests/cover/test_type_lookup.py b/hypothesis-python/tests/cover/test_type_lookup.py index 9d239496ef..49b0c2ac8b 100644 --- a/hypothesis-python/tests/cover/test_type_lookup.py +++ b/hypothesis-python/tests/cover/test_type_lookup.py @@ -33,7 +33,7 @@ check_can_generate_examples, find_any, ) -from tests.common.utils import fails_with, temp_registered +from tests.common.utils import Why, fails_with, temp_registered, xfail_on_crosshair types_with_core_strat = { type_ @@ -256,8 +256,19 @@ def _check_instances(t): ) +def maybe_mark(x): + if x.__name__ in "Match Decimal IPv4Address": + marks = xfail_on_crosshair(Why.other, as_marks=True, strict=False) + return pytest.param(x, marks=marks) + return x + + @pytest.mark.parametrize( - "typ", sorted((x for x in _global_type_lookup if _check_instances(x)), key=str) + "typ", + sorted( + (maybe_mark(x) for x in _global_type_lookup if _check_instances(x)), + key=str, + ), ) @given(data=st.data()) def test_can_generate_from_all_registered_types(data, typ): diff --git a/hypothesis-python/tests/datetime/test_dateutil_timezones.py b/hypothesis-python/tests/datetime/test_dateutil_timezones.py index e3afd61fbd..933b574a91 100644 --- a/hypothesis-python/tests/datetime/test_dateutil_timezones.py +++ b/hypothesis-python/tests/datetime/test_dateutil_timezones.py @@ -20,7 +20,7 @@ from hypothesis.strategies._internal.datetime import datetime_does_not_exist from tests.common.debug import assert_all_examples, find_any, minimal -from tests.common.utils import fails_with +from tests.common.utils import Why, fails_with, xfail_on_crosshair with warnings.catch_warnings(): if sys.version_info[:2] >= (3, 12): @@ -103,6 +103,7 @@ def test_datetimes_stay_within_naive_bounds(data, lo, hi): } +@xfail_on_crosshair(Why.other, strict=False) # fromutc argument must be a datetime @given(datetimes(timezones=timezones()) | datetimes(**DAY_WITH_IMAGINARY_HOUR_KWARGS)) def test_dateutil_exists_our_not_exists_are_inverse(value): assert datetime_does_not_exist(value) == (not tz.datetime_exists(value)) diff --git a/hypothesis-python/tests/datetime/test_pytz_timezones.py b/hypothesis-python/tests/datetime/test_pytz_timezones.py index aa0611c28d..b48987f0e9 100644 --- a/hypothesis-python/tests/datetime/test_pytz_timezones.py +++ b/hypothesis-python/tests/datetime/test_pytz_timezones.py @@ -21,6 +21,7 @@ from hypothesis.strategies._internal.datetime import datetime_does_not_exist from tests.common.debug import assert_all_examples, find_any, minimal +from tests.common.utils import Why, xfail_on_crosshair with warnings.catch_warnings(): if sys.version_info[:2] >= (3, 12): @@ -152,6 +153,7 @@ def test_datetimes_stay_within_naive_bounds(data, lo, hi): }, ], ) +@xfail_on_crosshair(Why.symbolic_outside_context, strict=False) def test_datetimes_can_exclude_imaginary(kw): # Sanity check: fail unless those days contain an imaginary hour to filter out find_any(datetimes(**kw, allow_imaginary=True), lambda x: not datetime_exists(x)) diff --git a/hypothesis-python/tests/nocover/test_baseexception.py b/hypothesis-python/tests/nocover/test_baseexception.py index 09c53a43bf..7f7656cb9a 100644 --- a/hypothesis-python/tests/nocover/test_baseexception.py +++ b/hypothesis-python/tests/nocover/test_baseexception.py @@ -14,6 +14,8 @@ from hypothesis.errors import Flaky, FlakyFailure from hypothesis.strategies import composite, integers, none +from tests.common.utils import Why, xfail_on_crosshair + @pytest.mark.parametrize( "e", [KeyboardInterrupt, SystemExit, GeneratorExit, ValueError] @@ -46,6 +48,7 @@ def test_do_nothing(x): test_do_nothing() +@xfail_on_crosshair(Why.other, strict=False) # extra replay from backend switch @pytest.mark.parametrize("e", [KeyboardInterrupt, ValueError]) def test_baseexception_no_rerun_no_flaky(e): runs = [0] diff --git a/hypothesis-python/tests/nocover/test_collective_minimization.py b/hypothesis-python/tests/nocover/test_collective_minimization.py index ebd846096b..0468415f86 100644 --- a/hypothesis-python/tests/nocover/test_collective_minimization.py +++ b/hypothesis-python/tests/nocover/test_collective_minimization.py @@ -16,8 +16,10 @@ from tests.common import standard_types from tests.common.debug import minimal +from tests.common.utils import Why, xfail_on_crosshair +@xfail_on_crosshair(Why.floats, strict=False) @pytest.mark.parametrize("spec", standard_types, ids=repr) def test_can_collectively_minimize(spec): n = 10 diff --git a/hypothesis-python/tests/nocover/test_compat.py b/hypothesis-python/tests/nocover/test_compat.py index b96dc87930..9cbb13396e 100644 --- a/hypothesis-python/tests/nocover/test_compat.py +++ b/hypothesis-python/tests/nocover/test_compat.py @@ -13,6 +13,8 @@ from hypothesis import given, strategies as st from hypothesis.internal.compat import ceil, floor, int_from_bytes, int_to_bytes +from tests.common.utils import Why, xfail_on_crosshair + @given(st.binary()) def test_convert_back(bs): @@ -38,6 +40,7 @@ def test_to_bytes_in_big_endian_order(x, y): assert int_to_bytes(x, 8) <= int_to_bytes(y, 8) +@xfail_on_crosshair(Why.other) @given(st.fractions()) def test_ceil(x): assert isinstance(ceil(x), int) @@ -45,6 +48,7 @@ def test_ceil(x): assert ceil(x) == math.ceil(x) +@xfail_on_crosshair(Why.other) @given(st.fractions()) def test_floor(x): assert isinstance(floor(x), int) diff --git a/hypothesis-python/tests/nocover/test_conjecture_engine.py b/hypothesis-python/tests/nocover/test_conjecture_engine.py index 3771960c7f..b3d9168dc7 100644 --- a/hypothesis-python/tests/nocover/test_conjecture_engine.py +++ b/hypothesis-python/tests/nocover/test_conjecture_engine.py @@ -14,10 +14,16 @@ from hypothesis.internal.conjecture.engine import ConjectureRunner from hypothesis.internal.conjecture.shrinker import Shrinker, node_program -from tests.common.utils import counts_calls, non_covering_examples +from tests.common.utils import ( + Why, + counts_calls, + non_covering_examples, + xfail_on_crosshair, +) from tests.conjecture.common import run_to_buffer, shrinking_from +@xfail_on_crosshair(Why.nested_given) def test_lot_of_dead_nodes(): @run_to_buffer def x(data): @@ -29,6 +35,7 @@ def x(data): assert x == bytes([0, 1, 2, 3]) +@xfail_on_crosshair(Why.nested_given) def test_saves_data_while_shrinking(monkeypatch): key = b"hi there" n = 5 @@ -58,6 +65,7 @@ def f(data): assert in_db == seen +@xfail_on_crosshair(Why.nested_given) def test_can_discard(monkeypatch): n = 8 @@ -79,6 +87,7 @@ def x(data): assert len(x) == n +@xfail_on_crosshair(Why.nested_given) @given(st.integers(0, 255), st.integers(0, 255)) def test_cached_with_masked_byte_agrees_with_results(byte_a, byte_b): def f(data): @@ -99,6 +108,7 @@ def f(data): assert (cached_a is cached_b) == (cached_a.buffer == data_b.buffer) +@xfail_on_crosshair(Why.other) def test_node_programs_fail_efficiently(monkeypatch): # Create 256 byte-sized blocks. None of the blocks can be deleted, and # every deletion attempt produces a different buffer. diff --git a/hypothesis-python/tests/nocover/test_database_usage.py b/hypothesis-python/tests/nocover/test_database_usage.py index ae029a5fe5..ba79bb7b9d 100644 --- a/hypothesis-python/tests/nocover/test_database_usage.py +++ b/hypothesis-python/tests/nocover/test_database_usage.py @@ -20,7 +20,12 @@ from hypothesis.errors import NoSuchExample, Unsatisfiable from hypothesis.internal.entropy import deterministic_PRNG -from tests.common.utils import all_values, non_covering_examples +from tests.common.utils import ( + Why, + all_values, + non_covering_examples, + xfail_on_crosshair, +) def has_a_non_zero_byte(x): @@ -39,6 +44,7 @@ def test_saves_incremental_steps_in_database(): assert len(all_values(database)) > 1 +@xfail_on_crosshair(Why.symbolic_outside_context, strict=False) def test_clears_out_database_as_things_get_boring(): key = b"a database key" database = InMemoryExampleDatabase() @@ -71,6 +77,7 @@ def stuff(): raise AssertionError +@xfail_on_crosshair(Why.other, strict=False) def test_trashes_invalid_examples(): key = b"a database key" database = InMemoryExampleDatabase() diff --git a/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py b/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py index e5ff3f8af0..29b60d0100 100644 --- a/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py +++ b/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py @@ -26,6 +26,8 @@ from hypothesis.internal.conjecture.data import Status from hypothesis.internal.conjecture.engine import ConjectureRunner +from tests.common.utils import Why, xfail_on_crosshair + @attr.s() class Write: @@ -113,6 +115,7 @@ def test(local_data): assume(runner.interesting_examples) +@xfail_on_crosshair(Why.nested_given) # technically nested-engine, but same problem @settings( suppress_health_check=list(HealthCheck), deadline=None, diff --git a/hypothesis-python/tests/nocover/test_given_error_conditions.py b/hypothesis-python/tests/nocover/test_given_error_conditions.py index 5ed442936a..d4e034c9bc 100644 --- a/hypothesis-python/tests/nocover/test_given_error_conditions.py +++ b/hypothesis-python/tests/nocover/test_given_error_conditions.py @@ -8,18 +8,16 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -import pytest - from hypothesis import HealthCheck, given, reject, settings from hypothesis.errors import Unsatisfiable from hypothesis.strategies import integers +from tests.common.utils import Why, fails_with, xfail_on_crosshair -def test_raises_unsatisfiable_if_all_false(): - @given(integers()) - @settings(max_examples=50, suppress_health_check=list(HealthCheck)) - def test_assume_false(x): - reject() - with pytest.raises(Unsatisfiable): - test_assume_false() +@xfail_on_crosshair(Why.no_unsatisfiable) +@fails_with(Unsatisfiable) +@given(integers()) +@settings(max_examples=50, suppress_health_check=list(HealthCheck)) +def test_raises_unsatisfiable_if_all_false(x): + reject() diff --git a/hypothesis-python/tests/nocover/test_integer_ranges.py b/hypothesis-python/tests/nocover/test_integer_ranges.py index 421fc7f956..3b8e3b9f70 100644 --- a/hypothesis-python/tests/nocover/test_integer_ranges.py +++ b/hypothesis-python/tests/nocover/test_integer_ranges.py @@ -11,7 +11,10 @@ from hypothesis import given, settings from hypothesis.strategies import integers +from tests.common.utils import Why, xfail_on_crosshair + +@xfail_on_crosshair(Why.symbolic_outside_context) def test_bounded_integers_distribution_of_bit_width_issue_1387_regression(): values = [] diff --git a/hypothesis-python/tests/nocover/test_limits.py b/hypothesis-python/tests/nocover/test_limits.py index 8b7762a778..6d26b33590 100644 --- a/hypothesis-python/tests/nocover/test_limits.py +++ b/hypothesis-python/tests/nocover/test_limits.py @@ -10,7 +10,10 @@ from hypothesis import given, settings, strategies as st +from tests.common.utils import Why, xfail_on_crosshair + +@xfail_on_crosshair(Why.other, strict=False) # might run fewer def test_max_examples_are_respected(): counter = [0] diff --git a/hypothesis-python/tests/nocover/test_nesting.py b/hypothesis-python/tests/nocover/test_nesting.py index 5aed586fe3..d44b8e42fc 100644 --- a/hypothesis-python/tests/nocover/test_nesting.py +++ b/hypothesis-python/tests/nocover/test_nesting.py @@ -12,9 +12,10 @@ from hypothesis import Verbosity, given, settings, strategies as st -from tests.common.utils import no_shrink +from tests.common.utils import Why, no_shrink, xfail_on_crosshair +@xfail_on_crosshair(Why.nested_given) def test_nesting_1(): @given(st.integers(0, 100)) @settings(max_examples=5, database=None, deadline=None) diff --git a/hypothesis-python/tests/nocover/test_randomization.py b/hypothesis-python/tests/nocover/test_randomization.py index 8877245e4d..6e79a2a08b 100644 --- a/hypothesis-python/tests/nocover/test_randomization.py +++ b/hypothesis-python/tests/nocover/test_randomization.py @@ -12,7 +12,7 @@ from hypothesis import Verbosity, core, find, given, settings, strategies as st -from tests.common.utils import no_shrink +from tests.common.utils import Why, no_shrink, xfail_on_crosshair def test_seeds_off_internal_random(): @@ -24,6 +24,7 @@ def test_seeds_off_internal_random(): assert x == y +@xfail_on_crosshair(Why.nested_given) def test_nesting_with_control_passes_health_check(): @given(st.integers(0, 100), st.random_module()) @settings(max_examples=5, database=None, deadline=None) diff --git a/hypothesis-python/tests/nocover/test_targeting.py b/hypothesis-python/tests/nocover/test_targeting.py index 25d74881f9..c8465d39cc 100644 --- a/hypothesis-python/tests/nocover/test_targeting.py +++ b/hypothesis-python/tests/nocover/test_targeting.py @@ -12,6 +12,8 @@ from hypothesis import Phase, given, seed, settings, strategies as st, target +from tests.common.utils import Why, xfail_on_crosshair + pytest_plugins = "pytester" TESTSUITE = """ @@ -57,6 +59,7 @@ def test_target_returns_value(a, b): assert isinstance(difference, int) +@xfail_on_crosshair(Why.symbolic_outside_context) def test_targeting_can_be_disabled(): strat = st.lists(st.integers(0, 255)) diff --git a/hypothesis-python/tests/nocover/test_testdecorators.py b/hypothesis-python/tests/nocover/test_testdecorators.py index 8d7f73bdc5..e8370d4256 100644 --- a/hypothesis-python/tests/nocover/test_testdecorators.py +++ b/hypothesis-python/tests/nocover/test_testdecorators.py @@ -15,7 +15,10 @@ from hypothesis import HealthCheck, given, reject, settings, strategies as st from hypothesis.errors import InvalidArgument, Unsatisfiable +from tests.common.utils import Why, xfail_on_crosshair + +@xfail_on_crosshair(Why.no_unsatisfiable) def test_contains_the_test_function_name_in_the_exception_string(): look_for_one = settings(max_examples=1, suppress_health_check=list(HealthCheck))