diff --git a/README.md b/README.md index a5f43d4d..acbbb291 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ This is the test suite for array libraries adopting the [Python Array API standard](https://data-apis.org/array-api/latest). -Note the suite is still a **work in progress**. Feedback and contributions are -welcome! +Keeping full coverage of the spec is an on-going priority as the Array API evolves. +Feedback and contributions are welcome! ## Quickstart @@ -33,11 +33,23 @@ You need to specify the array library to test. It can be specified via the `ARRAY_API_TESTS_MODULE` environment variable, e.g. ```bash -$ export ARRAY_API_TESTS_MODULE=numpy.array_api +$ export ARRAY_API_TESTS_MODULE=array_api_strict ``` Alternately, import/define the `xp` variable in `array_api_tests/__init__.py`. +### Specifying the API version + +You can specify the API version to use when testing via the +`ARRAY_API_TESTS_VERSION` environment variable, e.g. + +```bash +$ export ARRAY_API_TESTS_VERSION="2023.12" +``` + +Currently this defaults to the array module's `__array_api_version__` value, and +if that attribute doesn't exist then we fallback to `"2021.12"`. + ### Run the suite Simply run `pytest` against the `array_api_tests/` folder to run the full suite. @@ -154,13 +166,6 @@ library to fail. ### Configuration -#### API version - -You can specify the API version to use when testing via the -`ARRAY_API_TESTS_VERSION` environment variable. Currently this defaults to the -array module's `__array_api_version__` value, and if that attribute doesn't -exist then we fallback to `"2021.12"`. - #### Data-dependent shapes Use the `--disable-data-dependent-shapes` flag to skip testing functions which have @@ -349,26 +354,6 @@ into a release. If you want, you can add release notes, which GitHub can generate for you. -## Future plans - -Keeping full coverage of the spec is an on-going priority as the Array API -evolves. - -Additionally, we have features and general improvements planned. Work on such -functionality is guided primarily by the concerete needs of developers -implementing and using the Array API—be sure to [let us -know](https://github.com/data-apis/array-api-tests/issues) any limitations you -come across. - -* A dependency graph for every test case, which could be used to modify pytest's - collection so that low-dependency tests are run first, and tests with faulty - dependencies would skip/xfail. - -* In some tests we've found it difficult to find appropaite assertion parameters - for output values (particularly epsilons for floating-point outputs), so we - need to review these and either implement assertions or properly note the lack - thereof. - --- 1The only exceptions to having just one primary test per function are: diff --git a/array_api_tests/stubs.py b/array_api_tests/stubs.py index 39bb1223..50f99c1c 100644 --- a/array_api_tests/stubs.py +++ b/array_api_tests/stubs.py @@ -44,7 +44,7 @@ category_to_funcs: Dict[str, List[FunctionType]] = {} for name, mod in name_to_mod.items(): - if name.endswith("_functions"): + if name.endswith("_functions") or name == "info": # info functions file just named info.py category = name.replace("_functions", "") objects = [getattr(mod, name) for name in mod.__all__] assert all(isinstance(o, FunctionType) for o in objects) # sanity check diff --git a/array_api_tests/test_linalg.py b/array_api_tests/test_linalg.py index 76dc08dd..807eaa87 100644 --- a/array_api_tests/test_linalg.py +++ b/array_api_tests/test_linalg.py @@ -37,6 +37,7 @@ from . import dtype_helpers as dh from . import pytest_helpers as ph from . import shape_helpers as sh +from . import api_version from .typing import Array from . import _array_module @@ -873,7 +874,7 @@ def test_trace(x, kw): # See https://github.com/data-apis/array-api-tests/issues/160 if x.dtype in dh.uint_dtypes: assert dh.is_int_dtype(res.dtype) # sanity check - else: + elif api_version < "2023.12": # TODO: update dtype assertion for >2023.12 - see #234 ph.assert_dtype("trace", in_dtype=x.dtype, out_dtype=res.dtype, expected=expected_dtype) n, m = x.shape[-2:] diff --git a/array_api_tests/test_statistical_functions.py b/array_api_tests/test_statistical_functions.py index f1ba2e1d..cb9843ed 100644 --- a/array_api_tests/test_statistical_functions.py +++ b/array_api_tests/test_statistical_functions.py @@ -11,7 +11,7 @@ from . import hypothesis_helpers as hh from . import pytest_helpers as ph from . import shape_helpers as sh -from . import xps +from . import api_version, xps from ._array_module import _UndefinedStub from .typing import DataType @@ -148,7 +148,7 @@ def test_prod(x, data): # See https://github.com/data-apis/array-api-tests/issues/106 if x.dtype in dh.uint_dtypes: assert dh.is_int_dtype(out.dtype) # sanity check - else: + elif api_version < "2023.12": # TODO: update dtype assertion for >2023.12 - see #234 ph.assert_dtype("prod", in_dtype=x.dtype, out_dtype=out.dtype, expected=expected_dtype) _axes = sh.normalise_axis(kw.get("axis", None), x.ndim) ph.assert_keepdimable_shape( @@ -207,7 +207,6 @@ def test_std(x, data): @pytest.mark.unvectorized -@pytest.mark.skip("flaky") # TODO: fix! @given( x=hh.arrays( dtype=xps.numeric_dtypes(), @@ -238,7 +237,7 @@ def test_sum(x, data): # See https://github.com/data-apis/array-api-tests/issues/160 if x.dtype in dh.uint_dtypes: assert dh.is_int_dtype(out.dtype) # sanity check - else: + elif api_version < "2023.12": # TODO: update dtype assertion for >2023.12 - see #234 ph.assert_dtype("sum", in_dtype=x.dtype, out_dtype=out.dtype, expected=expected_dtype) _axes = sh.normalise_axis(kw.get("axis", None), x.ndim) ph.assert_keepdimable_shape(