From 5597ec755d44cb005f01601b3c2193f9f56b604f Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Sun, 11 May 2025 12:56:49 +0200 Subject: [PATCH 1/5] CI: use ARRAY_API_TESTS_XFAIL_MARK on CI --- .github/workflows/array-api-tests-dask.yml | 2 ++ .github/workflows/array-api-tests-numpy-1-22.yml | 2 ++ .github/workflows/array-api-tests-numpy-1-26.yml | 2 ++ .github/workflows/array-api-tests-numpy-dev.yml | 2 ++ .github/workflows/array-api-tests-numpy-latest.yml | 2 ++ .github/workflows/array-api-tests-torch.yml | 1 + 6 files changed, 11 insertions(+) diff --git a/.github/workflows/array-api-tests-dask.yml b/.github/workflows/array-api-tests-dask.yml index afc67975..964fb52d 100644 --- a/.github/workflows/array-api-tests-dask.yml +++ b/.github/workflows/array-api-tests-dask.yml @@ -16,3 +16,5 @@ jobs: # the full test suite with at least 200 examples. pytest-extra-args: --max-examples=5 python-versions: '[''3.10'', ''3.13'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-1-22.yml b/.github/workflows/array-api-tests-numpy-1-22.yml index d8f60432..1cf6e26d 100644 --- a/.github/workflows/array-api-tests-numpy-1-22.yml +++ b/.github/workflows/array-api-tests-numpy-1-22.yml @@ -10,3 +10,5 @@ jobs: package-version: '== 1.22.*' xfails-file-extra: '-1-22' python-versions: '[''3.10'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-1-26.yml b/.github/workflows/array-api-tests-numpy-1-26.yml index 33780760..a2788d2f 100644 --- a/.github/workflows/array-api-tests-numpy-1-26.yml +++ b/.github/workflows/array-api-tests-numpy-1-26.yml @@ -10,3 +10,5 @@ jobs: package-version: '== 1.26.*' xfails-file-extra: '-1-26' python-versions: '[''3.10'', ''3.12'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-dev.yml b/.github/workflows/array-api-tests-numpy-dev.yml index d6de1a53..dce0813f 100644 --- a/.github/workflows/array-api-tests-numpy-dev.yml +++ b/.github/workflows/array-api-tests-numpy-dev.yml @@ -10,3 +10,5 @@ jobs: extra-requires: '--pre --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple' xfails-file-extra: '-dev' python-versions: '[''3.11'', ''3.13'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-latest.yml b/.github/workflows/array-api-tests-numpy-latest.yml index 4d3667f6..54b21a25 100644 --- a/.github/workflows/array-api-tests-numpy-latest.yml +++ b/.github/workflows/array-api-tests-numpy-latest.yml @@ -8,3 +8,5 @@ jobs: with: package-name: numpy python-versions: '[''3.10'', ''3.13'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-torch.yml b/.github/workflows/array-api-tests-torch.yml index ac20df25..4dcb3347 100644 --- a/.github/workflows/array-api-tests-torch.yml +++ b/.github/workflows/array-api-tests-torch.yml @@ -10,4 +10,5 @@ jobs: extra-requires: '--index-url https://download.pytorch.org/whl/cpu' extra-env-vars: | ARRAY_API_TESTS_SKIP_DTYPES=uint16,uint32,uint64 + ARRAY_API_TESTS_XFAIL_MARK=skip python-versions: '[''3.10'', ''3.13'']' From 3350f670e1b67a37888228c102e0e560f43077bd Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Sun, 11 May 2025 14:12:51 +0200 Subject: [PATCH 2/5] CI: run 500 examples on NumPy and PyTorch; 50 on Dask --- .github/workflows/array-api-tests-dask.yml | 2 +- .github/workflows/array-api-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/array-api-tests-dask.yml b/.github/workflows/array-api-tests-dask.yml index 964fb52d..a60b28a4 100644 --- a/.github/workflows/array-api-tests-dask.yml +++ b/.github/workflows/array-api-tests-dask.yml @@ -14,7 +14,7 @@ jobs: # workflow is barely more than a smoke test, and one should expect extreme # flakiness. Before changes to dask-xfails.txt or dask-skips.txt, please run # the full test suite with at least 200 examples. - pytest-extra-args: --max-examples=5 + pytest-extra-args: --max-examples=50 python-versions: '[''3.10'', ''3.13'']' extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index 31bedde6..f652438b 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -37,7 +37,7 @@ on: description: Multiline string of environment variables to set for the test run. env: - PYTEST_ARGS: "--max-examples 200 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 10" + PYTEST_ARGS: "--max-examples 500 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" jobs: tests: From 5b1ece468fb9b9b789304b57035de6801a39c7b1 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 12 May 2025 14:18:30 +0200 Subject: [PATCH 3/5] CI: use 4 workers; bump the # of examples to 1000 (np/torch), 200 (dask) --- .github/workflows/array-api-tests-dask.yml | 2 +- .github/workflows/array-api-tests-numpy-1-22.yml | 1 + .github/workflows/array-api-tests-numpy-1-26.yml | 1 + .github/workflows/array-api-tests-numpy-dev.yml | 1 + .github/workflows/array-api-tests-numpy-latest.yml | 1 + .github/workflows/array-api-tests-torch.yml | 1 + .github/workflows/array-api-tests.yml | 3 ++- 7 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/array-api-tests-dask.yml b/.github/workflows/array-api-tests-dask.yml index a60b28a4..ef430d9c 100644 --- a/.github/workflows/array-api-tests-dask.yml +++ b/.github/workflows/array-api-tests-dask.yml @@ -14,7 +14,7 @@ jobs: # workflow is barely more than a smoke test, and one should expect extreme # flakiness. Before changes to dask-xfails.txt or dask-skips.txt, please run # the full test suite with at least 200 examples. - pytest-extra-args: --max-examples=50 + pytest-extra-args: --max-examples=200 -n 4 python-versions: '[''3.10'', ''3.13'']' extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-1-22.yml b/.github/workflows/array-api-tests-numpy-1-22.yml index 1cf6e26d..83d4cf1d 100644 --- a/.github/workflows/array-api-tests-numpy-1-22.yml +++ b/.github/workflows/array-api-tests-numpy-1-22.yml @@ -10,5 +10,6 @@ jobs: package-version: '== 1.22.*' xfails-file-extra: '-1-22' python-versions: '[''3.10'']' + pytest-extra-args: -n 4 extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-1-26.yml b/.github/workflows/array-api-tests-numpy-1-26.yml index a2788d2f..13124644 100644 --- a/.github/workflows/array-api-tests-numpy-1-26.yml +++ b/.github/workflows/array-api-tests-numpy-1-26.yml @@ -10,5 +10,6 @@ jobs: package-version: '== 1.26.*' xfails-file-extra: '-1-26' python-versions: '[''3.10'', ''3.12'']' + pytest-extra-args: -n 4 extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-dev.yml b/.github/workflows/array-api-tests-numpy-dev.yml index dce0813f..dec4c7ae 100644 --- a/.github/workflows/array-api-tests-numpy-dev.yml +++ b/.github/workflows/array-api-tests-numpy-dev.yml @@ -10,5 +10,6 @@ jobs: extra-requires: '--pre --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple' xfails-file-extra: '-dev' python-versions: '[''3.11'', ''3.13'']' + pytest-extra-args: -n 4 extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-latest.yml b/.github/workflows/array-api-tests-numpy-latest.yml index 54b21a25..65bbc9a2 100644 --- a/.github/workflows/array-api-tests-numpy-latest.yml +++ b/.github/workflows/array-api-tests-numpy-latest.yml @@ -8,5 +8,6 @@ jobs: with: package-name: numpy python-versions: '[''3.10'', ''3.13'']' + pytest-extra-args: -n 4 extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-torch.yml b/.github/workflows/array-api-tests-torch.yml index 4dcb3347..4b4b945e 100644 --- a/.github/workflows/array-api-tests-torch.yml +++ b/.github/workflows/array-api-tests-torch.yml @@ -12,3 +12,4 @@ jobs: ARRAY_API_TESTS_SKIP_DTYPES=uint16,uint32,uint64 ARRAY_API_TESTS_XFAIL_MARK=skip python-versions: '[''3.10'', ''3.13'']' + pytest-extra-args: -n 4 diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index f652438b..53c1474d 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -37,7 +37,7 @@ on: description: Multiline string of environment variables to set for the test run. env: - PYTEST_ARGS: "--max-examples 500 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" + PYTEST_ARGS: "--max-examples 1000 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" jobs: tests: @@ -76,6 +76,7 @@ jobs: python -m pip install --upgrade pip python -m pip install '${{ inputs.package-name }} ${{ inputs.package-version }}' ${{ inputs.extra-requires }} python -m pip install -r ${GITHUB_WORKSPACE}/array-api-tests/requirements.txt + python -m pip install pytest-xdist - name: Dump pip environment run: pip freeze From 37d5d668674f10ae41709a7ebc3d2ab2ae6b25c4 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 12 May 2025 16:42:01 +0200 Subject: [PATCH 4/5] MAINT: update numpy-1.22 xfails --- numpy-1-22-xfails.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy-1-22-xfails.txt b/numpy-1-22-xfails.txt index cacb95b7..d4022b31 100644 --- a/numpy-1-22-xfails.txt +++ b/numpy-1-22-xfails.txt @@ -133,7 +133,7 @@ array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_sc array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[maximum] array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[copysign] array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[pow] -array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[pow] +array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[minimum] array_api_tests/test_operators_and_elementwise_functions.py::test_where_with_scalars From 43435808041951df2c7b7cae28204b3ce61f6e46 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 12 May 2025 16:42:29 +0200 Subject: [PATCH 5/5] MAINT: remove --ci pytest switch The warning says it's deprecated. --- .github/workflows/array-api-tests.yml | 2 +- numpy-1-22-xfails.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index 53c1474d..e832f870 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -37,7 +37,7 @@ on: description: Multiline string of environment variables to set for the test run. env: - PYTEST_ARGS: "--max-examples 1000 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" + PYTEST_ARGS: "--max-examples 1000 -v -rxXfE ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" jobs: tests: diff --git a/numpy-1-22-xfails.txt b/numpy-1-22-xfails.txt index d4022b31..e1c4f832 100644 --- a/numpy-1-22-xfails.txt +++ b/numpy-1-22-xfails.txt @@ -134,6 +134,7 @@ array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_sc array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[copysign] array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[pow] array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[minimum] +array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[atan2] array_api_tests/test_operators_and_elementwise_functions.py::test_where_with_scalars