From c5ba49bd086df4bbc80016984b2a051bcf34546e Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 4 Nov 2024 13:48:56 -0900 Subject: [PATCH 01/11] use nox everywhere for testing --- .github/actions/install-pkg/action.yml | 24 ------------------------ .github/workflows/integration-test.yml | 8 ++++---- .github/workflows/test-mindeps.yml | 6 +++--- .github/workflows/test.yml | 9 +++++---- noxfile.py | 26 ++++++++++++++++++++++---- scripts/integration-test.sh | 16 ---------------- tests/integration/conftest.py | 12 +++++++++++- 7 files changed, 45 insertions(+), 56 deletions(-) delete mode 100644 .github/actions/install-pkg/action.yml delete mode 100755 scripts/integration-test.sh diff --git a/.github/actions/install-pkg/action.yml b/.github/actions/install-pkg/action.yml deleted file mode 100644 index 4b4e2d92..00000000 --- a/.github/actions/install-pkg/action.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Install earthaccess Python package -description: Install earthaccess Python package and testing dependencies - -inputs: - python-version: - description: Version of Python to use - required: true - -runs: - using: composite - steps: - - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - cache: pip - - - name: Display full python version - shell: bash - id: full-python-version - run: echo "version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")" >> $GITHUB_OUTPUT - - - name: Install package and test dependencies - shell: bash - run: pip install --root-user-action ignore ".[test]" diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 62c1043a..05e5bb2b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -94,16 +94,16 @@ jobs: # this point. ref: ${{ github.event.pull_request.head.sha }} - - name: Install package with dependencies - uses: ./.github/actions/install-pkg + - name: Setup nox + uses: wntrblm/nox@2024.10.09 with: - python-version: ${{ matrix.python-version }} + python-versions: ${{ matrix.python-version }} - name: Run integration tests env: EARTHDATA_USERNAME: ${{ secrets.EDL_USERNAME }} EARTHDATA_PASSWORD: ${{ secrets.EDL_PASSWORD }} - run: ./scripts/integration-test.sh + run: nox -s integration-tests -- --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO - name: Upload coverage report # Don't upload coverage when using the `act` tool to run the workflow locally diff --git a/.github/workflows/test-mindeps.yml b/.github/workflows/test-mindeps.yml index 3705c69b..f7de53d7 100644 --- a/.github/workflows/test-mindeps.yml +++ b/.github/workflows/test-mindeps.yml @@ -27,8 +27,8 @@ jobs: version: "0.4.7" enable-cache: true - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup nox + uses: wntrblm/nox@2024.10.09 with: python-version: 3.9 @@ -39,7 +39,7 @@ jobs: run: uv pip install --no-deps . - name: Test - run: uv run pytest tests/unit --cov=earthaccess --cov=tests --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO + run: nox -s test-min-deps -- --verbose --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO - name: Upload coverage # Don't upload coverage when using the `act` tool to run the workflow locally diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cd7b6cd3..7bc6bde6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,15 +19,16 @@ jobs: - name: Checkout sources uses: actions/checkout@v4 - - uses: ./.github/actions/install-pkg + - name: Setup nox + uses: wntrblm/nox@2024.10.09 with: - python-version: ${{ matrix.python-version }} + python-versions: ${{ matrix.python-version }} - name: Typecheck - run: mypy + run: nox -s typecheck - name: Test - run: pytest tests/unit --verbose --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO + run: nox -s tests -- --verbose --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO - name: Upload coverage # Don't upload coverage when using the `act` tool to run the workflow locally diff --git a/noxfile.py b/noxfile.py index badf6bec..23341d2a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -27,27 +27,45 @@ def tests(session: nox.Session) -> None: session.run( "pytest", "tests/unit", - "-rxXs", # Show provided reason in summary for (x)fail, (X)pass, and (s)kipped statuses + "-rxXs", # Show provided reason in summary for (x)fail, (X)pass, and (s)kipped tests *session.posargs, ) +@nox.session(name="test-min-deps", python="3.9") +def test_min_deps(session: nox.Session) -> None: + """Run the unit tests.""" + session.install("--resolution", "lowest-direct", "--editable", ".[test]") + session.run( + "pytest", + "tests/unit", + "-rxXs", # Show provided reason in summary for (x)fail, (X)pass, and (s)kipped tests + *session.posargs, + ) + + @nox.session(name="integration-tests") def integration_tests(session: nox.Session) -> None: """Run the integration tests.""" session.install("--editable", ".[test]") session.run( - "scripts/integration-test.sh", + "pytest", + "tests/integration", + "-rxXs", # Show provided reason in summary for (x)fail, (X)pass, and (s)kipped tests *session.posargs, env=dict( EARTHDATA_USERNAME=os.environ["EARTHDATA_USERNAME"], EARTHDATA_PASSWORD=os.environ["EARTHDATA_PASSWORD"], ), external=True, + # NOTE: integration test are permitted to pass if the failure rate was less than a hardcoded threshold. + # PyTest will return 99 if there were some failures, but less than the threshold. For more details, see: + # `pytest_sessionfinish` in tests/integration/conftest.py + success_codes=[0, 99], ) -@nox.session +@nox.session(name="build-pkg") def build_pkg(session: nox.Session) -> None: """Build a source distribution and binary distribution (wheel).""" build_path = DIR.joinpath("build") @@ -58,7 +76,7 @@ def build_pkg(session: nox.Session) -> None: session.run("python", "-m", "build") -@nox.session +@nox.session(name="serve-docs") def serve_docs(session: nox.Session) -> None: """Build the documentation and serve it.""" session.install("--editable", ".[docs]") diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh deleted file mode 100755 index 457e8469..00000000 --- a/scripts/integration-test.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -x -pytest tests/integration -rxXs --cov=earthaccess --cov-report=term-missing "${@}" --capture=no --tb=native --log-cli-level=INFO -RET=$? -set +x - -set -e -# NOTE: 99 is a special return code we selected for this case, it has no other meaning. -if [[ $RET == 99 ]]; then - echo -e "\e[0;31mWARNING: The integration test suite has been permitted to return 0 because the failure rate was less than a hardcoded threshold.\e[0m" - echo "For more details, see conftest.py." - exit 0 -elif [[ $RET != 0 ]]; then - exit $RET -fi diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8c206885..6d702a50 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,6 @@ import os import pathlib +from warnings import warn import earthaccess import pytest @@ -22,6 +23,9 @@ def pytest_sessionfinish(session, exitstatus): ratio will change depending on which tests are executed! E.g. executing integration tests and unit tests at the same time allows more tests to fail than executing integration tests alone. + + NOTE: The return exit code can be customized with the `EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE` + environment variable. """ if exitstatus != pytest.ExitCode.TESTS_FAILED: # Exit status 1 in PyTest indicates "Tests were collected and run but some of @@ -30,9 +34,15 @@ def pytest_sessionfinish(session, exitstatus): # want to defer to original pytest behavior. return + failure_rate = (100.0 * session.testsfailed) / session.testscollected if failure_rate <= ACCEPTABLE_FAILURE_RATE: - session.exitstatus = 99 + warn( + '\nWARNING: The integration test suite has been allowed to pass because the ' + 'failure rate was less than a hardcoded threshold. For more details see:\n' + 'tests/integration/conftest.py.' + ) + session.exitstatus = os.environ.get('EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE', 99) @pytest.fixture From 2ae222093154dd90b6ae12dd74b94904b07e520f Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 13:43:26 -0900 Subject: [PATCH 02/11] update docs --- docs/contributing/development.md | 49 +++++++++++++++++++++----------- noxfile.py | 2 +- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/docs/contributing/development.md b/docs/contributing/development.md index f97ae682..7f5a21ad 100644 --- a/docs/contributing/development.md +++ b/docs/contributing/development.md @@ -20,28 +20,45 @@ If you don't have pipx (pip for applications), then you can install with pip is reasonable). If you use macOS, then pipx and nox are both in brew, use `brew install pipx nox`. -To use, run `nox` without any arguments. This will run type checks and unit -tests using the installed version of Python on your system. +To use, run `nox` without any arguments. This will run the type check and unit +test "sessions" (tasks) using your local (and active) Python version. +Nox handles everything for you, including setting up a temporary virtual +environment for each run. +You can see all available sessions with `nox --list`: + +``` +$ nox --list +Sessions defined in earthaccess/noxfile.py: + +* typecheck -> Typecheck with mypy. +* tests -> Run the unit tests. +- test-min-deps -> Run the unit tests using the lowest compatible version of all direct dependencies. +- integration-tests -> Run the integration tests. +- build-pkg -> Build a source distribution and binary distribution (wheel). +- serve-docs -> Build the documentation and serve it. + +sessions marked with * are selected, sessions marked with - are skipped. +``` You can also run individual tasks (_sessions_ in `nox` parlance, hence the `-s` option below), like so: - -```console -nox -s typecheck # Run typechecks -nox -s tests # Run unit tests -nox -s integration-tests # Run integration tests (see note below) -nox -s serve_docs # Build and serve the docs -nox -s build_pkg # Build an SDist and Wheel +```bash +nox -s integration-tests +``` +and pass options to the underlying session like: +```bash +nox -s integration-tests -- [ARGS] ``` -Nox handles everything for you, including setting up a temporary virtual -environment for each run. +!!! tip + + In order to run integration tests locally, you must set the + environment variables `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` to your + username and password, respectively, of your + [NASA Earthdata](https://urs.earthdata.nasa.gov/) account (registration is + free). + -**NOTE:** In order to run integration tests locally, you must set the -environment variables `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` to your -username and password, respectively, of your -[NASA Earthdata](https://urs.earthdata.nasa.gov/) account (registration is -free). ## Manual development environment setup diff --git a/noxfile.py b/noxfile.py index 23341d2a..1386948b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -33,7 +33,7 @@ def tests(session: nox.Session) -> None: @nox.session(name="test-min-deps", python="3.9") def test_min_deps(session: nox.Session) -> None: - """Run the unit tests.""" + """Run the unit tests using the lowest compatible version of all direct dependencies.""" session.install("--resolution", "lowest-direct", "--editable", ".[test]") session.run( "pytest", From d728b6f7520176f93d68812e735a40544086dd83 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 13:49:45 -0900 Subject: [PATCH 03/11] explicit status code in warning --- tests/integration/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6d702a50..17458dfc 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -37,12 +37,13 @@ def pytest_sessionfinish(session, exitstatus): failure_rate = (100.0 * session.testsfailed) / session.testscollected if failure_rate <= ACCEPTABLE_FAILURE_RATE: + status_code = os.environ.get('EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE', 99) warn( - '\nWARNING: The integration test suite has been allowed to pass because the ' + f'\nWARNING: The integration test suite has returned {status_code} because the ' 'failure rate was less than a hardcoded threshold. For more details see:\n' 'tests/integration/conftest.py.' ) - session.exitstatus = os.environ.get('EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE', 99) + session.exitstatus = status_code @pytest.fixture From 433990a6fd40b19267541f027ad533cd650de85f Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 14:02:36 -0900 Subject: [PATCH 04/11] let nox handle setup --- .github/workflows/test-mindeps.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/test-mindeps.yml b/.github/workflows/test-mindeps.yml index f7de53d7..ff8b0bce 100644 --- a/.github/workflows/test-mindeps.yml +++ b/.github/workflows/test-mindeps.yml @@ -21,23 +21,11 @@ jobs: - name: Checkout source uses: actions/checkout@v4.1.1 - - name: Install uv - uses: astral-sh/setup-uv@v3 - with: - version: "0.4.7" - enable-cache: true - - name: Setup nox uses: wntrblm/nox@2024.10.09 with: python-version: 3.9 - - name: Install minimum-compatible dependencies - run: uv sync --resolution lowest-direct --extra test - - - name: Install earthaccess - run: uv pip install --no-deps . - - name: Test run: nox -s test-min-deps -- --verbose --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO From 883e80069f481613580ce88ae2eec9bd0de12ffc Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 14:03:41 -0900 Subject: [PATCH 05/11] typo --- .github/workflows/test-mindeps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mindeps.yml b/.github/workflows/test-mindeps.yml index ff8b0bce..0f012cf8 100644 --- a/.github/workflows/test-mindeps.yml +++ b/.github/workflows/test-mindeps.yml @@ -24,7 +24,7 @@ jobs: - name: Setup nox uses: wntrblm/nox@2024.10.09 with: - python-version: 3.9 + python-versions: 3.9 - name: Test run: nox -s test-min-deps -- --verbose --cov=earthaccess --cov-report=term-missing --capture=no --tb=native --log-cli-level=INFO From a7c238edc9660e6a64737779cc996189c57f67a5 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 14:18:01 -0900 Subject: [PATCH 06/11] force uv backend for test-min-deps --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 1386948b..40b0d420 100644 --- a/noxfile.py +++ b/noxfile.py @@ -31,7 +31,7 @@ def tests(session: nox.Session) -> None: *session.posargs, ) -@nox.session(name="test-min-deps", python="3.9") +@nox.session(name="test-min-deps", python="3.9", venv_backend="uv") def test_min_deps(session: nox.Session) -> None: """Run the unit tests using the lowest compatible version of all direct dependencies.""" session.install("--resolution", "lowest-direct", "--editable", ".[test]") From e4f101f214b427cbe741a879a19ef197d3b958eb Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 14:22:23 -0900 Subject: [PATCH 07/11] mak uv available on GH Actions runners --- .github/workflows/integration-test.yml | 5 +++++ .github/workflows/test-mindeps.yml | 5 +++++ .github/workflows/test.yml | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 05e5bb2b..bbd14157 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -94,6 +94,11 @@ jobs: # this point. ref: ${{ github.event.pull_request.head.sha }} + - name: Install uv + uses: astral-sh/setup-uv@v3.2.2 + with: + enable-cache: true + - name: Setup nox uses: wntrblm/nox@2024.10.09 with: diff --git a/.github/workflows/test-mindeps.yml b/.github/workflows/test-mindeps.yml index 0f012cf8..a2412708 100644 --- a/.github/workflows/test-mindeps.yml +++ b/.github/workflows/test-mindeps.yml @@ -21,6 +21,11 @@ jobs: - name: Checkout source uses: actions/checkout@v4.1.1 + - name: Install uv + uses: astral-sh/setup-uv@v3.2.2 + with: + enable-cache: true + - name: Setup nox uses: wntrblm/nox@2024.10.09 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7bc6bde6..487ab15f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,11 @@ jobs: - name: Checkout sources uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3.2.2 + with: + enable-cache: true + - name: Setup nox uses: wntrblm/nox@2024.10.09 with: From b1f1e52b687109053dbabd9db637c9dbba864989 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 14:23:56 -0900 Subject: [PATCH 08/11] ruff ruff --- noxfile.py | 2 +- tests/integration/conftest.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/noxfile.py b/noxfile.py index 40b0d420..f1db1c8c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -31,6 +31,7 @@ def tests(session: nox.Session) -> None: *session.posargs, ) + @nox.session(name="test-min-deps", python="3.9", venv_backend="uv") def test_min_deps(session: nox.Session) -> None: """Run the unit tests using the lowest compatible version of all direct dependencies.""" @@ -43,7 +44,6 @@ def test_min_deps(session: nox.Session) -> None: ) - @nox.session(name="integration-tests") def integration_tests(session: nox.Session) -> None: """Run the integration tests.""" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 17458dfc..7663c480 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -34,14 +34,13 @@ def pytest_sessionfinish(session, exitstatus): # want to defer to original pytest behavior. return - failure_rate = (100.0 * session.testsfailed) / session.testscollected if failure_rate <= ACCEPTABLE_FAILURE_RATE: - status_code = os.environ.get('EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE', 99) + status_code = os.environ.get("EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE", 99) warn( - f'\nWARNING: The integration test suite has returned {status_code} because the ' - 'failure rate was less than a hardcoded threshold. For more details see:\n' - 'tests/integration/conftest.py.' + f"\nWARNING: The integration test suite has returned {status_code} because the " + "failure rate was less than a hardcoded threshold. For more details see:\n" + "tests/integration/conftest.py." ) session.exitstatus = status_code From be7a416910fe76916e154264e683e9af1edc0e7a Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 14:27:21 -0900 Subject: [PATCH 09/11] more specificity to warning --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7663c480..1283f43d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -40,7 +40,7 @@ def pytest_sessionfinish(session, exitstatus): warn( f"\nWARNING: The integration test suite has returned {status_code} because the " "failure rate was less than a hardcoded threshold. For more details see:\n" - "tests/integration/conftest.py." + "`pytest_sessionfinish` in tests/integration/conftest.py." ) session.exitstatus = status_code From f654a3fb46b9ee4615d465704c03ddeea1d3fd0b Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 6 Nov 2024 14:36:54 -0900 Subject: [PATCH 10/11] Add changelog --- CHANGELOG.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e043d21..e33af427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,6 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ## [Unreleased] -- Fix `earthaccess.download` to not ignore errors by default - ([#581](https://github.com/nsidc/earthaccess/issues/581)) - ([**@Sherwin-14**](https://github.com/Sherwin-14), - [**@chuckwondo**](https://github.com/chuckwondo), - [**@mfisher87**](https://github.com/mfisher87)) - ### Changed - Use built-in `assert` statements instead of `unittest` assertions in @@ -25,16 +19,23 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) location ([#480](https://github.com/nsidc/earthaccess/issues/480)) ([@chuckwondo](https://github.com/chuckwondo)) - Add `nox` session for running integration tests locally - ([#815](https://github.com/nsidc/earthaccess/issues/815)) ([@chuckwondo](https://github.com/chuckwondo)) + ([#815](https://github.com/nsidc/earthaccess/issues/815); [@chuckwondo](https://github.com/chuckwondo) and ([#872](https://github.com/nsidc/earthaccess/issues/872); [@jhkennedy](https://github.com/jhkennedy)) - Auto-add comment to PR that requires maintainer to review and re-run integration tests ([#824](https://github.com/nsidc/earthaccess/issues/824)) ([@chuckwondo](https://github.com/chuckwondo)) ### Removed -### Fixed +- The `scripts/integration-test.sh` script has been removed in favor of the `integration-tests` nox session. + ([#872](https://github.com/nsidc/earthaccess/issues/872)) ([@jhkennedy](https://github.com/jhkennedy)) +### Fixed +- `earthaccess.download` will not ignore errors by default + ([#581](https://github.com/nsidc/earthaccess/issues/581)) + ([**@Sherwin-14**](https://github.com/Sherwin-14), + [**@chuckwondo**](https://github.com/chuckwondo), + [**@mfisher87**](https://github.com/mfisher87)) - Integration tests no longer clobber existing `.netrc` file ([#806](https://github.com/nsidc/earthaccess/issues/806)) (@chuckwondo) From a1d7b01b0e4f6aa9a701bde0e85c819bf6506f60 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 7 Nov 2024 10:07:18 -0900 Subject: [PATCH 11/11] Apply suggestions from code review Co-authored-by: Chuck Daniels --- docs/contributing/development.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/contributing/development.md b/docs/contributing/development.md index 7f5a21ad..1bd52211 100644 --- a/docs/contributing/development.md +++ b/docs/contributing/development.md @@ -40,12 +40,16 @@ Sessions defined in earthaccess/noxfile.py: sessions marked with * are selected, sessions marked with - are skipped. ``` + You can also run individual tasks (_sessions_ in `nox` parlance, hence the `-s` option below), like so: + ```bash nox -s integration-tests ``` + and pass options to the underlying session like: + ```bash nox -s integration-tests -- [ARGS] ```