From 56dc23cbecf4d9e708221ad91c2e57faf1a3f1c9 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 12 Aug 2024 13:25:23 -0600 Subject: [PATCH] chore(tox.ini): Simplify configuration, fix minor bugs (#5607) When referencing a command from another environment, it will cause errors when the other environment already exists. Fix it by avoiding indirection in environment command definitions. Additionally, simplify envoronment dependency management by defining two lists of dependencies: a default one with pinned versions for all environments, and an unpinned on for "tip" environments. Several dependencies have been missed in the mypy envornments, so this should make it easier by standardizing environment dependencies to be consistent across environments. --- pyproject.toml | 1 - tox.ini | 251 ++++++++++++++++++------------------------------- 2 files changed, 91 insertions(+), 161 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dbf31f33e55..da98e92b39c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,6 @@ build-backend = "setuptools.build_meta" line-length = 79 include = '(brpm|bddeb|\.py)$' - [tool.isort] profile = "black" line_length = 79 diff --git a/tox.ini b/tox.ini index d6982cbe382..3c95c91001b 100644 --- a/tox.ini +++ b/tox.ini @@ -15,28 +15,47 @@ package = skip basepython = python3 setenv = LC_ALL = en_US.utf-8 -passenv= +passenv = PYTEST_ADDOPTS HYPOTHESIS_PROFILE +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt -[format_deps] -black==22.3.0 -hypothesis==6.31.6 -hypothesis_jsonschema==0.20.1 -isort==5.10.1 -mypy==0.950 -pylint==3.2.0 -pytest==7.0.1 -ruff==0.4.3 -types-jsonschema==4.4.2 -types-Jinja2==2.11.9 -types-oauthlib==3.1.6 -types-passlib==1.7.7.12 -types-PyYAML==6.0.4 -types-requests==2.27.8 -types-setuptools==57.4.9 -typing-extensions==4.1.1 +[types] +deps = + # each release of type stubs relates to a specific version of a library + # so leave these unpinned + types-jsonschema + types-Jinja2 + types-oauthlib + types-passlib + types-PyYAML + types-requests + types-setuptools + typing-extensions +[pinned_versions] +deps = + {[types]deps} + black==24.8.0 + hypothesis==6.111.0 + hypothesis_jsonschema==0.23.1 + isort==5.13.2 + mypy==1.11.1 + pylint==3.2.6 + ruff==0.5.7 + +[latest_versions] +deps = + {[types]deps} + black + hypothesis + hypothesis_jsonschema + isort + mypy + pylint + ruff [files] schema = cloudinit/config/schemas/schema-cloud-config-v1.json @@ -45,100 +64,54 @@ network_v1 = cloudinit/config/schemas/schema-network-config-v1.json network_v2 = cloudinit/config/schemas/schema-network-config-v2.json [testenv:ruff] -deps = - ruff=={[format_deps]ruff} +deps = {[pinned_versions]deps} commands = {envpython} -m ruff check {posargs:.} [testenv:pylint] -deps = - pylint=={[format_deps]pylint} - -r{toxinidir}/test-requirements.txt - -r{toxinidir}/integration-requirements.txt +deps = {[pinned_versions]deps} commands = {envpython} -m pylint {posargs:.} [testenv:black] -deps = - black=={[format_deps]black} +deps = {[pinned_versions]deps} commands = {envpython} -m black --check {posargs:.} [testenv:isort] -deps = - isort=={[format_deps]isort} +deps = {[pinned_versions]deps} commands = {envpython} -m isort --check-only --diff {posargs:.} [testenv:mypy] deps = - -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt - -r{toxinidir}/doc-requirements.txt - hypothesis=={[format_deps]hypothesis} - hypothesis_jsonschema=={[format_deps]hypothesis_jsonschema} - mypy=={[format_deps]mypy} - types-jsonschema=={[format_deps]types-jsonschema} - types-Jinja2=={[format_deps]types-Jinja2} - types-passlib=={[format_deps]types-passlib} - types-pyyaml=={[format_deps]types-PyYAML} - types-oauthlib=={[format_deps]types-oauthlib} - types-requests=={[format_deps]types-requests} - types-setuptools=={[format_deps]types-setuptools} - typing-extensions=={[format_deps]typing-extensions} + {[testenv]deps} + {[pinned_versions]deps} commands = {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} [testenv:check_format] deps = - black=={[format_deps]black} - ruff=={[format_deps]ruff} - hypothesis=={[format_deps]hypothesis} - hypothesis_jsonschema=={[format_deps]hypothesis_jsonschema} - isort=={[format_deps]isort} - mypy=={[format_deps]mypy} - pylint=={[format_deps]pylint} - types-jsonschema=={[format_deps]types-jsonschema} - types-Jinja2=={[format_deps]types-Jinja2} - types-oauthlib=={[format_deps]types-oauthlib} - types-passlib=={[format_deps]types-passlib} - types-pyyaml=={[format_deps]types-PyYAML} - types-oauthlib=={[format_deps]types-oauthlib} - types-requests=={[format_deps]types-requests} - types-setuptools=={[format_deps]types-setuptools} - typing-extensions=={[format_deps]typing-extensions} - -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt - -r{toxinidir}/doc-requirements.txt + {[testenv]deps} + {[pinned_versions]deps} commands = - {[testenv:black]commands} - {[testenv:ruff]commands} - {[testenv:isort]commands} - {[testenv:mypy]commands} - {[testenv:pylint]commands} + {envpython} -m ruff check {posargs:.} + {envpython} -m pylint {posargs:.} + {envpython} -m black --check {posargs:.} + {envpython} -m isort --check-only --diff {posargs:.} + {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} [testenv:check_format_tip] deps = - black - ruff - hypothesis - hypothesis_jsonschema - isort - mypy - pylint - types-jsonschema - types-Jinja2 - types-oauthlib - types-passlib - types-pyyaml - types-oauthlib - types-requests - types-setuptools - -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt - -r{toxinidir}/doc-requirements.txt + {[testenv]deps} + {[latest_versions]deps} commands = - {[testenv:check_format]commands} + {envpython} -m ruff check {posargs:.} + {envpython} -m pylint {posargs:.} + {envpython} -m black --check {posargs:.} + {envpython} -m isort --check-only --diff {posargs:.} + {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} [testenv:do_format] -deps = - black=={[format_deps]black} - isort=={[format_deps]isort} +deps = {[pinned_versions]deps} commands = {envpython} -m isort . {envpython} -m black . @@ -148,35 +121,26 @@ commands = {envpython} -m json.tool --indent 2 {[files]network_v2} {[files]network_v2} [testenv:do_format_tip] -deps = - black - isort +deps = {[latest_versions]deps} commands = - {[testenv:do_format]commands} + {envpython} -m isort . + {envpython} -m black . + {envpython} -m json.tool --indent 2 {[files]schema} {[files]schema} + {envpython} -m json.tool --indent 2 {[files]version} {[files]version} + {envpython} -m json.tool --indent 2 {[files]network_v1} {[files]network_v1} + {envpython} -m json.tool --indent 2 {[files]network_v2} {[files]network_v2} [testenv:py3] -deps = - -r{toxinidir}/test-requirements.txt -commands = {envpython} -m pytest \ - -vvvv --showlocals \ - --durations 10 \ - -m "not hypothesis_slow" \ - --cov=cloudinit --cov-branch \ - {posargs:tests/unittests} +commands = {envpython} -m pytest -m "not hypothesis_slow" --cov=cloud-init --cov-branch {posargs:tests/unittests} -# experimental [testenv:py3-fast] deps = - -r{toxinidir}/test-requirements.txt + {[testenv]deps} pytest-xdist -commands = {envpython} -m pytest -n auto -m "not hypothesis_slow" -m "not serial"\ - {posargs:tests/unittests} +commands = {envpython} -m pytest -n auto -m "not hypothesis_slow" -m "not serial" {posargs:tests/unittests} [testenv:hypothesis-slow] -deps = - hypothesis==6.31.6 - hypothesis_jsonschema==0.20.1 - -r{toxinidir}/test-requirements.txt +deps = {[pinned_versions]deps} commands = {envpython} -m pytest \ -m hypothesis_slow \ --hypothesis-show-statistics \ @@ -184,11 +148,7 @@ commands = {envpython} -m pytest \ #commands = {envpython} -X tracemalloc=40 -Werror::ResourceWarning:cloudinit -m pytest \ [testenv:py3-leak] -deps = {[testenv:py3]deps} -commands = {envpython} -X tracemalloc=40 -Wall -m pytest \ - --durations 10 \ - --cov=cloudinit --cov-branch \ - {posargs:tests/unittests} +commands = {envpython} -X tracemalloc=40 -Wall -m pytest {posargs:tests/unittests} [testenv:lowest-supported] @@ -217,20 +177,17 @@ deps = attrs==17.4.0 responses==0.5.1 passlib -commands = {[testenv:py3]commands} +commands = {envpython} -m pytest -m "not hypothesis_slow" --cov=cloud-init --cov-branch {posargs:tests/unittests} [testenv:doc] -deps = - -r{toxinidir}/doc-requirements.txt +deps = -r{toxinidir}/doc-requirements.txt commands = {envpython} -m sphinx {posargs:-W doc/rtd doc/rtd_html} - doc8 doc/rtd + {envpython} -m doc8 doc/rtd [testenv:doc-spelling] -deps = - -r{toxinidir}/doc-requirements.txt -commands = - {envpython} -m sphinx -b spelling {posargs:-W doc/rtd doc/rtd_html} +deps = -r{toxinidir}/doc-requirements.txt +commands = {envpython} -m sphinx -b spelling {posargs:-W doc/rtd doc/rtd_html} # linkcheck shows false positives and has noisy output. # Despite these limitations, it is better than a manual search of the docs. @@ -240,61 +197,36 @@ commands = # # followed by manual verification of the links reported [testenv:linkcheck] -deps = - -r{toxinidir}/doc-requirements.txt +deps = -r{toxinidir}/doc-requirements.txt commands = {envpython} -m sphinx {posargs:-b linkcheck doc/rtd doc/rtd_html} [testenv:tip-ruff] -deps = ruff -commands = {[testenv:ruff]commands} +deps = {[latest_versions]deps} +commands = {envpython} -m ruff check {posargs:.} [testenv:tip-mypy] deps = - -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt - -r{toxinidir}/doc-requirements.txt - hypothesis - hypothesis_jsonschema - mypy - pytest - types-Jinja2 - types-jsonschema - types-oauthlib - types-PyYAML - types-passlib - types-pyyaml - types-oauthlib - types-requests - types-setuptools - typing-extensions + {[testenv]deps} + {[latest_versions]deps} commands = {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} [testenv:tip-pylint] -deps = - # requirements - pylint - # test-requirements - -r{toxinidir}/test-requirements.txt - -r{toxinidir}/integration-requirements.txt +deps = {[latest_versions]deps} commands = {envpython} -m pylint {posargs:.} - [testenv:tip-black] -deps = black -commands = {[testenv:black]commands} +deps = {[latest_versions]deps} +commands = {envpython} -m black --check {posargs:.} [testenv:tip-isort] -deps = isort -commands = {[testenv:isort]commands} +deps = {[latest_versions]deps} +commands = {envpython} -m isort --check-only --diff {posargs:.} [testenv:integration-tests] -commands = {envpython} -m pytest -vv \ - --log-cli-level=INFO \ - --durations 10 \ - {posargs:tests/integration_tests} -deps = - -r{toxinidir}/integration-requirements.txt +deps = -r{toxinidir}/integration-requirements.txt +commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests} passenv = CLOUD_INIT_* PYCLOUDLIB_* @@ -302,22 +234,21 @@ passenv = OS_* [testenv:integration-tests-ci] -commands = {[testenv:integration-tests]commands} -deps = {[testenv:integration-tests]deps} +deps = -r{toxinidir}/integration-requirements.txt +commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests} passenv = CLOUD_INIT_* SSH_AUTH_SOCK OS_* - TRAVIS setenv = PYTEST_ADDOPTS="-m ci and not adhoc" [testenv:integration-tests-jenkins] # Pytest's RC=1 means "Tests were collected and run but some of the tests failed". # Do not fail in this case, but let Jenkins handle it using the junit report. +deps = -r{toxinidir}/integration-requirements.txt allowlist_externals = sh -commands = sh -c "{envpython} -m pytest --log-cli-level=INFO -vv {posargs:tests/integration_tests/none} || [ $? -eq 1 ]" -deps = {[testenv:integration-tests]deps} +commands = sh -c "{envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests/none} || [ $? -eq 1 ]" passenv = *_proxy CLOUD_INIT_*