From 8839ec99a70f0e7581faf2644fb3f283031bc561 Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Wed, 26 Jul 2023 12:46:53 +0100 Subject: [PATCH 1/5] Update modular_pipelines.md in docs to make it clear that `params:` are namespaced (#2839) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update modular_pipelines.md * Update modular_pipelines.md * Update docs/source/nodes_and_pipelines/modular_pipelines.md Co-authored-by: Juan Luis Cano Rodríguez --------- Co-authored-by: Juan Luis Cano Rodríguez --- docs/source/nodes_and_pipelines/modular_pipelines.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/nodes_and_pipelines/modular_pipelines.md b/docs/source/nodes_and_pipelines/modular_pipelines.md index 4232b33f17..12d69509c6 100644 --- a/docs/source/nodes_and_pipelines/modular_pipelines.md +++ b/docs/source/nodes_and_pipelines/modular_pipelines.md @@ -241,9 +241,8 @@ final_pipeline = ( * `kedro run --namespace=` could be used to only run nodes with a specific namespace. ```{note} -Parameter references (`params:` and `parameters`) will not be namespaced. +`parameters` references will not be namespaced, but `params:` references will. ``` - ## How to use a modular pipeline with different parameters From c74114cce12285cc2d9418deae8aeba9e5b533b4 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Date: Wed, 26 Jul 2023 17:56:13 +0100 Subject: [PATCH 2/5] Pin pip < 23.2 and Port unit tests, lint and e2e tests to Github Actions unblock CI (#2843) * port tests to GA Signed-off-by: Ankita Katiyar * Revert changes to CircleCI config Signed-off-by: Ankita Katiyar * Revert changes to CircleCI config Signed-off-by: Ankita Katiyar * Suggestions from code review Signed-off-by: Ankita Katiyar * Suggestions from code review Signed-off-by: Ankita Katiyar * Create merge-gatekeeper.yml * Update release notes Signed-off-by: Ankita Katiyar * Update merge-gatekeeper.yml * Update merge-gatekeeper.yml * Update merge-gatekeeper.yml * Update merge-gatekeeper.yml * Update merge-gatekeeper.yml --------- Signed-off-by: Ankita Katiyar Co-authored-by: Nok Lam Chan --- .github/workflows/all-checks.yml | 44 +++++++++++++++++++++ .github/workflows/docs-only-checks.yml | 24 ++++++++++++ .github/workflows/e2e-tests.yml | 46 ++++++++++++++++++++++ .github/workflows/lint.yml | 32 ++++++++++++++++ .github/workflows/merge-gatekeeper.yml | 27 +++++++++++++ .github/workflows/unit-tests.yml | 53 ++++++++++++++++++++++++++ Makefile | 8 +++- RELEASE.md | 1 + features/environment.py | 4 +- pyproject.toml | 2 +- trufflehog-ignore.txt | 2 + 11 files changed, 240 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/all-checks.yml create mode 100644 .github/workflows/docs-only-checks.yml create mode 100644 .github/workflows/e2e-tests.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/merge-gatekeeper.yml create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml new file mode 100644 index 0000000000..9f7abf3163 --- /dev/null +++ b/.github/workflows/all-checks.yml @@ -0,0 +1,44 @@ +name: Run all checks on Kedro + +on: + push: + branches: + - main + paths-ignore: + - "docs/**" + pull_request: + branches: + - main + paths-ignore: + - "docs/**" + +jobs: + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.10" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + e2e-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/docs-only-checks.yml b/.github/workflows/docs-only-checks.yml new file mode 100644 index 0000000000..aa53fdd604 --- /dev/null +++ b/.github/workflows/docs-only-checks.yml @@ -0,0 +1,24 @@ +name: Run linter on Kedro Docs + +on: + push: + branches: + - main + paths: + - "docs/**" + pull_request: + branches: + - main + paths: + - "docs/**" + +jobs: + lint-tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000000..6c317c7c40 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,46 @@ +name: Run e2e-tests on Kedro + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string + +env: + COLUMNS: 120 + LINES: 25 + +jobs: + e2e-tests: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{inputs.python-version}} + - run: make install-pip-setuptools + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Install dependencies + run: | + pip --version + make install-test-requirements + make install-pre-commit + - name: pip freeze + run: pip freeze + - name: Run e2e tests + run: make e2e-tests diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..81712415fd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,32 @@ +name: Run linters on Kedro + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string + +jobs: + lint: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ inputs.python-version }} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Install dependencies + run: | + make install-test-requirements + make install-pre-commit + pip freeze + - name: Run linter + run: make lint diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml new file mode 100644 index 0000000000..4f5393a1e6 --- /dev/null +++ b/.github/workflows/merge-gatekeeper.yml @@ -0,0 +1,27 @@ +name: Merge Gatekeeper + +on: + pull_request: + branches: + - main + - develop + +jobs: + merge-gatekeeper: + runs-on: ubuntu-latest + # Restrict permissions of the GITHUB_TOKEN. + # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs + permissions: + checks: read + statuses: read + steps: + - name: Run Merge Gatekeeper + # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs: + # https://github.com/upsidr/merge-gatekeeper/tags + # https://github.com/upsidr/merge-gatekeeper/branches + uses: upsidr/merge-gatekeeper@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + timeout: 1800 + interval: 30 + ignored: 'ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circlecici: check-updated-files,regular' diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..87ca8e0eed --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,53 @@ +name: Run unit-tests on Kedro + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + unit-tests: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{inputs.python-version}} + - run: make install-pip-setuptools + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Install dependencies + run: | + make install-test-requirements + make install-pre-commit + - name: Install pytables (only for windows) + if: inputs.os == 'windows-latest' + run: pip install tables + - name: pip freeze + run: pip freeze + - name: Run unit tests + if: inputs.os == 'ubuntu-latest' && inputs.python-version == '3.10' + run: make test-sequential + - name: Run unit tests + if: inputs.os == 'ubuntu-latest' && inputs.python-version != '3.10' + run: make test + - name: Run unit tests (Windows) + if: inputs.os == 'windows-latest' && inputs.python-version == '3.10' + run: make test-no-spark-sequential + - name: Run unit tests (Windows) + if: inputs.os == 'windows-latest' && inputs.python-version != '3.10' + run: make test-no-spark diff --git a/Makefile b/Makefile index 5a1e85b558..bea1c7af02 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ clean: pre-commit clean || true install-pip-setuptools: - pip install -U "pip>=21.2" "setuptools>=65.5.1" wheel + python -m pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel lint: pre-commit run -a --hook-stage manual $(hook) @@ -19,6 +19,12 @@ test: test-no-spark: pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile +test-sequential: + pytest tests --cov-config pyproject.toml + +test-no-spark-sequential: + pytest tests --no-cov --ignore tests/extras/datasets/spark + test-no-datasets: pytest --no-cov --ignore tests/extras/datasets/ --numprocesses 4 --dist loadfile diff --git a/RELEASE.md b/RELEASE.md index 0c0c40fd5c..2fa06b789f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,6 +16,7 @@ ## Bug fixes and other changes * Consolidated dependencies and optional dependencies in `pyproject.toml`. +* Pin `pip<23.2` for CI due to a breaking change. See https://github.com/kedro-org/kedro/pull/2813 ## Documentation changes diff --git a/features/environment.py b/features/environment.py index 218ff097e4..671bee0178 100644 --- a/features/environment.py +++ b/features/environment.py @@ -103,7 +103,9 @@ def _setup_minimal_env(context): "pip", "install", "-U", - "pip>=21.2", + # pip==23.2 breaks pip-tools<7.0, and pip-tools>=7.0 does not support Python 3.7 + "pip>=21.2,<23.2; python_version < '3.8'", + "pip>=21.2; python_version >= '3.8'", "setuptools>=65.5.1", "wheel", ], diff --git a/pyproject.toml b/pyproject.toml index ae5fc96167..50bf4a8483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "more_itertools~=9.0", "omegaconf~=2.3", "parse~=1.19.0", - "pip-tools~=6.5", + "pip-tools>=6.5,<8", "pluggy~=1.0", "PyYAML>=4.2, <7.0", "rich>=12.0, <14.0", diff --git a/trufflehog-ignore.txt b/trufflehog-ignore.txt index 14719c544e..cb5551a327 100644 --- a/trufflehog-ignore.txt +++ b/trufflehog-ignore.txt @@ -9,3 +9,5 @@ static/img/kedro_gitflow.svg .coverage.* .*\.log .*\.iml +tests/extras/datasets/tensorflow/test_tensorflow_model_dataset.py +docs/source/meta/images/kedro_gitflow.svg From 638c01b64e4d3c9498ae03db8a234f12fea68d39 Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Wed, 26 Jul 2023 19:56:07 +0100 Subject: [PATCH 3/5] Introducing ruff as a linter to replace pylint, flake8 and isort (#2634) * first go Signed-off-by: Nok * Add isort rules Signed-off-by: Nok * update isort setting Signed-off-by: Nok * Add comments for pyproject ruff settings Signed-off-by: Nok * update pre-commit setting Signed-off-by: Nok * remove pylint from pre-commit Signed-off-by: Nok * remove linting library from test requirements Signed-off-by: Nok * fix bash script to skip pylint isort Signed-off-by: Nok * Update menitions to pylint, isort and flake8 Signed-off-by: Nok * Remove flake8 plugin section in the manifest Signed-off-by: Nok * First draft to update linting docs Signed-off-by: Nok * Apply suggestions from code review Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Fix subprocess.pipe with ruff recommendation Signed-off-by: Nok * Fix test asserting stdout,stderr to use capture_output=True Signed-off-by: Nok * fix more docs Signed-off-by: Nok * Add pylint rule sets and replace pylint disable with noqa Signed-off-by: Nok * Fix according to ruff, merge isinstance check PLR1701 Merge these isinstance calls: `isinstance(app.builder, (CheckExternalLinksBuilder, LaTeXBuilder))` Signed-off-by: Nok * fix test according to lint: tests/framework/session/test_session.py:588:38: PLC1901 `exception["value"] == ""` can be simplified to `not exception["value"]` as an empty string is falsey Signed-off-by: Nok * fix dataset linting Signed-off-by: Nok * fix config linting Signed-off-by: Nok * temporary commit for merge Signed-off-by: Nok * remove unused command in makefile Signed-off-by: Nok * Fix linting for runner Signed-off-by: Nok * fix micropkg lint Signed-off-by: Nok * Fix PLW2901 # noqa: redefined-loop-name Signed-off-by: Nok * PLR0913 - Too many arguments to function call Signed-off-by: Nok * PLR2004 - magic value Signed-off-by: Nok * linting Signed-off-by: Nok * kedro/framework/context/context.py:151:13: PLR5501 Use `elif` instead of `else` then `if`, to reduce indentation Signed-off-by: Nok * fix lint Signed-off-by: Nok * fix doc links Signed-off-by: Nok * Update config to show-fix for debugging Signed-off-by: Nok * Fix ruff args Signed-off-by: Nok * Update ruff version Signed-off-by: Nok * Update ruff versions Signed-off-by: Nok * Suppress sort import error for linting Signed-off-by: Nok --------- Signed-off-by: Nok Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> --- .pre-commit-config.yaml | 111 +++++++----------- Makefile | 1 - RELEASE.md | 1 + docs/source/conf.py | 3 +- .../developer_contributor_guidelines.md | 2 +- docs/source/development/commands_reference.md | 2 +- docs/source/development/linting.md | 57 +++++---- docs/source/kedro_project_setup/starters.md | 2 +- features/environment.py | 2 +- features/steps/cli_steps.py | 6 +- features/steps/sh_run.py | 4 +- features/steps/test_plugin/plugin.py | 4 +- .../pipelines/data_science/nodes.py | 2 +- kedro/config/common.py | 4 +- kedro/config/config.py | 2 +- kedro/config/omegaconf_config.py | 8 +- kedro/config/templated_config.py | 4 +- kedro/extras/datasets/api/api_dataset.py | 3 +- .../biosequence/biosequence_dataset.py | 3 +- kedro/extras/datasets/dask/parquet_dataset.py | 3 +- .../extras/datasets/email/message_dataset.py | 3 +- .../datasets/geopandas/geojson_dataset.py | 3 +- .../datasets/holoviews/holoviews_writer.py | 3 +- kedro/extras/datasets/json/json_dataset.py | 3 +- .../datasets/matplotlib/matplotlib_writer.py | 3 +- kedro/extras/datasets/networkx/gml_dataset.py | 3 +- .../datasets/networkx/graphml_dataset.py | 3 +- .../extras/datasets/networkx/json_dataset.py | 3 +- kedro/extras/datasets/pandas/csv_dataset.py | 3 +- kedro/extras/datasets/pandas/excel_dataset.py | 3 +- .../extras/datasets/pandas/feather_dataset.py | 3 +- kedro/extras/datasets/pandas/gbq_dataset.py | 6 +- .../extras/datasets/pandas/generic_dataset.py | 3 +- kedro/extras/datasets/pandas/hdf_dataset.py | 3 +- kedro/extras/datasets/pandas/json_dataset.py | 3 +- .../extras/datasets/pandas/parquet_dataset.py | 3 +- kedro/extras/datasets/pandas/sql_dataset.py | 2 +- kedro/extras/datasets/pandas/xml_dataset.py | 3 +- .../extras/datasets/pickle/pickle_dataset.py | 3 +- kedro/extras/datasets/pillow/image_dataset.py | 3 +- kedro/extras/datasets/plotly/json_dataset.py | 3 +- .../extras/datasets/plotly/plotly_dataset.py | 3 +- kedro/extras/datasets/redis/redis_dataset.py | 3 +- kedro/extras/datasets/spark/spark_dataset.py | 4 +- .../datasets/spark/spark_hive_dataset.py | 7 +- .../datasets/spark/spark_jdbc_dataset.py | 3 +- .../datasets/svmlight/svmlight_dataset.py | 3 +- .../tensorflow/tensorflow_model_dataset.py | 11 +- kedro/extras/datasets/yaml/yaml_dataset.py | 3 +- kedro/extras/extensions/ipython.py | 2 +- kedro/framework/cli/catalog.py | 8 +- kedro/framework/cli/cli.py | 4 +- kedro/framework/cli/hooks/manager.py | 4 +- kedro/framework/cli/jupyter.py | 12 +- kedro/framework/cli/micropkg.py | 31 +++-- kedro/framework/cli/pipeline.py | 12 +- kedro/framework/cli/project.py | 24 ++-- kedro/framework/cli/registry.py | 4 +- kedro/framework/cli/starters.py | 18 +-- kedro/framework/cli/utils.py | 28 ++--- kedro/framework/context/context.py | 13 +- kedro/framework/hooks/specs.py | 8 +- kedro/framework/project/__init__.py | 14 +-- kedro/framework/session/session.py | 15 ++- kedro/framework/startup.py | 2 +- kedro/io/__init__.py | 16 ++- kedro/io/cached_dataset.py | 4 +- kedro/io/core.py | 4 +- kedro/io/data_catalog.py | 10 +- kedro/io/lambda_dataset.py | 3 +- kedro/io/memory_dataset.py | 2 +- kedro/io/partitioned_dataset.py | 19 ++- kedro/ipython/__init__.py | 13 +- kedro/pipeline/modular_pipeline.py | 4 +- kedro/pipeline/node.py | 4 +- kedro/pipeline/pipeline.py | 7 +- kedro/runner/parallel_runner.py | 16 ++- kedro/runner/runner.py | 13 +- kedro/runner/thread_runner.py | 2 +- .../pipeline.py | 2 +- .../src/tests/test_run.py | 2 +- pyproject.toml | 16 +++ .../datasets/spark/test_spark_dataset.py | 6 +- tests/framework/cli/test_jupyter.py | 2 +- tests/framework/cli/test_project.py | 5 +- tests/framework/cli/test_starters.py | 2 +- tests/framework/session/test_session.py | 2 +- tests/io/test_data_catalog.py | 2 +- tests/pipeline/test_modular_pipeline.py | 2 +- tests/runner/test_parallel_runner.py | 12 +- tests/runner/test_thread_runner.py | 2 +- tools/print_env.sh | 2 - 92 files changed, 322 insertions(+), 377 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 163bff3f2f..57bafd2416 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,28 @@ repos: exclude: "^kedro/templates/|^features/steps/test_starter/" - id: requirements-txt-fixer # Sorts entries in requirements.txt exclude: "^kedro/templates/|^features/steps/test_starter/" + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.961 + hooks: + - id: mypy + args: [--allow-redefinition, --ignore-missing-imports] + exclude: | + (?x)( + ^kedro/templates/| + ^docs/| + ^features/steps/test_starter/ + ) + additional_dependencies: + - types-cachetools + - types-filelock + - types-PyYAML + - types-redis + - types-requests + - types-setuptools + - types-toml + - attrs + - repo: https://github.com/asottile/blacken-docs rev: v1.12.1 hooks: @@ -26,20 +48,28 @@ repos: additional_dependencies: [black~=22.0] entry: blacken-docs --skip-errors - - repo: https://github.com/asottile/pyupgrade - rev: v2.26.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.0.277 + hooks: + - id: ruff + name: "ruff on kedro/" + args: ["--fix", "--show-fixes", "--exit-non-zero-on-fix"] + exclude: "^kedro/templates/|^features/steps/test_starter/|tests|docs" + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.0.277 hooks: - - id: pyupgrade - args: [--py36-plus] + - id: ruff + name: "ruff on tests/ and docs/" + # PLR2004: Magic value used + # PLR0913: Too many arguments + args: ["--fix", "--show-fixes", "--exit-non-zero-on-fix", "--ignore=PLR2004,PLR0913"] + # include: "tests" + exclude: "^kedro/templates/|^features/steps/test_starter/|kedro" - repo: local hooks: - - id: isort - name: "Sort imports" - language: system - types: [file, python] - exclude: ^kedro/templates/|^features/steps/test_starter - entry: isort - id: black name: "Black" language: system @@ -53,65 +83,6 @@ repos: pass_filenames: false entry: lint-imports - - # It's impossible to specify per-directory configuration, so we just run it many times. - # https://github.com/PyCQA/pylint/issues/618 - # The first set of pylint checks if for local pre-commit, it only runs on the files changed. - - id: pylint-quick-kedro - name: "Quick Pylint on kedro/*" - language: system - types: [file, python] - files: ^kedro/ - exclude: ^kedro/templates/ - entry: pylint -j 4 --disable=unnecessary-pass - stages: [commit] - - id: pylint-quick-tests - name: "Quick Pylint on tests/*" - language: system - types: [file, python] - files: ^tests/ - entry: pylint -j 4 --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,too-many-public-methods - stages: [commit] - - id: pylint-quick-features - name: "Quick Pylint on features/*" - language: system - types: [file, python] - files: ^features/ - exclude: ^features/steps/test_starter - entry: pylint -j 4 --disable=missing-docstring,no-name-in-module - stages: [commit] - - # The same pylint checks, but running on all files. It's for manual run with `make lint` - - id: pylint-kedro - name: "Pylint on kedro/*" - language: system - pass_filenames: false - stages: [manual] - entry: pylint -j 4 --disable=unnecessary-pass --init-hook="import sys; sys.setrecursionlimit(2000)" kedro - - id: pylint-tests - name: "Pylint on tests/*" - language: system - pass_filenames: false - stages: [manual] - entry: pylint -j 4 --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,too-many-public-methods,use-implicit-booleaness-not-comparison tests - - id: pylint-features - name: "Pylint on features/*" - language: system - pass_filenames: false - stages: [manual] - exclude: ^features/steps/test_starter - entry: pylint -j 4 --disable=missing-docstring,no-name-in-module features - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.3 - hooks: - - id: flake8 - args: - - "--max-line-length=88" - - "--max-complexity=18" - - "--max-complexity=18" - - "--select=B,C,E,F,W,T4,B9" - - "--ignore=E203,E266,E501,W503" - exclude: "^kedro/templates/|^features/steps/test_starter/" - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.961 hooks: @@ -147,3 +118,5 @@ repos: types: [file, python] exclude: ^kedro/templates/|^tests/|^features/steps/test_starter entry: bandit -ll + +# Manual only diff --git a/Makefile b/Makefile index bea1c7af02..2e8436390b 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,6 @@ install-pip-setuptools: lint: pre-commit run -a --hook-stage manual $(hook) - test: pytest --numprocesses 4 --dist loadfile diff --git a/RELEASE.md b/RELEASE.md index 2fa06b789f..01a9d1ed67 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -19,6 +19,7 @@ * Pin `pip<23.2` for CI due to a breaking change. See https://github.com/kedro-org/kedro/pull/2813 ## Documentation changes +- Recommended `ruff` as the linter and remove mentions of `pylint`, `isort`, `flake8`. ## Breaking changes to the API diff --git a/docs/source/conf.py b/docs/source/conf.py index 346ccaa8c6..5accc21725 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -499,8 +499,7 @@ def _add_jinja_filters(app): # LaTeXBuilder is used in the PDF docs build, # and it doesn't have attribute 'templates' if not ( - isinstance(app.builder, LaTeXBuilder) - or isinstance(app.builder, CheckExternalLinksBuilder) + isinstance(app.builder, (LaTeXBuilder,CheckExternalLinksBuilder)) ): app.builder.templates.environment.filters["env_override"] = env_override diff --git a/docs/source/contribution/developer_contributor_guidelines.md b/docs/source/contribution/developer_contributor_guidelines.md index 667f1a2dbd..787a838d90 100644 --- a/docs/source/contribution/developer_contributor_guidelines.md +++ b/docs/source/contribution/developer_contributor_guidelines.md @@ -131,7 +131,7 @@ We will work with you to complete your contribution, but we reserve the right to ``` Ensure that your PR builds cleanly before you submit it, by running the CI/CD checks locally, as follows: -* `make lint`: PEP-8 Standards (`pylint`, `flake8`) +* `make lint`: PEP-8 Standards (`ruff`, `black`) * `make test`: unit tests, 100% coverage (`pytest`, `pytest-cov`) * `make e2e-tests`: end-to-end tests (`behave`) diff --git a/docs/source/development/commands_reference.md b/docs/source/development/commands_reference.md index ae2933e256..99f6892bdc 100644 --- a/docs/source/development/commands_reference.md +++ b/docs/source/development/commands_reference.md @@ -402,7 +402,7 @@ The `build-docs` command builds [project documentation](../tutorial/package_a_pr #### Lint your project ```{note} -_This command will be deprecated from Kedro version 0.19.0._ +_This command will be deprecated from Kedro version 0.19.0._. We still recommend to (../development/linting.md) and you can find more help here ``` ```bash diff --git a/docs/source/development/linting.md b/docs/source/development/linting.md index e2c0f31037..d795086b51 100644 --- a/docs/source/development/linting.md +++ b/docs/source/development/linting.md @@ -10,25 +10,23 @@ As a project grows and goes through various stages of development it becomes imp ## Set up Python tools There are a variety of Python tools available to use with your Kedro projects. This guide shows you how to use -[`black`](https://github.com/psf/black), [`flake8`](https://github.com/PyCQA/flake8), and -[`isort`](https://github.com/PyCQA/isort) to format and lint your Kedro projects. +[`black`](https://github.com/psf/black), [`ruff`](https://beta.ruff.rs). - **`black`** is a [PEP 8](https://peps.python.org/pep-0008/) compliant opinionated Python code formatter. `black` can check for styling inconsistencies and reformat your files in place. [You can read more in the `black` documentation](https://black.readthedocs.io/en/stable/). -- **`flake8`** is a wrapper around [`pep8`](https://pypi.org/project/pep8/), -[`pyflakes`](https://pypi.org/project/pyflakes/), and [`mccabe`](https://pypi.org/project/mccabe/) which can lint code and format it with respect to [PEP 8](https://peps.python.org/pep-0008/), -and check the [cyclomatic complexity](https://www.ibm.com/docs/en/raa/6.1?topic=metrics-cyclomatic-complexity) of your code base. -[You can read more in the `flake8` documentation](https://flake8.pycqa.org/en/latest/). -- **`isort`** is a Python library used to reformat code by sorting imports alphabetically and automatically separating them into sections by +- **`ruff`** is a fast linter that replaces `flake8`, `pylint`, `pyupgrade`, `isort` and [more](https://beta.ruff.rs/docs/rules/). + - It helps to make your code compliant to [`pep8`](https://pypi.org/project/pep8/). + - It reformats code by sorting imports alphabetically and automatically separating them into sections by type. [You can read more in the `isort` documentation](https://pycqa.github.io/isort/). + ### Install the tools -Install `black`, `flake8`, and `isort` by adding the following lines to your project's `src/requirements.txt` +Install `black` and `ruff` by adding the following lines to your project's `src/requirements.txt` file: ```text black # Used for formatting code -flake8 # Used for linting and formatting -isort # Used for formatting code (sorting module imports) +ruff # Used for linting, formatting and sorting module imports + ``` To install all the project-specific dependencies, including the linting tools, navigate to the root directory of the project and run: @@ -37,10 +35,29 @@ pip install -r src/requirements.txt ``` Alternatively, you can individually install the linting tools using the following shell commands: ```bash -pip install black -pip install flake8 -pip install isort +pip install black ruff ``` +#### Configure `ruff` +`ruff` read configurations from `pyproject.toml` within your project root. You can enable different rule sets within the `[tool.ruff]` section. For example, the rule set `F` is equivalent to `Pyflakes`. + +To start with `ruff`, we recommend adding this section to enable a few basic rules sets. +```toml +[tool.ruff] +select = [ + "F", # Pyflakes + "E", # Pycodestyle + "W", # Pycodestyle + "UP", # pyupgrade + "I", # isort + "PL", # Pylint +] +ignore = ["E501"] # Black take care off line-too-long +``` + +```{note} +It is a good practice to [split your line when it is too long](https://beta.ruff.rs/docs/rules/line-too-long/), so it can be read easily even in a small screen. `ruff` treats this slightly different from `black`, when using together we recommend to disable this rule, i.e. `E501` to avoid conflicts. +``` + #### Configure `flake8` Store your `flake8` configuration in a file named `setup.cfg` within your project root. The Kedro starters use the [following configuration](https://github.com/kedro-org/kedro-starters/blob/main/pandas-iris/%7B%7B%20cookiecutter.repo_name%20%7D%7D/setup.cfg): @@ -87,17 +104,11 @@ you want to run before each `commit`. Below is a sample `YAML` file with entries for `black`,`flake8`, and `isort`: ```yaml repos: - - repo: https://github.com/pycqa/isort - rev: 5.10.1 - hooks: - - id: isort - name: isort (python) - args: ["--profile", "black"] - - - repo: https://github.com/pycqa/flake8 - rev: '' # pick a git hash / tag to point to + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.0.270 hooks: - - id: flake8 + - id: ruff - repo: https://github.com/psf/black rev: 22.8.0 diff --git a/docs/source/kedro_project_setup/starters.md b/docs/source/kedro_project_setup/starters.md index aac0dc331f..4ad60cb0c2 100644 --- a/docs/source/kedro_project_setup/starters.md +++ b/docs/source/kedro_project_setup/starters.md @@ -155,7 +155,7 @@ Here is the layout of the project as a Cookiecutter template: ├── docs # Project documentation ├── notebooks # Project related Jupyter notebooks (can be used for experimental code before moving the code to src) ├── README.md # Project README -├── setup.cfg # Configuration options for tools e.g. `pytest` or `flake8` +├── setup.cfg # Configuration options for tools e.g. `pytest` or `black` └── src # Project source code └── {{ cookiecutter.python_package }} ├── __init.py__ diff --git a/features/environment.py b/features/environment.py index 671bee0178..172dfd006a 100644 --- a/features/environment.py +++ b/features/environment.py @@ -1,5 +1,5 @@ """Behave environment setup commands.""" -# pylint: disable=unused-argument +# noqa: unused-argument from __future__ import annotations import os diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index f4f6e887fd..f5931828fa 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -229,7 +229,7 @@ def add_test_jupyter_nb(context): """Create a test jupyter notebook using TEST_JUPYTER_ORG.""" with open( str(context.root_project_dir / "notebooks" / "hello_world.ipynb"), - "wt", + "w", encoding="utf-8", ) as test_nb_fh: test_nb_fh.write(TEST_JUPYTER_ORG) @@ -366,7 +366,7 @@ def simulate_nb_execution(context): """ with open( str(context.root_project_dir / "notebooks" / "hello_world.ipynb"), - "wt", + "w", encoding="utf-8", ) as test_nb_fh: test_nb_fh.write(TEST_JUPYTER_AFTER_EXEC) @@ -554,7 +554,7 @@ def check_additional_cell_added(context): encoding="utf-8", ) as test_nb_fh: context.nb_data = json.load(test_nb_fh) - assert len(context.nb_data["cells"]) == 2 + assert len(context.nb_data["cells"]) == 2 # noqa: PLR2004 @then("the output should be empty in all the cells in the jupyter notebook") diff --git a/features/steps/sh_run.py b/features/steps/sh_run.py index 7e49e7a0ca..476cee7106 100644 --- a/features/steps/sh_run.py +++ b/features/steps/sh_run.py @@ -37,9 +37,7 @@ def run( if isinstance(cmd, str) and split: cmd = shlex.split(cmd) # pylint: disable=subprocess-run-check - result = subprocess.run( - cmd, input="", stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) + result = subprocess.run(cmd, input="", capture_output=True, **kwargs) result.stdout = result.stdout.decode("utf-8") result.stderr = result.stderr.decode("utf-8") if print_output: diff --git a/features/steps/test_plugin/plugin.py b/features/steps/test_plugin/plugin.py index 02ef228b3e..277fb1f18e 100644 --- a/features/steps/test_plugin/plugin.py +++ b/features/steps/test_plugin/plugin.py @@ -11,9 +11,7 @@ class MyPluginHook: @hook_impl - def after_catalog_created( - self, catalog - ): # pylint: disable=unused-argument, no-self-use + def after_catalog_created(self, catalog): # noqa: unused-argument, no-self-use logger.info("Reached after_catalog_created hook") diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_science/nodes.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_science/nodes.py index 41efe547cd..9217c2d3b1 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_science/nodes.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_science/nodes.py @@ -3,7 +3,7 @@ Delete this when you start working on your own Kedro project. """ -# pylint: disable=invalid-name +# noqa: invalid-name from __future__ import annotations import logging diff --git a/kedro/config/common.py b/kedro/config/common.py index afa823cc9d..35bcdcda89 100644 --- a/kedro/config/common.py +++ b/kedro/config/common.py @@ -124,7 +124,7 @@ def _load_config_file( Parsed configuration. """ # for performance reasons - import anyconfig # pylint: disable=import-outside-toplevel + import anyconfig # noqa: import-outside-toplevel try: # Default to UTF-8, which is Python 3 default encoding, to decode the file @@ -230,7 +230,7 @@ def _check_duplicate_keys( if overlapping_keys: sorted_keys = ", ".join(sorted(overlapping_keys)) - if len(sorted_keys) > 100: + if len(sorted_keys) > 100: # noqa: PLR2004 sorted_keys = sorted_keys[:100] + "..." duplicates.append(f"{processed_file}: {sorted_keys}") diff --git a/kedro/config/config.py b/kedro/config/config.py index f50e4b8f52..1af44fc4ab 100644 --- a/kedro/config/config.py +++ b/kedro/config/config.py @@ -66,7 +66,7 @@ class ConfigLoader(AbstractConfigLoader): """ - def __init__( + def __init__( # noqa: too-many-arguments self, conf_source: str, env: str = None, diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 155d1c8295..d7d9bd245b 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -73,7 +73,7 @@ class OmegaConfigLoader(AbstractConfigLoader): """ - def __init__( + def __init__( # noqa: too-many-arguments self, conf_source: str, env: str = None, @@ -206,7 +206,7 @@ def __repr__(self): # pragma: no cover f"config_patterns={self.config_patterns})" ) - def load_and_merge_dir_config( # pylint: disable=too-many-arguments + def load_and_merge_dir_config( # noqa: too-many-arguments self, conf_path: str, patterns: Iterable[str], @@ -233,7 +233,7 @@ def load_and_merge_dir_config( # pylint: disable=too-many-arguments Resulting configuration dictionary. """ - # pylint: disable=too-many-locals + # noqa: too-many-locals if not self._fs.isdir(Path(conf_path).as_posix()): raise MissingConfigException( @@ -319,7 +319,7 @@ def _check_duplicates(seen_files_to_keys: dict[Path, set[Any]]): if overlapping_keys: sorted_keys = ", ".join(sorted(overlapping_keys)) - if len(sorted_keys) > 100: + if len(sorted_keys) > 100: # noqa: PLR2004 sorted_keys = sorted_keys[:100] + "..." duplicates.append( f"Duplicate keys found in {filepath1} and {filepath2}: {sorted_keys}" diff --git a/kedro/config/templated_config.py b/kedro/config/templated_config.py index b0e319ad8f..1c343ec41f 100644 --- a/kedro/config/templated_config.py +++ b/kedro/config/templated_config.py @@ -89,7 +89,7 @@ class TemplatedConfigLoader(AbstractConfigLoader): https://github.com/jmespath/jmespath.py and https://jmespath.org/. """ - def __init__( + def __init__( # noqa: too-many-arguments self, conf_source: str, env: str = None, @@ -264,7 +264,7 @@ def _format_string(match): f"'{formatted_key}' found" ) - key = formatted_key + key = formatted_key # noqa: PLW2901 new_dict[key] = _format_object(value, format_dict) diff --git a/kedro/extras/datasets/api/api_dataset.py b/kedro/extras/datasets/api/api_dataset.py index d0c87a45e6..cdfaa93c83 100644 --- a/kedro/extras/datasets/api/api_dataset.py +++ b/kedro/extras/datasets/api/api_dataset.py @@ -56,8 +56,7 @@ class APIDataSet(AbstractDataSet[None, requests.Response]): >>> data = data_set.load() """ - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, url: str, method: str = "GET", diff --git a/kedro/extras/datasets/biosequence/biosequence_dataset.py b/kedro/extras/datasets/biosequence/biosequence_dataset.py index 8db3a4371c..4888158774 100644 --- a/kedro/extras/datasets/biosequence/biosequence_dataset.py +++ b/kedro/extras/datasets/biosequence/biosequence_dataset.py @@ -44,8 +44,7 @@ class BioSequenceDataSet(AbstractDataSet[List, List]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/dask/parquet_dataset.py b/kedro/extras/datasets/dask/parquet_dataset.py index 58d4645d97..08c93b1d49 100644 --- a/kedro/extras/datasets/dask/parquet_dataset.py +++ b/kedro/extras/datasets/dask/parquet_dataset.py @@ -91,8 +91,7 @@ class ParquetDataSet(AbstractDataSet[dd.DataFrame, dd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {"write_index": False} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/email/message_dataset.py b/kedro/extras/datasets/email/message_dataset.py index 5a568190d1..8a725540c2 100644 --- a/kedro/extras/datasets/email/message_dataset.py +++ b/kedro/extras/datasets/email/message_dataset.py @@ -60,8 +60,7 @@ class EmailMessageDataSet( DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/geopandas/geojson_dataset.py b/kedro/extras/datasets/geopandas/geojson_dataset.py index 692079f747..88cce18dee 100644 --- a/kedro/extras/datasets/geopandas/geojson_dataset.py +++ b/kedro/extras/datasets/geopandas/geojson_dataset.py @@ -52,8 +52,7 @@ class GeoJSONDataSet( DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {"driver": "GeoJSON"} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/holoviews/holoviews_writer.py b/kedro/extras/datasets/holoviews/holoviews_writer.py index 31cfe7aa2d..2ed30f7156 100644 --- a/kedro/extras/datasets/holoviews/holoviews_writer.py +++ b/kedro/extras/datasets/holoviews/holoviews_writer.py @@ -44,8 +44,7 @@ class HoloviewsWriter(AbstractVersionedDataSet[HoloViews, NoReturn]): DEFAULT_SAVE_ARGS = {"fmt": "png"} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, fs_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/json/json_dataset.py b/kedro/extras/datasets/json/json_dataset.py index b364209331..17cc2cf69e 100644 --- a/kedro/extras/datasets/json/json_dataset.py +++ b/kedro/extras/datasets/json/json_dataset.py @@ -56,8 +56,7 @@ class JSONDataSet(AbstractVersionedDataSet[Any, Any]): DEFAULT_SAVE_ARGS = {"indent": 2} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, save_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/matplotlib/matplotlib_writer.py b/kedro/extras/datasets/matplotlib/matplotlib_writer.py index af82994b43..00a365f2ec 100644 --- a/kedro/extras/datasets/matplotlib/matplotlib_writer.py +++ b/kedro/extras/datasets/matplotlib/matplotlib_writer.py @@ -111,8 +111,7 @@ class MatplotlibWriter( DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, fs_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/networkx/gml_dataset.py b/kedro/extras/datasets/networkx/gml_dataset.py index e8bdfc7048..d48f7d37e2 100644 --- a/kedro/extras/datasets/networkx/gml_dataset.py +++ b/kedro/extras/datasets/networkx/gml_dataset.py @@ -44,8 +44,7 @@ class GMLDataSet(AbstractVersionedDataSet[networkx.Graph, networkx.Graph]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/networkx/graphml_dataset.py b/kedro/extras/datasets/networkx/graphml_dataset.py index b5ecdd09a7..54f5d496f7 100644 --- a/kedro/extras/datasets/networkx/graphml_dataset.py +++ b/kedro/extras/datasets/networkx/graphml_dataset.py @@ -43,8 +43,7 @@ class GraphMLDataSet(AbstractVersionedDataSet[networkx.Graph, networkx.Graph]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/networkx/json_dataset.py b/kedro/extras/datasets/networkx/json_dataset.py index 53e661afa2..4ae9940601 100644 --- a/kedro/extras/datasets/networkx/json_dataset.py +++ b/kedro/extras/datasets/networkx/json_dataset.py @@ -44,8 +44,7 @@ class JSONDataSet(AbstractVersionedDataSet[networkx.Graph, networkx.Graph]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/pandas/csv_dataset.py b/kedro/extras/datasets/pandas/csv_dataset.py index 34eeae6f4d..597d03ecf9 100644 --- a/kedro/extras/datasets/pandas/csv_dataset.py +++ b/kedro/extras/datasets/pandas/csv_dataset.py @@ -73,8 +73,7 @@ class CSVDataSet(AbstractVersionedDataSet[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {"index": False} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/pandas/excel_dataset.py b/kedro/extras/datasets/pandas/excel_dataset.py index ef26d32395..05c1144721 100644 --- a/kedro/extras/datasets/pandas/excel_dataset.py +++ b/kedro/extras/datasets/pandas/excel_dataset.py @@ -113,8 +113,7 @@ class ExcelDataSet( DEFAULT_LOAD_ARGS = {"engine": "openpyxl"} DEFAULT_SAVE_ARGS = {"index": False} - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, engine: str = "openpyxl", diff --git a/kedro/extras/datasets/pandas/feather_dataset.py b/kedro/extras/datasets/pandas/feather_dataset.py index 628797d4b9..534d84d9bf 100644 --- a/kedro/extras/datasets/pandas/feather_dataset.py +++ b/kedro/extras/datasets/pandas/feather_dataset.py @@ -73,8 +73,7 @@ class FeatherDataSet(AbstractVersionedDataSet[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/pandas/gbq_dataset.py b/kedro/extras/datasets/pandas/gbq_dataset.py index 7c2446d3a8..dda5cf9d35 100644 --- a/kedro/extras/datasets/pandas/gbq_dataset.py +++ b/kedro/extras/datasets/pandas/gbq_dataset.py @@ -70,8 +70,7 @@ class GBQTableDataSet(AbstractDataSet[None, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {"progress_bar": False} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, dataset: str, table_name: str, @@ -210,8 +209,7 @@ class GBQQueryDataSet(AbstractDataSet[None, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, sql: str = None, project: str = None, diff --git a/kedro/extras/datasets/pandas/generic_dataset.py b/kedro/extras/datasets/pandas/generic_dataset.py index a7d294570b..bf44694a26 100644 --- a/kedro/extras/datasets/pandas/generic_dataset.py +++ b/kedro/extras/datasets/pandas/generic_dataset.py @@ -90,8 +90,7 @@ class GenericDataSet(AbstractVersionedDataSet[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, file_format: str, diff --git a/kedro/extras/datasets/pandas/hdf_dataset.py b/kedro/extras/datasets/pandas/hdf_dataset.py index 4d89afd0b5..d60161d095 100644 --- a/kedro/extras/datasets/pandas/hdf_dataset.py +++ b/kedro/extras/datasets/pandas/hdf_dataset.py @@ -64,8 +64,7 @@ class HDFDataSet(AbstractVersionedDataSet[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, key: str, diff --git a/kedro/extras/datasets/pandas/json_dataset.py b/kedro/extras/datasets/pandas/json_dataset.py index a08dfb977e..1d5e3cb2d1 100644 --- a/kedro/extras/datasets/pandas/json_dataset.py +++ b/kedro/extras/datasets/pandas/json_dataset.py @@ -68,8 +68,7 @@ class JSONDataSet(AbstractVersionedDataSet[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/pandas/parquet_dataset.py b/kedro/extras/datasets/pandas/parquet_dataset.py index 105c771485..bf03f97ccd 100644 --- a/kedro/extras/datasets/pandas/parquet_dataset.py +++ b/kedro/extras/datasets/pandas/parquet_dataset.py @@ -80,8 +80,7 @@ class ParquetDataSet(AbstractVersionedDataSet[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/pandas/sql_dataset.py b/kedro/extras/datasets/pandas/sql_dataset.py index 70047caaa9..7c084cb82e 100644 --- a/kedro/extras/datasets/pandas/sql_dataset.py +++ b/kedro/extras/datasets/pandas/sql_dataset.py @@ -334,7 +334,7 @@ class SQLQueryDataSet(AbstractDataSet[None, pd.DataFrame]): # sqlalchemy.engine.Engine or sqlalchemy.engine.base.Engine engines: Dict[str, Any] = {} - def __init__( # pylint: disable=too-many-arguments + def __init__( # noqa: too-many-arguments self, sql: str = None, credentials: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/pandas/xml_dataset.py b/kedro/extras/datasets/pandas/xml_dataset.py index 0683bac00a..9433ae238d 100644 --- a/kedro/extras/datasets/pandas/xml_dataset.py +++ b/kedro/extras/datasets/pandas/xml_dataset.py @@ -51,8 +51,7 @@ class XMLDataSet(AbstractVersionedDataSet[pd.DataFrame, pd.DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {"index": False} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/pickle/pickle_dataset.py b/kedro/extras/datasets/pickle/pickle_dataset.py index b3df76d79c..eb9fb55594 100644 --- a/kedro/extras/datasets/pickle/pickle_dataset.py +++ b/kedro/extras/datasets/pickle/pickle_dataset.py @@ -76,8 +76,7 @@ class PickleDataSet(AbstractVersionedDataSet[Any, Any]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments,too-many-locals - def __init__( + def __init__( # noqa: too-many-arguments,too-many-locals self, filepath: str, backend: str = "pickle", diff --git a/kedro/extras/datasets/pillow/image_dataset.py b/kedro/extras/datasets/pillow/image_dataset.py index 11beb93c90..35c84995f4 100644 --- a/kedro/extras/datasets/pillow/image_dataset.py +++ b/kedro/extras/datasets/pillow/image_dataset.py @@ -40,8 +40,7 @@ class ImageDataSet(AbstractVersionedDataSet[Image.Image, Image.Image]): DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, save_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/plotly/json_dataset.py b/kedro/extras/datasets/plotly/json_dataset.py index a46ce97662..a03ee5b812 100644 --- a/kedro/extras/datasets/plotly/json_dataset.py +++ b/kedro/extras/datasets/plotly/json_dataset.py @@ -57,8 +57,7 @@ class JSONDataSet( DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/plotly/plotly_dataset.py b/kedro/extras/datasets/plotly/plotly_dataset.py index 97da08f68f..68dc27b012 100644 --- a/kedro/extras/datasets/plotly/plotly_dataset.py +++ b/kedro/extras/datasets/plotly/plotly_dataset.py @@ -70,8 +70,7 @@ class PlotlyDataSet(JSONDataSet): """ - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, plotly_args: Dict[str, Any], diff --git a/kedro/extras/datasets/redis/redis_dataset.py b/kedro/extras/datasets/redis/redis_dataset.py index 87374de333..c2bb2ca660 100644 --- a/kedro/extras/datasets/redis/redis_dataset.py +++ b/kedro/extras/datasets/redis/redis_dataset.py @@ -64,8 +64,7 @@ class PickleDataSet(AbstractDataSet[Any, Any]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, key: str, backend: str = "pickle", diff --git a/kedro/extras/datasets/spark/spark_dataset.py b/kedro/extras/datasets/spark/spark_dataset.py index 90355a557b..b27147b7a6 100644 --- a/kedro/extras/datasets/spark/spark_dataset.py +++ b/kedro/extras/datasets/spark/spark_dataset.py @@ -41,7 +41,7 @@ def _parse_glob_pattern(pattern: str) -> str: def _split_filepath(filepath: str) -> Tuple[str, str]: split_ = filepath.split("://", 1) - if len(split_) == 2: + if len(split_) == 2: # noqa: PLR2004 return split_[0] + "://", split_[1] return "", split_[0] @@ -232,7 +232,7 @@ class SparkDataSet(AbstractVersionedDataSet[DataFrame, DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - def __init__( # pylint: disable=too-many-arguments + def __init__( # noqa: too-many-arguments self, filepath: str, file_format: str = "parquet", diff --git a/kedro/extras/datasets/spark/spark_hive_dataset.py b/kedro/extras/datasets/spark/spark_hive_dataset.py index 8c79bba362..81f09b9daa 100644 --- a/kedro/extras/datasets/spark/spark_hive_dataset.py +++ b/kedro/extras/datasets/spark/spark_hive_dataset.py @@ -15,7 +15,7 @@ # in kedro-plugins (https://github.com/kedro-org/kedro-plugins) -# pylint:disable=too-many-instance-attributes +# noqa: too-many-instance-attributes class SparkHiveDataSet(AbstractDataSet[DataFrame, DataFrame]): """``SparkHiveDataSet`` loads and saves Spark dataframes stored on Hive. This data set also handles some incompatible file types such as using partitioned parquet on @@ -71,8 +71,7 @@ class SparkHiveDataSet(AbstractDataSet[DataFrame, DataFrame]): DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint:disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, database: str, table: str, @@ -210,7 +209,7 @@ def _validate_save(self, data: DataFrame): ) def _exists(self) -> bool: - # noqa # pylint:disable=protected-access + # noqa # noqa: protected-access return ( self._get_spark() ._jsparkSession.catalog() diff --git a/kedro/extras/datasets/spark/spark_jdbc_dataset.py b/kedro/extras/datasets/spark/spark_jdbc_dataset.py index 1b843d8d65..26a1ed2481 100644 --- a/kedro/extras/datasets/spark/spark_jdbc_dataset.py +++ b/kedro/extras/datasets/spark/spark_jdbc_dataset.py @@ -71,8 +71,7 @@ class SparkJDBCDataSet(AbstractDataSet[DataFrame, DataFrame]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, url: str, table: str, diff --git a/kedro/extras/datasets/svmlight/svmlight_dataset.py b/kedro/extras/datasets/svmlight/svmlight_dataset.py index 63fc406ff9..f8820b036f 100644 --- a/kedro/extras/datasets/svmlight/svmlight_dataset.py +++ b/kedro/extras/datasets/svmlight/svmlight_dataset.py @@ -90,8 +90,7 @@ class SVMLightDataSet(AbstractVersionedDataSet[_DI, _DO]): DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, diff --git a/kedro/extras/datasets/tensorflow/tensorflow_model_dataset.py b/kedro/extras/datasets/tensorflow/tensorflow_model_dataset.py index 5afab1c7d6..e1b35e6620 100644 --- a/kedro/extras/datasets/tensorflow/tensorflow_model_dataset.py +++ b/kedro/extras/datasets/tensorflow/tensorflow_model_dataset.py @@ -68,8 +68,7 @@ class TensorFlowModelDataset(AbstractVersionedDataSet[tf.keras.Model, tf.keras.M DEFAULT_LOAD_ARGS = {} # type: Dict[str, Any] DEFAULT_SAVE_ARGS = {"save_format": "tf"} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, load_args: Dict[str, Any] = None, @@ -134,7 +133,9 @@ def _load(self) -> tf.keras.Model: with tempfile.TemporaryDirectory(prefix=self._tmp_prefix) as path: if self._is_h5: - path = str(PurePath(path) / TEMPORARY_H5_FILE) + path = str( # noqa: PLW2901 + PurePath(path) / TEMPORARY_H5_FILE + ) # noqa: redefined-loop-name self._fs.copy(load_path, path) else: self._fs.get(load_path, path, recursive=True) @@ -153,7 +154,9 @@ def _save(self, data: tf.keras.Model) -> None: with tempfile.TemporaryDirectory(prefix=self._tmp_prefix) as path: if self._is_h5: - path = str(PurePath(path) / TEMPORARY_H5_FILE) + path = str( # noqa: PLW2901 + PurePath(path) / TEMPORARY_H5_FILE + ) # noqa: redefined-loop-name tf.keras.models.save_model(data, path, **self._save_args) diff --git a/kedro/extras/datasets/yaml/yaml_dataset.py b/kedro/extras/datasets/yaml/yaml_dataset.py index d3b93e3995..91c6b474cf 100644 --- a/kedro/extras/datasets/yaml/yaml_dataset.py +++ b/kedro/extras/datasets/yaml/yaml_dataset.py @@ -53,8 +53,7 @@ class YAMLDataSet(AbstractVersionedDataSet[Dict, Dict]): DEFAULT_SAVE_ARGS = {"default_flow_style": False} # type: Dict[str, Any] - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, filepath: str, save_args: Dict[str, Any] = None, diff --git a/kedro/extras/extensions/ipython.py b/kedro/extras/extensions/ipython.py index 2e38dc3772..ee700b571e 100644 --- a/kedro/extras/extensions/ipython.py +++ b/kedro/extras/extensions/ipython.py @@ -10,7 +10,7 @@ """ import warnings -from ...ipython import ( # noqa # pylint: disable=unused-import +from ...ipython import ( # noqa # noqa: unused-import load_ipython_extension, reload_kedro, ) diff --git a/kedro/framework/cli/catalog.py b/kedro/framework/cli/catalog.py index 2fa9528eb2..3a16bafa47 100644 --- a/kedro/framework/cli/catalog.py +++ b/kedro/framework/cli/catalog.py @@ -21,7 +21,7 @@ def _create_session(package_name: str, **kwargs): ) from exc -# pylint: disable=missing-function-docstring +# noqa: missing-function-docstring @click.group(name="Kedro") def catalog_cli(): # pragma: no cover pass @@ -32,7 +32,7 @@ def catalog(): """Commands for working with catalog.""" -# pylint: disable=too-many-locals +# noqa: too-many-locals @catalog.command("list") @env_option @click.option( @@ -53,7 +53,7 @@ def list_datasets(metadata: ProjectMetadata, pipeline, env): session = _create_session(metadata.package_name, env=env) context = session.load_context() - datasets_meta = context.catalog._data_sets # pylint: disable=protected-access + datasets_meta = context.catalog._data_sets # noqa: protected-access catalog_ds = set(context.catalog.list()) target_pipelines = pipeline or pipelines.keys() @@ -140,7 +140,7 @@ def create_catalog(metadata: ProjectMetadata, pipeline_name, env): catalog_datasets = { ds_name - for ds_name in context.catalog._data_sets.keys() # pylint: disable=protected-access + for ds_name in context.catalog._data_sets.keys() # noqa: protected-access if not ds_name.startswith("params:") and ds_name != "parameters" } diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index c1e8e33c9f..03c9743500 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -28,7 +28,7 @@ _get_entry_points, load_entry_points, ) -from kedro.framework.project import LOGGING # noqa # pylint:disable=unused-import +from kedro.framework.project import LOGGING # noqa # noqa: unused-import from kedro.framework.startup import _is_project, bootstrap_project LOGO = rf""" @@ -161,7 +161,7 @@ def global_groups(self) -> Sequence[click.MultiCommand]: @property def project_groups(self) -> Sequence[click.MultiCommand]: - # pylint: disable=line-too-long + # noqa: line-too-long """Property which loads all project command groups from the project and the plugins, then combines them with the built-in ones. Built-in commands can be overridden by plugins, which can be diff --git a/kedro/framework/cli/hooks/manager.py b/kedro/framework/cli/hooks/manager.py index 19457af690..a1be3e5784 100644 --- a/kedro/framework/cli/hooks/manager.py +++ b/kedro/framework/cli/hooks/manager.py @@ -1,5 +1,5 @@ """This module defines a dedicated hook manager for hooks that extends Kedro CLI behaviour.""" -# pylint: disable=global-statement,invalid-name +# noqa: global-statement,invalid-name import logging from pluggy import PluginManager @@ -16,7 +16,7 @@ def get_cli_hook_manager(): """Create or return the global _hook_manager singleton instance.""" - global _cli_hook_manager + global _cli_hook_manager # noqa: PLW0603 if _cli_hook_manager is None: _cli_hook_manager = CLIHooksManager() _cli_hook_manager.trace.root.setwriter(logger.debug) diff --git a/kedro/framework/cli/jupyter.py b/kedro/framework/cli/jupyter.py index 68eb8b2c85..e7cfbc166e 100644 --- a/kedro/framework/cli/jupyter.py +++ b/kedro/framework/cli/jupyter.py @@ -42,7 +42,7 @@ def list_commands(self, ctx): return ["setup", "notebook", "lab", "convert"] -# pylint: disable=missing-function-docstring +# noqa: missing-function-docstring @click.group(name="Kedro") def jupyter_cli(): # pragma: no cover pass @@ -57,7 +57,7 @@ def jupyter(): @forward_command(jupyter, "setup", forward_help=True) @click.pass_obj # this will pass the metadata as first argument -def setup(metadata: ProjectMetadata, args, **kwargs): # pylint: disable=unused-argument +def setup(metadata: ProjectMetadata, args, **kwargs): # noqa: unused-argument """Initialise the Jupyter Kernel for a kedro project.""" _check_module_importable("ipykernel") validate_settings() @@ -75,7 +75,7 @@ def jupyter_notebook( env, args, **kwargs, -): # pylint: disable=unused-argument +): # noqa: unused-argument """Open Jupyter Notebook with project specific variables loaded.""" _check_module_importable("notebook") validate_settings() @@ -101,7 +101,7 @@ def jupyter_lab( env, args, **kwargs, -): # pylint: disable=unused-argument +): # noqa: unused-argument """Open Jupyter Lab with project specific variables loaded.""" _check_module_importable("jupyterlab") validate_settings() @@ -160,7 +160,7 @@ def _create_kernel(kernel_name: str, display_name: str) -> str: """ # These packages are required by jupyter lab and notebook, which we have already # checked are importable, so we don't run _check_module_importable on them. - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel from ipykernel.kernelspec import install try: @@ -205,7 +205,7 @@ def _create_kernel(kernel_name: str, display_name: str) -> str: @click.pass_obj # this will pass the metadata as first argument def convert_notebook( metadata: ProjectMetadata, all_flag, overwrite_flag, filepath, env, **kwargs -): # pylint: disable=unused-argument, too-many-locals +): # noqa: unused-argument, too-many-locals """Convert selected or all notebooks found in a Kedro project to Kedro code, by exporting code from the appropriately-tagged cells: Cells tagged as `node` will be copied over to a Python file matching diff --git a/kedro/framework/cli/micropkg.py b/kedro/framework/cli/micropkg.py index a2de8c9a16..3e0a559cf1 100644 --- a/kedro/framework/cli/micropkg.py +++ b/kedro/framework/cli/micropkg.py @@ -1,4 +1,5 @@ """A collection of CLI commands for working with Kedro micro-packages.""" +# ruff: noqa: I001 # https://github.com/kedro-org/kedro/pull/2634 from __future__ import annotations import re @@ -101,7 +102,7 @@ def __eq__(self, other: Any) -> bool: ) -def _check_module_path(ctx, param, value): # pylint: disable=unused-argument +def _check_module_path(ctx, param, value): # noqa: unused-argument if value and not re.match(r"^[\w.]+$", value): message = ( "The micro-package location you provided is not a valid Python module path" @@ -110,7 +111,7 @@ def _check_module_path(ctx, param, value): # pylint: disable=unused-argument return value -# pylint: disable=missing-function-docstring +# noqa: missing-function-docstring @click.group(name="Kedro") def micropkg_cli(): # pragma: no cover pass @@ -150,7 +151,7 @@ def micropkg(): help="Location of a configuration file for the fsspec filesystem used to pull the package.", ) @click.pass_obj # this will pass the metadata as first argument -def pull_package( # pylint:disable=unused-argument, too-many-arguments +def pull_package( # noqa: unused-argument, too-many-arguments metadata: ProjectMetadata, package_path, env, @@ -185,8 +186,7 @@ def pull_package( # pylint:disable=unused-argument, too-many-arguments click.secho(message, fg="green") -# pylint: disable=too-many-arguments -def _pull_package( +def _pull_package( # noqa: too-many-arguments package_path: str, metadata: ProjectMetadata, env: str = None, @@ -255,7 +255,7 @@ def _pull_package( def _pull_packages_from_manifest(metadata: ProjectMetadata) -> None: - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel import anyconfig # for performance reasons config_dict = anyconfig.load(metadata.config_file) @@ -279,7 +279,7 @@ def _pull_packages_from_manifest(metadata: ProjectMetadata) -> None: def _package_micropkgs_from_manifest(metadata: ProjectMetadata) -> None: - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel import anyconfig # for performance reasons config_dict = anyconfig.load(metadata.config_file) @@ -328,7 +328,7 @@ def _package_micropkgs_from_manifest(metadata: ProjectMetadata) -> None: ) @click.argument("module_path", nargs=1, required=False, callback=_check_module_path) @click.pass_obj # this will pass the metadata as first argument -def package_micropkg( +def package_micropkg( # noqa: too-many-arguments metadata: ProjectMetadata, module_path, env, @@ -336,7 +336,7 @@ def package_micropkg( destination, all_flag, **kwargs, -): # pylint: disable=unused-argument +): """Package up a modular pipeline or micro-package as a Python source distribution.""" if not module_path and not all_flag: click.secho( @@ -362,7 +362,7 @@ def package_micropkg( def _get_fsspec_filesystem(location: str, fs_args: str | None): - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel import anyconfig import fsspec @@ -373,7 +373,7 @@ def _get_fsspec_filesystem(location: str, fs_args: str | None): try: return fsspec.filesystem(protocol, **fs_args_config) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: # noqa: broad-except # Specified protocol is not supported by `fsspec` # or requires extra dependencies click.secho(str(exc), fg="red") @@ -391,7 +391,7 @@ def safe_extract(tar, path): for member in tar.getmembers(): member_path = path / member.name if not _is_within_directory(path, member_path): - # pylint: disable=broad-exception-raised + # noqa: broad-exception-raised raise Exception("Failed to safely extract tar file.") tar.extractall(path) # nosec B202 @@ -440,7 +440,7 @@ def _rename_files(conf_source: Path, old_name: str, new_name: str): config_file.rename(config_file.parent / new_config_name) -def _refactor_code_for_unpacking( +def _refactor_code_for_unpacking( # noqa: too-many-arguments project: Project, package_path: Path, tests_path: Path, @@ -521,7 +521,7 @@ def _move_package_with_conflicting_name( return refactored_package_path, refactored_tests_path -def _install_files( # pylint: disable=too-many-arguments, too-many-locals +def _install_files( # noqa: too-many-arguments, too-many-locals project_metadata: ProjectMetadata, package_name: str, source_path: Path, @@ -811,8 +811,7 @@ def _move_package_with_conflicting_name(target: Path, conflicting_name: str): _SourcePathType = Union[Path, List[Tuple[Path, str]]] -# pylint: disable=too-many-arguments,too-many-locals -def _generate_sdist_file( +def _generate_sdist_file( # noqa: too-many-arguments,too-many-locals micropkg_name: str, destination: Path, source_paths: tuple[_SourcePathType, ...], diff --git a/kedro/framework/cli/pipeline.py b/kedro/framework/cli/pipeline.py index 225ad5fd74..09e3c6986c 100644 --- a/kedro/framework/cli/pipeline.py +++ b/kedro/framework/cli/pipeline.py @@ -56,7 +56,7 @@ def _assert_pkg_name_ok(pkg_name: str): if not re.match(r"^[a-zA-Z_]", pkg_name): message = base_message + " It must start with a letter or underscore." raise KedroCliError(message) - if len(pkg_name) < 2: + if len(pkg_name) < 2: # noqa: PLR2004 message = base_message + " It must be at least 2 characters long." raise KedroCliError(message) if not re.match(r"^\w+$", pkg_name[1:]): @@ -66,13 +66,13 @@ def _assert_pkg_name_ok(pkg_name: str): raise KedroCliError(message) -def _check_pipeline_name(ctx, param, value): # pylint: disable=unused-argument +def _check_pipeline_name(ctx, param, value): # noqa: unused-argument if value: _assert_pkg_name_ok(value) return value -# pylint: disable=missing-function-docstring +# noqa: missing-function-docstring @click.group(name="Kedro") def pipeline_cli(): # pragma: no cover pass @@ -94,7 +94,7 @@ def pipeline(): @click.pass_obj # this will pass the metadata as first argument def create_pipeline( metadata: ProjectMetadata, name, skip_config, env, **kwargs -): # pylint: disable=unused-argument +): # noqa: unused-argument """Create a new modular pipeline by providing a name.""" package_dir = metadata.source_dir / metadata.package_name conf_source = settings.CONF_SOURCE @@ -124,7 +124,7 @@ def create_pipeline( @click.pass_obj # this will pass the metadata as first argument def delete_pipeline( metadata: ProjectMetadata, name, env, yes, **kwargs -): # pylint: disable=unused-argument +): # noqa: unused-argument """Delete a modular pipeline by providing a name.""" package_dir = metadata.source_dir / metadata.package_name conf_source = settings.CONF_SOURCE @@ -189,7 +189,7 @@ def _echo_deletion_warning(message: str, **paths: list[Path]): def _create_pipeline(name: str, output_dir: Path) -> Path: with _filter_deprecation_warnings(): - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel from cookiecutter.main import cookiecutter template_path = Path(kedro.__file__).parent / "templates" / "pipeline" diff --git a/kedro/framework/cli/project.py b/kedro/framework/cli/project.py index 034b460023..f3cf141dfa 100644 --- a/kedro/framework/cli/project.py +++ b/kedro/framework/cli/project.py @@ -68,7 +68,7 @@ CONF_SOURCE_HELP = """Path of a directory where project configuration is stored.""" -# pylint: disable=missing-function-docstring +# noqa: missing-function-docstring @click.group(name="Kedro") def project_group(): # pragma: no cover pass @@ -76,7 +76,7 @@ def project_group(): # pragma: no cover @forward_command(project_group, forward_help=True) @click.pass_obj # this will pass the metadata as first argument -def test(metadata: ProjectMetadata, args, **kwargs): # pylint: disable=unused-argument +def test(metadata: ProjectMetadata, args, **kwargs): # noqa: ument """Run the test suite. (DEPRECATED)""" deprecation_message = ( "DeprecationWarning: Command 'kedro test' is deprecated and " @@ -101,7 +101,7 @@ def test(metadata: ProjectMetadata, args, **kwargs): # pylint: disable=unused-a @click.pass_obj # this will pass the metadata as first argument def lint( metadata: ProjectMetadata, files, check_only, **kwargs -): # pylint: disable=unused-argument +): # noqa: unused-argument """Run flake8, isort and black. (DEPRECATED)""" deprecation_message = ( "DeprecationWarning: Command 'kedro lint' is deprecated and " @@ -134,9 +134,7 @@ def lint( @forward_command(project_group, forward_help=True) @env_option @click.pass_obj # this will pass the metadata as first argument -def ipython( - metadata: ProjectMetadata, env, args, **kwargs -): # pylint: disable=unused-argument +def ipython(metadata: ProjectMetadata, env, args, **kwargs): # noqa: unused-argument """Open IPython with project specific variables loaded.""" _check_module_importable("IPython") @@ -238,7 +236,7 @@ def build_docs(metadata: ProjectMetadata, open_docs): @click.pass_obj # this will pass the metadata as first argument def build_reqs( metadata: ProjectMetadata, input_file, output_file, args, **kwargs -): # pylint: disable=unused-argument +): # noqa: unused-argument """Run `pip-compile` on src/requirements.txt or the user defined input file and save the compiled requirements to src/requirements.lock or the user defined output file. (DEPRECATED) @@ -281,9 +279,7 @@ def build_reqs( @command_with_verbosity(project_group, "activate-nbstripout") @click.pass_obj # this will pass the metadata as first argument -def activate_nbstripout( - metadata: ProjectMetadata, **kwargs -): # pylint: disable=unused-argument +def activate_nbstripout(metadata: ProjectMetadata, **kwargs): # noqa: unused-argument """Install the nbstripout git hook to automatically clean notebooks. (DEPRECATED)""" deprecation_message = ( "DeprecationWarning: Command 'kedro activate-nbstripout' is deprecated and " @@ -308,10 +304,9 @@ def activate_nbstripout( ) from exc try: - res = subprocess.run( # pylint: disable=subprocess-run-check + res = subprocess.run( # noqa: subprocess-run-check ["git", "rev-parse", "--git-dir"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, ) if res.returncode: raise KedroCliError("Not a git repository. Run 'git init' first.") @@ -417,8 +412,7 @@ def activate_nbstripout( help=PARAMS_ARG_HELP, callback=_split_params, ) -# pylint: disable=too-many-arguments,unused-argument,too-many-locals -def run( +def run( # noqa: too-many-arguments,unused-argument,too-many-locals tag, tags, env, diff --git a/kedro/framework/cli/registry.py b/kedro/framework/cli/registry.py index ea456016a7..05b05f9afd 100644 --- a/kedro/framework/cli/registry.py +++ b/kedro/framework/cli/registry.py @@ -7,7 +7,7 @@ from kedro.framework.startup import ProjectMetadata -# pylint: disable=missing-function-docstring +# noqa: missing-function-docstring @click.group(name="Kedro") def registry_cli(): # pragma: no cover pass @@ -29,7 +29,7 @@ def list_registered_pipelines(): @click.pass_obj def describe_registered_pipeline( metadata: ProjectMetadata, name, **kwargs -): # pylint: disable=unused-argument, protected-access +): # noqa: unused-argument, protected-access """Describe a registered pipeline by providing a pipeline name. Defaults to the `__default__` pipeline. """ diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 77491d391f..25e68f3699 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -37,7 +37,7 @@ @define(order=True) -class KedroStarterSpec: # pylint: disable=too-few-public-methods +class KedroStarterSpec: # noqa: too-few-public-methods """Specification of custom kedro starter template Args: alias: alias of the starter which shows up on `kedro starter list` and is used @@ -92,7 +92,7 @@ class KedroStarterSpec: # pylint: disable=too-few-public-methods ) -# pylint: disable=unused-argument +# noqa: unused-argument def _remove_readonly(func: Callable, path: Path, excinfo: tuple): # pragma: no cover """Remove readonly files on Windows See: https://docs.python.org/3/library/shutil.html?highlight=shutil#rmtree-example @@ -160,7 +160,7 @@ def _starter_spec_to_dict( return format_dict -# pylint: disable=missing-function-docstring +# noqa: missing-function-docstring @click.group(context_settings=CONTEXT_SETTINGS, name="Kedro") def create_cli(): # pragma: no cover pass @@ -347,7 +347,7 @@ def _create_project(template_path: str, cookiecutter_args: dict[str, Any]): KedroCliError: If it fails to generate a project. """ with _filter_deprecation_warnings(): - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel from cookiecutter.main import cookiecutter # for performance reasons try: @@ -389,7 +389,7 @@ def _get_cookiecutter_dir( clones it to ``tmpdir``; if template_path is a file path then directly uses that path without copying anything. """ - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel from cookiecutter.exceptions import RepositoryCloneFailed, RepositoryNotFound from cookiecutter.repository import determine_repo_dir # for performance reasons @@ -447,7 +447,7 @@ def _fetch_config_from_user_prompts( Configuration for starting a new project. This is passed as ``extra_context`` to cookiecutter and will overwrite the cookiecutter.json defaults. """ - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel from cookiecutter.environment import StrictEnvironment from cookiecutter.prompt import read_user_variable, render_variable @@ -472,7 +472,7 @@ def _fetch_config_from_user_prompts( def _make_cookiecutter_context_for_prompts(cookiecutter_dir: Path): - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel from cookiecutter.generate import generate_context cookiecutter_context = generate_context(cookiecutter_dir / "cookiecutter.json") @@ -482,7 +482,7 @@ def _make_cookiecutter_context_for_prompts(cookiecutter_dir: Path): class _Prompt: """Represent a single CLI prompt for `kedro new`""" - def __init__(self, *args, **kwargs) -> None: # pylint: disable=unused-argument + def __init__(self, *args, **kwargs) -> None: # noqa: unused-argument try: self.title = kwargs["title"] except KeyError as exc: @@ -512,7 +512,7 @@ def validate(self, user_input: str) -> None: def _get_available_tags(template_path: str) -> list: # Not at top level so that kedro CLI works without a working git executable. - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel import git try: diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py index bd1c59a2ec..3240c3c4ab 100644 --- a/kedro/framework/cli/utils.py +++ b/kedro/framework/cli/utils.py @@ -52,7 +52,7 @@ def call(cmd: list[str], **kwargs): # pragma: no cover click.exceptions.Exit: If `subprocess.run` returns non-zero code. """ click.echo(" ".join(shlex.quote(c) for c in cmd)) - # pylint: disable=subprocess-run-check + # noqa: subprocess-run-check code = subprocess.run(cmd, **kwargs).returncode if code: raise click.exceptions.Exit(code=code) @@ -222,14 +222,14 @@ def get_pkg_version(reqs_path: (str | Path), package_name: str) -> str: pattern = re.compile(package_name + r"([^\w]|$)") with reqs_path.open("r", encoding="utf-8") as reqs_file: for req_line in reqs_file: - req_line = req_line.strip() + req_line = req_line.strip() # noqa: redefined-loop-name if pattern.search(req_line): return req_line raise KedroCliError(f"Cannot find '{package_name}' package in '{reqs_path}'.") -def _update_verbose_flag(ctx, param, value): # pylint: disable=unused-argument +def _update_verbose_flag(ctx, param, value): # noqa: unused-argument KedroCliError.VERBOSE_ERROR = value @@ -265,7 +265,7 @@ class KedroCliError(click.exceptions.ClickException): def show(self, file=None): if file is None: - # pylint: disable=protected-access + # noqa: protected-access file = click._compat.get_text_stderr() if self.VERBOSE_ERROR: click.secho(traceback.format_exc(), nl=False, fg="yellow") @@ -291,12 +291,12 @@ def _clean_pycache(path: Path): shutil.rmtree(each, ignore_errors=True) -def split_string(ctx, param, value): # pylint: disable=unused-argument +def split_string(ctx, param, value): # noqa: unused-argument """Split string by comma.""" return [item.strip() for item in value.split(",") if item.strip()] -# pylint: disable=unused-argument,missing-param-doc,missing-type-doc +# noqa: unused-argument,missing-param-doc,missing-type-doc def split_node_names(ctx, param, to_split: str) -> list[str]: """Split string by comma, ignoring commas enclosed by square parentheses. This avoids splitting the string of nodes names on commas included in @@ -363,13 +363,13 @@ def _get_entry_points(name: str) -> importlib_metadata.EntryPoints: return importlib_metadata.entry_points().select(group=ENTRY_POINT_GROUPS[name]) -def _safe_load_entry_point( # pylint: disable=inconsistent-return-statements +def _safe_load_entry_point( # noqa: inconsistent-return-statements entry_point, ): """Load entrypoint safely, if fails it will just skip the entrypoint.""" try: return entry_point.load() - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: # noqa: broad-except logger.warning( "Failed to load %s commands from %s. Full exception: %s", entry_point.module, @@ -401,13 +401,13 @@ def load_entry_points(name: str) -> Sequence[click.MultiCommand]: return entry_point_commands -def _config_file_callback(ctx, param, value): # pylint: disable=unused-argument +def _config_file_callback(ctx, param, value): # noqa: unused-argument """CLI callback that replaces command line options with values specified in a config file. If command line options are passed, they override config file values. """ # for performance reasons - import anyconfig # pylint: disable=import-outside-toplevel + import anyconfig # noqa: import-outside-toplevel ctx.default_map = ctx.default_map or {} section = ctx.info_name @@ -428,9 +428,9 @@ def _reformat_load_versions(ctx, param, value) -> dict[str, str]: load_versions_dict = {} for load_version in value: - load_version = load_version.strip() + load_version = load_version.strip() # noqa: PLW2901 load_version_list = load_version.split(":", 1) - if len(load_version_list) != 2: + if len(load_version_list) != 2: # noqa: PLR2004 raise KedroCliError( f"Expected the form of 'load_version' to be " f"'dataset_name:YYYY-MM-DDThh.mm.ss.sssZ'," @@ -453,9 +453,9 @@ def _split_params(ctx, param, value): # which should not be replaced by = pass else: - item = item.replace(":", "=", 1) + item = item.replace(":", "=", 1) # noqa: redefined-loop-name items = item.split("=", 1) - if len(items) != 2: + if len(items) != 2: # noqa: PLR2004 ctx.fail( f"Invalid format of `{param.name}` option: " f"Item `{items[0]}` must contain " diff --git a/kedro/framework/context/context.py b/kedro/framework/context/context.py index 2425bcc672..0ba116255a 100644 --- a/kedro/framework/context/context.py +++ b/kedro/framework/context/context.py @@ -131,7 +131,7 @@ def _validate_transcoded_datasets(catalog: DataCatalog): `_transcode_split` function. """ - # pylint: disable=protected-access + # noqa: protected-access for dataset_name in catalog._data_sets.keys(): _transcode_split(dataset_name) @@ -147,11 +147,10 @@ def _update_nested_dict(old_dict: dict[Any, Any], new_dict: dict[Any, Any]) -> N for key, value in new_dict.items(): if key not in old_dict: old_dict[key] = value + elif isinstance(old_dict[key], dict) and isinstance(value, dict): + _update_nested_dict(old_dict[key], value) else: - if isinstance(old_dict[key], dict) and isinstance(value, dict): - _update_nested_dict(old_dict[key], value) - else: - old_dict[key] = value + old_dict[key] = value class KedroContext: @@ -159,7 +158,7 @@ class KedroContext: Kedro's main functionality. """ - def __init__( + def __init__( # noqa: too-many-arguments self, package_name: str, project_path: Path | str, @@ -167,7 +166,7 @@ def __init__( hook_manager: PluginManager, env: str = None, extra_params: dict[str, Any] = None, - ): # pylint: disable=too-many-arguments + ): """Create a context object by providing the root of a Kedro project and the environment configuration subfolders (see ``kedro.config.ConfigLoader``) diff --git a/kedro/framework/hooks/specs.py b/kedro/framework/hooks/specs.py index 6bef21f3bb..aa10ab7276 100644 --- a/kedro/framework/hooks/specs.py +++ b/kedro/framework/hooks/specs.py @@ -18,7 +18,7 @@ class DataCatalogSpecs: """Namespace that defines all specifications for a data catalog's lifecycle hooks.""" @hook_spec - def after_catalog_created( # pylint: disable=too-many-arguments + def after_catalog_created( # noqa: too-many-arguments self, catalog: DataCatalog, conf_catalog: dict[str, Any], @@ -48,7 +48,7 @@ class NodeSpecs: """Namespace that defines all specifications for a node's lifecycle hooks.""" @hook_spec - def before_node_run( # pylint: disable=too-many-arguments + def before_node_run( # noqa: too-many-arguments self, node: Node, catalog: DataCatalog, @@ -76,7 +76,7 @@ def before_node_run( # pylint: disable=too-many-arguments pass @hook_spec - def after_node_run( # pylint: disable=too-many-arguments + def after_node_run( # noqa: too-many-arguments self, node: Node, catalog: DataCatalog, @@ -104,7 +104,7 @@ def after_node_run( # pylint: disable=too-many-arguments pass @hook_spec - def on_node_error( # pylint: disable=too-many-arguments + def on_node_error( # noqa: too-many-arguments self, error: Exception, node: Node, diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 0d6946555e..f266da430c 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -1,6 +1,6 @@ """``kedro.framework.project`` module provides utitlity to configure a Kedro project and access its settings.""" -# pylint: disable=redefined-outer-name,unused-argument,global-statement +# noqa: redefined-outer-name,unused-argument,global-statement from __future__ import annotations import importlib @@ -131,7 +131,7 @@ def _load_data_wrapper(func): Taking inspiration from dynaconf.utils.functional.new_method_proxy """ - # pylint: disable=protected-access + # noqa: protected-access def inner(self, *args, **kwargs): self._load_data() return func(self._content, *args, **kwargs) @@ -208,7 +208,7 @@ def configure(self, pipelines_module: str | None = None) -> None: class _ProjectLogging(UserDict): - # pylint: disable=super-init-not-called + # noqa: super-init-not-called def __init__(self): """Initialise project logging. The path to logging configuration is given in environment variable KEDRO_LOGGING_CONFIG (defaults to default_logging.yml).""" @@ -249,7 +249,7 @@ def configure_project(package_name: str): # global variable to make it easily accessible. This is used by validate_settings() # below, and also by ParallelRunner on Windows, as package_name is required every # time a new subprocess is spawned. - global PACKAGE_NAME + global PACKAGE_NAME # noqa: PLW0603 PACKAGE_NAME = package_name @@ -299,7 +299,7 @@ def _create_pipeline(pipeline_module: types.ModuleType) -> Pipeline | None: return obj -def find_pipelines() -> dict[str, Pipeline]: +def find_pipelines() -> dict[str, Pipeline]: # noqa: PLR0912 """Automatically find modular pipelines having a ``create_pipeline`` function. By default, projects created using Kedro 0.18.3 and higher call this function to autoregister pipelines upon creation/addition. @@ -325,7 +325,7 @@ def find_pipelines() -> dict[str, Pipeline]: pipeline_module_name = f"{PACKAGE_NAME}.pipeline" try: pipeline_module = importlib.import_module(pipeline_module_name) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: # noqa: broad-except if str(exc) != f"No module named '{pipeline_module_name}'": warnings.warn( IMPORT_ERROR_MESSAGE.format( @@ -355,7 +355,7 @@ def find_pipelines() -> dict[str, Pipeline]: pipeline_module_name = f"{PACKAGE_NAME}.pipelines.{pipeline_name}" try: pipeline_module = importlib.import_module(pipeline_module_name) - except: # pylint: disable=bare-except # noqa: E722 + except: # noqa: bare-except # noqa: E722 warnings.warn( IMPORT_ERROR_MESSAGE.format( module=pipeline_module_name, tb_exc=traceback.format_exc() diff --git a/kedro/framework/session/session.py b/kedro/framework/session/session.py index 90df0ffee9..b4503723b0 100644 --- a/kedro/framework/session/session.py +++ b/kedro/framework/session/session.py @@ -48,7 +48,7 @@ def _describe_git(project_path: Path) -> dict[str, dict[str, Any]]: git_data["dirty"] = bool(git_status_res.decode().strip()) # `subprocess.check_output()` raises `NotADirectoryError` on Windows - except Exception: # pylint: disable=broad-except + except Exception: # noqa: broad-except logger = logging.getLogger(__name__) logger.debug("Unable to git describe %s", project_path) logger.debug(traceback.format_exc()) @@ -74,7 +74,7 @@ class KedroSessionError(Exception): pass -# pylint: disable=too-many-instance-attributes +# noqa: too-many-instance-attributes class KedroSession: """``KedroSession`` is the object that is responsible for managing the lifecycle of a Kedro run. Use `KedroSession.create()` as @@ -99,8 +99,7 @@ class KedroSession: """ - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, session_id: str, package_name: str = None, @@ -125,7 +124,7 @@ def __init__( ) @classmethod - def create( # pylint: disable=too-many-arguments + def create( # noqa: too-many-arguments cls, package_name: str = None, project_path: Path | str | None = None, @@ -183,7 +182,7 @@ def create( # pylint: disable=too-many-arguments try: session_data["username"] = getpass.getuser() - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: # noqa: broad-except logging.getLogger(__name__).debug( "Unable to get username. Full exception: %s", exc ) @@ -304,7 +303,7 @@ def __exit__(self, exc_type, exc_value, tb_): self._log_exception(exc_type, exc_value, tb_) self.close() - def run( # pylint: disable=too-many-arguments,too-many-locals + def run( # noqa: too-many-arguments,too-many-locals self, pipeline_name: str = None, tags: Iterable[str] = None, @@ -406,7 +405,7 @@ def run( # pylint: disable=too-many-arguments,too-many-locals "runner": getattr(runner, "__name__", str(runner)), } - catalog = context._get_catalog( # pylint: disable=protected-access + catalog = context._get_catalog( # noqa: protected-access save_version=save_version, load_versions=load_versions, ) diff --git a/kedro/framework/startup.py b/kedro/framework/startup.py index 6b0d2cc2c7..287999125a 100644 --- a/kedro/framework/startup.py +++ b/kedro/framework/startup.py @@ -41,7 +41,7 @@ def _is_project(project_path: Union[str, Path]) -> bool: try: return "[tool.kedro]" in metadata_file.read_text(encoding="utf-8") - except Exception: # pylint: disable=broad-except + except Exception: # noqa: broad-except return False diff --git a/kedro/io/__init__.py b/kedro/io/__init__.py index 7bd381d4cf..0755af906c 100644 --- a/kedro/io/__init__.py +++ b/kedro/io/__init__.py @@ -3,7 +3,7 @@ """ from __future__ import annotations -from .cached_dataset import CachedDataset, CachedDataSet +from .cached_dataset import CachedDataSet, CachedDataset from .core import ( AbstractDataSet, AbstractVersionedDataSet, @@ -13,13 +13,13 @@ Version, ) from .data_catalog import DataCatalog -from .lambda_dataset import LambdaDataset, LambdaDataSet -from .memory_dataset import MemoryDataset, MemoryDataSet +from .lambda_dataset import LambdaDataSet, LambdaDataset +from .memory_dataset import MemoryDataSet, MemoryDataset from .partitioned_dataset import ( - IncrementalDataset, IncrementalDataSet, - PartitionedDataset, + IncrementalDataset, PartitionedDataSet, + PartitionedDataset, ) # https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901 @@ -29,11 +29,9 @@ def __getattr__(name): - import kedro.io.core # pylint: disable=import-outside-toplevel + import kedro.io.core # noqa: import-outside-toplevel - if name in ( - kedro.io.core._DEPRECATED_ERROR_CLASSES # pylint: disable=protected-access - ): + if name in (kedro.io.core._DEPRECATED_ERROR_CLASSES): # noqa: protected-access return getattr(kedro.io.core, name) raise AttributeError(f"module {repr(__name__)} has no attribute {repr(name)}") diff --git a/kedro/io/cached_dataset.py b/kedro/io/cached_dataset.py index 5e2a124b22..d3aee1a39e 100644 --- a/kedro/io/cached_dataset.py +++ b/kedro/io/cached_dataset.py @@ -96,8 +96,8 @@ def _from_config(config, version): def _describe(self) -> dict[str, Any]: return { - "dataset": self._dataset._describe(), # pylint: disable=protected-access - "cache": self._cache._describe(), # pylint: disable=protected-access + "dataset": self._dataset._describe(), # noqa: protected-access + "cache": self._cache._describe(), # noqa: protected-access } def _load(self): diff --git a/kedro/io/core.py b/kedro/io/core.py index b74d4fb6b4..f608f10840 100644 --- a/kedro/io/core.py +++ b/kedro/io/core.py @@ -580,7 +580,7 @@ def _fetch_latest_load_version(self) -> str: # 'key' is set to prevent cache key overlapping for load and save: # https://cachetools.readthedocs.io/en/stable/#cachetools.cachedmethod @cachedmethod(cache=attrgetter("_version_cache"), key=partial(hashkey, "save")) - def _fetch_latest_save_version(self) -> str: # pylint: disable=no-self-use + def _fetch_latest_save_version(self) -> str: # noqa: no-self-use """Generate and cache the current save version""" return generate_timestamp() @@ -627,7 +627,7 @@ def _get_save_path(self) -> PurePosixPath: def _get_versioned_path(self, version: str) -> PurePosixPath: return self._filepath / version / self._filepath.name - def load(self) -> _DO: # pylint: disable=useless-parent-delegation + def load(self) -> _DO: # noqa: useless-parent-delegation return super().load() def save(self, data: _DI) -> None: diff --git a/kedro/io/data_catalog.py b/kedro/io/data_catalog.py index 425e491c0b..98e8d8f289 100644 --- a/kedro/io/data_catalog.py +++ b/kedro/io/data_catalog.py @@ -140,7 +140,7 @@ class DataCatalog: to the underlying data sets. """ - def __init__( # pylint: disable=too-many-arguments + def __init__( # noqa: too-many-arguments self, data_sets: dict[str, AbstractDataSet] = None, feed_dict: dict[str, Any] = None, @@ -285,7 +285,9 @@ class to be loaded is specified with the key ``type`` and their layers: dict[str, set[str]] = defaultdict(set) for ds_name, ds_config in catalog.items(): - ds_config = _resolve_credentials(ds_config, credentials) + ds_config = _resolve_credentials( # noqa: redefined-loop-name + ds_config, credentials + ) if cls._is_pattern(ds_name): # Add each factory to the dataset_patterns dict. dataset_patterns[ds_name] = ds_config @@ -406,9 +408,7 @@ def _get_dataset( if version and isinstance(data_set, AbstractVersionedDataSet): # we only want to return a similar-looking dataset, # not modify the one stored in the current catalog - data_set = data_set._copy( # pylint: disable=protected-access - _version=version - ) + data_set = data_set._copy(_version=version) # noqa: protected-access return data_set diff --git a/kedro/io/lambda_dataset.py b/kedro/io/lambda_dataset.py index f7cca70da2..b2cca48921 100644 --- a/kedro/io/lambda_dataset.py +++ b/kedro/io/lambda_dataset.py @@ -80,8 +80,7 @@ def _release(self) -> None: else: self.__release() - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, load: Callable[[], Any] | None, save: Callable[[Any], None] | None, diff --git a/kedro/io/memory_dataset.py b/kedro/io/memory_dataset.py index b0ca1d2c30..1dc5ded1b0 100644 --- a/kedro/io/memory_dataset.py +++ b/kedro/io/memory_dataset.py @@ -93,7 +93,7 @@ def _infer_copy_mode(data: Any) -> str: Returns: One of "copy", "assign" or "deepcopy" as the copy mode to use. """ - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel try: import pandas as pd except ImportError: # pragma: no cover diff --git a/kedro/io/partitioned_dataset.py b/kedro/io/partitioned_dataset.py index da8c78edc2..1501fc4a04 100644 --- a/kedro/io/partitioned_dataset.py +++ b/kedro/io/partitioned_dataset.py @@ -37,7 +37,7 @@ class PartitionedDataset(AbstractDataSet): - # pylint: disable=too-many-instance-attributes,protected-access + # noqa: too-many-instance-attributes,protected-access """``PartitionedDataset`` loads and saves partitioned file-like data using the underlying dataset definition. For filesystem level operations it uses `fsspec`: https://github.com/intake/filesystem_spec. @@ -135,7 +135,7 @@ class PartitionedDataset(AbstractDataSet): """ - def __init__( # pylint: disable=too-many-arguments + def __init__( # noqa: too-many-arguments self, path: str, dataset: str | type[AbstractDataSet] | dict[str, Any], @@ -190,7 +190,7 @@ def __init__( # pylint: disable=too-many-arguments Raises: DatasetError: If versioning is enabled for the underlying dataset. """ - # pylint: disable=import-outside-toplevel + # noqa: import-outside-toplevel from fsspec.utils import infer_storage_options # for performance reasons super().__init__() @@ -247,7 +247,7 @@ def __init__( # pylint: disable=too-many-arguments @property def _filesystem(self): # for performance reasons - import fsspec # pylint: disable=import-outside-toplevel + import fsspec # noqa: import-outside-toplevel protocol = "s3" if self._protocol in S3_PROTOCOLS else self._protocol return fsspec.filesystem(protocol, **self._credentials, **self._fs_args) @@ -314,7 +314,7 @@ def _save(self, data: dict[str, Any]) -> None: kwargs[self._filepath_arg] = self._join_protocol(partition) dataset = self._dataset_type(**kwargs) # type: ignore if callable(partition_data): - partition_data = partition_data() + partition_data = partition_data() # noqa: redefined-loop-name dataset.save(partition_data) self._invalidate_caches() @@ -381,8 +381,7 @@ class IncrementalDataset(PartitionedDataset): DEFAULT_CHECKPOINT_TYPE = "kedro.extras.datasets.text.TextDataSet" DEFAULT_CHECKPOINT_FILENAME = "CHECKPOINT" - # pylint: disable=too-many-arguments - def __init__( + def __init__( # noqa: too-many-arguments self, path: str, dataset: str | type[AbstractDataSet] | dict[str, Any], @@ -500,10 +499,8 @@ def _parse_checkpoint_config( @cachedmethod(cache=operator.attrgetter("_partition_cache")) def _list_partitions(self) -> list[str]: checkpoint = self._read_checkpoint() - checkpoint_path = ( - self._filesystem._strip_protocol( # pylint: disable=protected-access - self._checkpoint_config[self._filepath_arg] - ) + checkpoint_path = self._filesystem._strip_protocol( # noqa: protected-access + self._checkpoint_config[self._filepath_arg] ) def _is_valid_partition(partition) -> bool: diff --git a/kedro/ipython/__init__.py b/kedro/ipython/__init__.py index 276a15bc1f..8341822255 100644 --- a/kedro/ipython/__init__.py +++ b/kedro/ipython/__init__.py @@ -16,8 +16,11 @@ from kedro.framework.cli import load_entry_points from kedro.framework.cli.project import PARAMS_ARG_HELP from kedro.framework.cli.utils import ENV_HELP, _split_params -from kedro.framework.project import LOGGING # noqa -from kedro.framework.project import configure_project, pipelines +from kedro.framework.project import ( + LOGGING, # noqa + configure_project, + pipelines, +) from kedro.framework.session import KedroSession from kedro.framework.startup import _is_project, bootstrap_project @@ -65,7 +68,7 @@ def load_ipython_extension(ipython): def magic_reload_kedro(line: str, local_ns: dict[str, Any] = None): """ The `%reload_kedro` IPython line magic. - See https://kedro.readthedocs.io/en/stable/notebooks_and_ipython/kedro_and_notebooks.html#reload-kedro-line-magic # pylint: disable=line-too-long + See https://kedro.readthedocs.io/en/stable/notebooks_and_ipython/kedro_and_notebooks.html#reload-kedro-line-magic # noqa: line-too-long for more. """ args = parse_argstring(magic_reload_kedro, line) @@ -128,7 +131,7 @@ def _resolve_project_path( project_path = Path(path).expanduser().resolve() else: if local_namespace and "context" in local_namespace: - # pylint: disable=protected-access + # noqa: protected-access project_path = local_namespace["context"]._project_path else: project_path = _find_kedro_project(Path.cwd()) @@ -139,7 +142,7 @@ def _resolve_project_path( project_path, ) - # pylint: disable=protected-access + # noqa: protected-access if ( project_path and local_namespace diff --git a/kedro/pipeline/modular_pipeline.py b/kedro/pipeline/modular_pipeline.py index b9dbe9b403..0f429eed56 100644 --- a/kedro/pipeline/modular_pipeline.py +++ b/kedro/pipeline/modular_pipeline.py @@ -150,7 +150,7 @@ def _get_param_names_mapping( return params -def pipeline( +def pipeline( # noqa: too-many-arguments pipe: Iterable[Node | Pipeline] | Pipeline, *, inputs: str | set[str] | dict[str, str] | None = None, @@ -212,7 +212,7 @@ def pipeline( if not any([inputs, outputs, parameters, namespace]): return pipe - # pylint: disable=protected-access + # noqa: protected-access inputs = _get_dataset_names_mapping(inputs) outputs = _get_dataset_names_mapping(outputs) parameters = _get_param_names_mapping(parameters) diff --git a/kedro/pipeline/node.py b/kedro/pipeline/node.py index e7a3963aba..c24780e88f 100644 --- a/kedro/pipeline/node.py +++ b/kedro/pipeline/node.py @@ -19,7 +19,7 @@ class Node: run user-provided functions as part of Kedro pipelines. """ - def __init__( + def __init__( # noqa: too-many-arguments self, func: Callable, inputs: None | str | list[str] | dict[str, str], @@ -518,7 +518,7 @@ def _node_error_message(msg) -> str: ) -def node( +def node( # noqa: too-many-arguments func: Callable, inputs: None | str | list[str] | dict[str, str], outputs: None | str | list[str] | dict[str, str], diff --git a/kedro/pipeline/pipeline.py b/kedro/pipeline/pipeline.py index 5ce57bac35..5b76416182 100644 --- a/kedro/pipeline/pipeline.py +++ b/kedro/pipeline/pipeline.py @@ -32,7 +32,7 @@ def _transcode_split(element: str) -> tuple[str, str]: """ split_name = element.split(TRANSCODING_SEPARATOR) - if len(split_name) > 2: + if len(split_name) > 2: # noqa: PLR2004 raise ValueError( f"Expected maximum 1 transcoding separator, found {len(split_name) - 1} " f"instead: '{element}'." @@ -71,7 +71,7 @@ class ConfirmNotUniqueError(Exception): pass -class Pipeline: # pylint: disable=too-many-public-methods +class Pipeline: # noqa: too-many-public-methods """A ``Pipeline`` defined as a collection of ``Node`` objects. This class treats nodes as part of a graph representation and provides inputs, outputs and execution order. @@ -679,8 +679,7 @@ def only_nodes_with_tags(self, *tags: str) -> Pipeline: nodes = [node for node in self.nodes if tags & node.tags] return Pipeline(nodes) - # pylint: disable=too-many-arguments - def filter( + def filter( # noqa: too-many-arguments self, tags: Iterable[str] = None, from_nodes: Iterable[str] = None, diff --git a/kedro/runner/parallel_runner.py b/kedro/runner/parallel_runner.py index a09d4a9dc2..b9a45792da 100644 --- a/kedro/runner/parallel_runner.py +++ b/kedro/runner/parallel_runner.py @@ -92,20 +92,18 @@ class ParallelRunnerManager(SyncManager): """ -ParallelRunnerManager.register( # pylint: disable=no-member - "MemoryDataset", MemoryDataset -) +ParallelRunnerManager.register("MemoryDataset", MemoryDataset) # noqa: no-member def _bootstrap_subprocess(package_name: str, logging_config: dict[str, Any]): - # pylint: disable=import-outside-toplevel,cyclic-import + # noqa: import-outside-toplevel,cyclic-import from kedro.framework.project import configure_logging, configure_project configure_project(package_name) configure_logging(logging_config) -def _run_node_synchronization( # pylint: disable=too-many-arguments +def _run_node_synchronization( # noqa: too-many-arguments node: Node, catalog: DataCatalog, is_async: bool = False, @@ -166,7 +164,7 @@ def __init__(self, max_workers: int = None, is_async: bool = False): """ super().__init__(is_async=is_async) self._manager = ParallelRunnerManager() - self._manager.start() # pylint: disable=consider-using-with + self._manager.start() # noqa: consider-using-with # This code comes from the concurrent.futures library # https://github.com/python/cpython/blob/master/Lib/concurrent/futures/process.py#L588 @@ -224,7 +222,7 @@ def _validate_catalog(cls, catalog: DataCatalog, pipeline: Pipeline): will not be synchronized across threads. """ - data_sets = catalog._data_sets # pylint: disable=protected-access + data_sets = catalog._data_sets # noqa: protected-access unserialisable = [] for name, data_set in data_sets.items(): @@ -277,7 +275,7 @@ def _get_required_workers_count(self, pipeline: Pipeline): return min(required_processes, self._max_workers) - def _run( # pylint: disable=too-many-locals,useless-suppression + def _run( # noqa: too-many-locals,useless-suppression self, pipeline: Pipeline, catalog: DataCatalog, @@ -300,7 +298,7 @@ def _run( # pylint: disable=too-many-locals,useless-suppression Exception: In case of any downstream node failure. """ - # pylint: disable=import-outside-toplevel,cyclic-import + # noqa: import-outside-toplevel,cyclic-import nodes = pipeline.nodes self._validate_catalog(catalog, pipeline) diff --git a/kedro/runner/runner.py b/kedro/runner/runner.py index 98ad6c4268..be379ace71 100644 --- a/kedro/runner/runner.py +++ b/kedro/runner/runner.py @@ -286,7 +286,7 @@ def _has_persistent_inputs(node: Node, catalog: DataCatalog) -> bool: """ for node_input in node.inputs: - # pylint: disable=protected-access + # noqa: protected-access if isinstance(catalog._data_sets[node_input], MemoryDataset): return False return True @@ -335,7 +335,7 @@ def run_node( return node -def _collect_inputs_from_hook( +def _collect_inputs_from_hook( # noqa: too-many-arguments node: Node, catalog: DataCatalog, inputs: dict[str, Any], @@ -343,7 +343,7 @@ def _collect_inputs_from_hook( hook_manager: PluginManager, session_id: str = None, ) -> dict[str, Any]: - # pylint: disable=too-many-arguments + inputs = inputs.copy() # shallow copy to prevent in-place modification by the hook hook_response = hook_manager.hook.before_node_run( node=node, @@ -364,13 +364,12 @@ def _collect_inputs_from_hook( f"'before_node_run' must return either None or a dictionary mapping " f"dataset names to updated values, got '{response_type}' instead." ) - response = response or {} - additional_inputs.update(response) + additional_inputs.update(response or {}) return additional_inputs -def _call_node_run( +def _call_node_run( # noqa: too-many-arguments node: Node, catalog: DataCatalog, inputs: dict[str, Any], @@ -378,7 +377,7 @@ def _call_node_run( hook_manager: PluginManager, session_id: str = None, ) -> dict[str, Any]: - # pylint: disable=too-many-arguments + try: outputs = node.run(inputs) except Exception as exc: diff --git a/kedro/runner/thread_runner.py b/kedro/runner/thread_runner.py index edf3e1f3c0..6f3d6818d1 100644 --- a/kedro/runner/thread_runner.py +++ b/kedro/runner/thread_runner.py @@ -81,7 +81,7 @@ def _get_required_workers_count(self, pipeline: Pipeline): else required_threads ) - def _run( # pylint: disable=too-many-locals,useless-suppression + def _run( # noqa: too-many-locals,useless-suppression self, pipeline: Pipeline, catalog: DataCatalog, diff --git a/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/pipeline.py b/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/pipeline.py index 670fd43fff..587123c64c 100644 --- a/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/pipeline.py +++ b/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/pipeline.py @@ -3,7 +3,7 @@ generated using Kedro {{ cookiecutter.kedro_version }} """ -from kedro.pipeline import Pipeline, node, pipeline +from kedro.pipeline import Pipeline, pipeline def create_pipeline(**kwargs) -> Pipeline: diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/tests/test_run.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/tests/test_run.py index e714f76314..785c5a40b9 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/tests/test_run.py +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/tests/test_run.py @@ -12,10 +12,10 @@ import pytest -from kedro.framework.project import settings from kedro.config import ConfigLoader from kedro.framework.context import KedroContext from kedro.framework.hooks import _create_hook_manager +from kedro.framework.project import settings @pytest.fixture diff --git a/pyproject.toml b/pyproject.toml index 50bf4a8483..4c5b9af390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ exclude = "/templates/|^features/steps/test_starter" [tool.isort] profile = "black" + [tool.pylint] [tool.pylint.master] ignore = "CVS" @@ -201,3 +202,18 @@ ignore_imports = [ "kedro.framework.context.context -> kedro.config", "kedro.framework.session.session -> kedro.config" ] + +[tool.ruff] +line-length = 88 +show-fixes = true +# select = ["A", "B", "C", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] +select = [ + "F", # Pyflakes + "E", # Pycodestyle + "W", # Pycodestyle + "UP", # pyupgrade + "I", # isort + "PL", # Pylint +] +ignore = ["E501"] # Black take care off line-too-long +unfixable = [] diff --git a/tests/extras/datasets/spark/test_spark_dataset.py b/tests/extras/datasets/spark/test_spark_dataset.py index 6b3a43f23e..da979901ac 100644 --- a/tests/extras/datasets/spark/test_spark_dataset.py +++ b/tests/extras/datasets/spark/test_spark_dataset.py @@ -528,7 +528,7 @@ def test_versioning_existing_dataset( sys.platform.startswith("win"), reason="DBFS doesn't work on Windows" ) class TestSparkDataSetVersionedDBFS: - def test_load_latest( # pylint: disable=too-many-arguments + def test_load_latest( # noqa: too-many-arguments self, mocker, versioned_dataset_dbfs, version, tmp_path, sample_spark_df ): mocked_glob = mocker.patch.object(versioned_dataset_dbfs, "_glob_function") @@ -555,7 +555,7 @@ def test_load_exact(self, tmp_path, sample_spark_df): assert reloaded.exceptAll(sample_spark_df).count() == 0 - def test_save( # pylint: disable=too-many-arguments + def test_save( # noqa: too-many-arguments self, mocker, versioned_dataset_dbfs, version, tmp_path, sample_spark_df ): mocked_glob = mocker.patch.object(versioned_dataset_dbfs, "_glob_function") @@ -568,7 +568,7 @@ def test_save( # pylint: disable=too-many-arguments ) assert (tmp_path / FILENAME / version.save / FILENAME).exists() - def test_exists( # pylint: disable=too-many-arguments + def test_exists( # noqa: too-many-arguments self, mocker, versioned_dataset_dbfs, version, tmp_path, sample_spark_df ): mocked_glob = mocker.patch.object(versioned_dataset_dbfs, "_glob_function") diff --git a/tests/framework/cli/test_jupyter.py b/tests/framework/cli/test_jupyter.py index d5e0b5dbd3..8f363bac3e 100644 --- a/tests/framework/cli/test_jupyter.py +++ b/tests/framework/cli/test_jupyter.py @@ -207,7 +207,7 @@ def tmp_file_path(self): with NamedTemporaryFile() as f: yield Path(f.name) - # pylint: disable=too-many-arguments + # noqa: too-many-arguments def test_convert_one_file_overwrite( self, mocker, diff --git a/tests/framework/cli/test_project.py b/tests/framework/cli/test_project.py index 92e0d024cd..d965113ea8 100644 --- a/tests/framework/cli/test_project.py +++ b/tests/framework/cli/test_project.py @@ -1,5 +1,4 @@ # pylint: disable=unused-argument -import subprocess import sys from pathlib import Path @@ -59,9 +58,7 @@ def test_install_successfully( call_mock.assert_called_once_with(["nbstripout", "--install"]) fake_git_repo.assert_called_once_with( - ["git", "rev-parse", "--git-dir"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + ["git", "rev-parse", "--git-dir"], capture_output=True ) def test_nbstripout_not_installed( diff --git a/tests/framework/cli/test_starters.py b/tests/framework/cli/test_starters.py index d9fe9fcb2a..644e67d592 100644 --- a/tests/framework/cli/test_starters.py +++ b/tests/framework/cli/test_starters.py @@ -48,7 +48,7 @@ def _make_cli_prompt_input(project_name="", repo_name="", python_package=""): return "\n".join([project_name, repo_name, python_package]) -# pylint: disable=too-many-arguments +# noqa: too-many-arguments def _assert_template_ok( result, project_name="New Kedro Project", diff --git a/tests/framework/session/test_session.py b/tests/framework/session/test_session.py index c547cdede3..c3cfb2bf7b 100644 --- a/tests/framework/session/test_session.py +++ b/tests/framework/session/test_session.py @@ -585,7 +585,7 @@ def test_log_error(self, fake_project, mock_package_name): exception = session.store["exception"] assert exception["type"] == "tests.framework.session.test_session.FakeException" - assert exception["value"] == "" + assert not exception["value"] assert any( "raise FakeException" in tb_line for tb_line in exception["traceback"] ) diff --git a/tests/io/test_data_catalog.py b/tests/io/test_data_catalog.py index 74858702bf..574f349809 100644 --- a/tests/io/test_data_catalog.py +++ b/tests/io/test_data_catalog.py @@ -336,7 +336,7 @@ def test_multi_catalog_list_bad_regex(self, multi_catalog): multi_catalog.list("((") def test_eq(self, multi_catalog, data_catalog): - assert multi_catalog == multi_catalog # pylint: disable=comparison-with-itself + assert multi_catalog == multi_catalog # noqa: PLR0124 assert multi_catalog == multi_catalog.shallow_copy() assert multi_catalog != data_catalog diff --git a/tests/pipeline/test_modular_pipeline.py b/tests/pipeline/test_modular_pipeline.py index ec37759137..8e4f06330f 100644 --- a/tests/pipeline/test_modular_pipeline.py +++ b/tests/pipeline/test_modular_pipeline.py @@ -219,7 +219,7 @@ def test_empty_output(self): ) def test_missing_dataset_name( self, func, inputs, outputs, inputs_map, outputs_map, expected_missing - ): # pylint: disable=too-many-arguments + ): # noqa: too-many-arguments raw_pipeline = modular_pipeline([node(func, inputs, outputs)]) with pytest.raises(ModularPipelineError, match=r"Failed to map datasets") as e: diff --git a/tests/runner/test_parallel_runner.py b/tests/runner/test_parallel_runner.py index 6870561d8d..a74cff8d53 100644 --- a/tests/runner/test_parallel_runner.py +++ b/tests/runner/test_parallel_runner.py @@ -103,7 +103,7 @@ def test_specified_max_workers_bellow_cpu_cores_count( cpu_cores, user_specified_number, expected_number, - ): # pylint: disable=too-many-arguments + ): # noqa: too-many-arguments """ The system has 2 cores, but we initialize the runner with max_workers=4. `fan_out_fan_in` pipeline needs 3 processes. @@ -250,9 +250,7 @@ def _describe(self) -> dict[str, Any]: if not sys.platform.startswith("win"): - ParallelRunnerManager.register( # pylint: disable=no-member - "LoggingDataset", LoggingDataset - ) + ParallelRunnerManager.register("LoggingDataset", LoggingDataset) # noqa: no-member @pytest.mark.skipif( @@ -267,7 +265,7 @@ def test_dont_release_inputs_and_outputs(self, is_async): pipeline = modular_pipeline( [node(identity, "in", "middle"), node(identity, "middle", "out")] ) - # pylint: disable=no-member + # noqa: no-member catalog = DataCatalog( { "in": runner._manager.LoggingDataset(log, "in", "stuff"), @@ -291,7 +289,7 @@ def test_release_at_earliest_opportunity(self, is_async): node(sink, "second", None), ] ) - # pylint: disable=no-member + # noqa: no-member catalog = DataCatalog( { "first": runner._manager.LoggingDataset(log, "first"), @@ -319,7 +317,7 @@ def test_count_multiple_loads(self, is_async): node(sink, "dataset", None, name="fred"), ] ) - # pylint: disable=no-member + # noqa: no-member catalog = DataCatalog( {"dataset": runner._manager.LoggingDataset(log, "dataset")} ) diff --git a/tests/runner/test_thread_runner.py b/tests/runner/test_thread_runner.py index 7d86592379..a9348548a7 100644 --- a/tests/runner/test_thread_runner.py +++ b/tests/runner/test_thread_runner.py @@ -55,7 +55,7 @@ def test_specified_max_workers( catalog, user_specified_number, expected_number, - ): # pylint: disable=too-many-arguments + ): # noqa: too-many-arguments """ We initialize the runner with max_workers=4. `fan_out_fan_in` pipeline needs 3 threads. diff --git a/tools/print_env.sh b/tools/print_env.sh index a82ac29153..0a559a6d25 100755 --- a/tools/print_env.sh +++ b/tools/print_env.sh @@ -16,11 +16,9 @@ eval_command() { eval_command CONDA "conda info 2>/dev/null || echo \"Conda not found\"" eval_command PYTHON "which python && python -V" eval_command PIP "python -m pip -V" -eval_command PYLINT "python -m pylint --version" eval_command PYTEST "python -m pytest --version" eval_command BLACK "python -m black --version" eval_command BEHAVE "python -m behave --version" -eval_command ISORT "python -m isort --version" eval_command PRE-COMMIT "python -m pre_commit --version" eval_command SPARK "python -c \\ \"import pyspark; print(f'PySpark: {pyspark.__version__}')\" 2>/dev/null && \\ From 0ac76752c469f8997602f61d0e5ff948d7855f14 Mon Sep 17 00:00:00 2001 From: Ivan Danov Date: Wed, 26 Jul 2023 20:48:31 +0100 Subject: [PATCH 4/5] Add downloads badges to README.md (#2795) * Update README.md Add downloads badges to the README.md * Update README.md --------- Co-authored-by: Jo Stichbury --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 80d92f723e..a909df2535 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ ![Develop Branch Build](https://img.shields.io/circleci/build/github/kedro-org/kedro/develop?label=develop) [![Documentation](https://readthedocs.org/projects/kedro/badge/?version=stable)](https://docs.kedro.org/) [![OpenSSF Best Practices](https://bestpractices.coreinfrastructure.org/projects/6711/badge)](https://bestpractices.coreinfrastructure.org/projects/6711) +[![Monthly downloads](https://static.pepy.tech/badge/kedro/month)](https://pepy.tech/project/kedro) +[![Total downloads](https://static.pepy.tech/badge/kedro)](https://pepy.tech/project/kedro) ## What is Kedro? From acff3e23557944dec65f0171a3f97c1e59e77659 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 09:16:19 +0200 Subject: [PATCH 5/5] Update jmespath requirement from <1.0,>=0.9.5 to >=0.9.5,<2.0 (#2810) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the requirements on [jmespath](https://github.com/jmespath/jmespath.py) to permit the latest version. - [Changelog](https://github.com/jmespath/jmespath.py/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/jmespath/jmespath.py/compare/0.9.5...1.0.1) --- updated-dependencies: - dependency-name: jmespath dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Juan Luis Cano Rodríguez --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4c5b9af390..0a46e7e0dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "importlib-metadata>=3.6; python_version >= '3.8'", "importlib_metadata>=3.6, <5.0; python_version < '3.8'", # The "selectable" entry points were introduced in `importlib_metadata` 3.6 and Python 3.10. Bandit on Python 3.7 relies on a library with `importlib_metadata` < 5.0 "importlib_resources>=1.3", # The `files()` API was introduced in `importlib_resources` 1.3 and Python 3.9. - "jmespath>=0.9.5, <1.0", + "jmespath>=0.9.5, <2.0", "more_itertools~=9.0", "omegaconf~=2.3", "parse~=1.19.0",