Merge branch 'main' into feat/add-factories-cli-commands

kedro-org · Jul 27, 2023 · c346c17 · c346c17
2 parents 219729b + acff3e2
commit c346c17
Show file tree

Hide file tree

Showing 101 changed files with 567 additions and 382 deletions.
diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml
@@ -0,0 +1,44 @@
+name: Run all checks on Kedro
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - "docs/**"
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - "docs/**"
+
+jobs:
+  unit-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  lint:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.10" ]
+    uses: ./.github/workflows/lint.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  e2e-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/e2e-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/docs-only-checks.yml b/.github/workflows/docs-only-checks.yml
@@ -0,0 +1,24 @@
+name: Run linter on Kedro Docs
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+
+jobs:
+  lint-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/lint.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -0,0 +1,46 @@
+name: Run e2e-tests on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+
+env:
+  COLUMNS: 120
+  LINES: 25
+
+jobs:
+  e2e-tests:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{inputs.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{inputs.python-version}}
+      - run: make install-pip-setuptools
+      - name: Cache python packages for Linux
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Cache python packages for Windows
+        if: inputs.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Install dependencies
+        run: |
+          pip --version
+          make install-test-requirements
+          make install-pre-commit
+      - name: pip freeze
+        run: pip freeze
+      - name: Run e2e tests
+        run: make e2e-tests
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -0,0 +1,32 @@
+name: Run linters on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+
+jobs:
+  lint:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Install dependencies
+        run: |
+            make install-test-requirements
+            make install-pre-commit
+            pip freeze
+      - name: Run linter
+        run: make lint
diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml
@@ -0,0 +1,27 @@
+name: Merge Gatekeeper
+
+on:
+  pull_request:
+    branches:
+      - main
+      - develop
+
+jobs:
+  merge-gatekeeper:
+    runs-on: ubuntu-latest
+    # Restrict permissions of the GITHUB_TOKEN.
+    # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+    permissions:
+      checks: read
+      statuses: read
+    steps:
+      - name: Run Merge Gatekeeper
+        # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs:
+        #       https://github.com/upsidr/merge-gatekeeper/tags
+        #       https://github.com/upsidr/merge-gatekeeper/branches
+        uses: upsidr/merge-gatekeeper@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          timeout: 1800
+          interval: 30
+          ignored: 'ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circlecici: check-updated-files,regular'
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -0,0 +1,53 @@
+name: Run unit-tests on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+jobs:
+  unit-tests:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{inputs.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{inputs.python-version}}
+      - run: make install-pip-setuptools
+      - name: Cache python packages for Linux
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Cache python packages for Windows
+        if: inputs.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Install dependencies
+        run: |
+            make install-test-requirements
+            make install-pre-commit
+      - name: Install pytables (only for windows)
+        if: inputs.os == 'windows-latest'
+        run: pip install tables
+      - name: pip freeze
+        run: pip freeze
+      - name: Run unit tests
+        if: inputs.os == 'ubuntu-latest' && inputs.python-version == '3.10'
+        run: make test-sequential
+      - name: Run unit tests
+        if: inputs.os == 'ubuntu-latest' && inputs.python-version != '3.10'
+        run: make test
+      - name: Run unit tests (Windows)
+        if: inputs.os == 'windows-latest' && inputs.python-version == '3.10'
+        run: make test-no-spark-sequential
+      - name: Run unit tests (Windows)
+        if: inputs.os == 'windows-latest' && inputs.python-version != '3.10'
+        run: make test-no-spark
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,27 +19,57 @@ repos:
             exclude: "^kedro/templates/|^features/steps/test_starter/"
           - id: requirements-txt-fixer # Sorts entries in requirements.txt
             exclude: "^kedro/templates/|^features/steps/test_starter/"
+
+    - repo: https://github.com/pre-commit/mirrors-mypy
+      rev: v0.961
+      hooks:
+          - id: mypy
+            args: [--allow-redefinition, --ignore-missing-imports]
+            exclude: |
+                (?x)(
+                  ^kedro/templates/|
+                  ^docs/|
+                  ^features/steps/test_starter/
+                )
+            additional_dependencies:
+            - types-cachetools
+            - types-filelock
+            - types-PyYAML
+            - types-redis
+            - types-requests
+            - types-setuptools
+            - types-toml
+            - attrs
+
     - repo: https://github.com/asottile/blacken-docs
       rev: v1.12.1
       hooks:
           - id: blacken-docs
             additional_dependencies: [black~=22.0]
             entry: blacken-docs --skip-errors
 
-    - repo: https://github.com/asottile/pyupgrade
-      rev: v2.26.0
+    - repo: https://github.com/astral-sh/ruff-pre-commit
+      # Ruff version.
+      rev: v0.0.277
+      hooks:
+        - id: ruff
+          name: "ruff on kedro/"
+          args: ["--fix", "--show-fixes", "--exit-non-zero-on-fix"]
+          exclude: "^kedro/templates/|^features/steps/test_starter/|tests|docs"
+    - repo: https://github.com/astral-sh/ruff-pre-commit
+      # Ruff version.
+      rev: v0.0.277
       hooks:
-          - id: pyupgrade
-            args: [--py36-plus]
+        - id: ruff
+          name: "ruff on tests/ and docs/"
+          # PLR2004: Magic value used
+          # PLR0913: Too many arguments
+          args: ["--fix", "--show-fixes", "--exit-non-zero-on-fix", "--ignore=PLR2004,PLR0913"]
+          # include: "tests"
+          exclude: "^kedro/templates/|^features/steps/test_starter/|kedro"
 
     - repo: local
       hooks:
-          - id: isort
-            name: "Sort imports"
-            language: system
-            types: [file, python]
-            exclude: ^kedro/templates/|^features/steps/test_starter
-            entry: isort
           - id: black
             name: "Black"
             language: system
@@ -53,65 +83,6 @@ repos:
             pass_filenames: false
             entry: lint-imports
 
-
-          # It's impossible to specify per-directory configuration, so we just run it many times.
-          # https://github.com/PyCQA/pylint/issues/618
-          # The first set of pylint checks if for local pre-commit, it only runs on the files changed.
-          - id: pylint-quick-kedro
-            name: "Quick Pylint on kedro/*"
-            language: system
-            types: [file, python]
-            files: ^kedro/
-            exclude: ^kedro/templates/
-            entry: pylint -j 4 --disable=unnecessary-pass
-            stages: [commit]
-          - id: pylint-quick-tests
-            name: "Quick Pylint on tests/*"
-            language: system
-            types: [file, python]
-            files: ^tests/
-            entry: pylint -j 4 --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,too-many-public-methods
-            stages: [commit]
-          - id: pylint-quick-features
-            name: "Quick Pylint on features/*"
-            language: system
-            types: [file, python]
-            files: ^features/
-            exclude: ^features/steps/test_starter
-            entry: pylint -j 4 --disable=missing-docstring,no-name-in-module
-            stages: [commit]
-
-          # The same pylint checks, but running on all files. It's for manual run with `make lint`
-          - id: pylint-kedro
-            name: "Pylint on kedro/*"
-            language: system
-            pass_filenames: false
-            stages: [manual]
-            entry: pylint -j 4 --disable=unnecessary-pass --init-hook="import sys; sys.setrecursionlimit(2000)" kedro
-          - id: pylint-tests
-            name: "Pylint on tests/*"
-            language: system
-            pass_filenames: false
-            stages: [manual]
-            entry: pylint -j 4 --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments,too-many-public-methods,use-implicit-booleaness-not-comparison tests
-          - id: pylint-features
-            name: "Pylint on features/*"
-            language: system
-            pass_filenames: false
-            stages: [manual]
-            exclude: ^features/steps/test_starter
-            entry: pylint -j 4 --disable=missing-docstring,no-name-in-module features
-    - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v2.2.3
-      hooks:
-          - id: flake8
-            args:
-            - "--max-line-length=88"
-            - "--max-complexity=18"
-            - "--max-complexity=18"
-            - "--select=B,C,E,F,W,T4,B9"
-            - "--ignore=E203,E266,E501,W503"
-            exclude: "^kedro/templates/|^features/steps/test_starter/"
     - repo: https://github.com/pre-commit/mirrors-mypy
       rev: v0.961
       hooks:
@@ -147,3 +118,5 @@ repos:
             types: [file, python]
             exclude: ^kedro/templates/|^tests/|^features/steps/test_starter
             entry: bandit -ll
+
+# Manual only
diff --git a/Makefile b/Makefile
@@ -8,17 +8,22 @@ clean:
 	pre-commit clean || true
 
 install-pip-setuptools:
-	pip install -U "pip>=21.2" "setuptools>=65.5.1" wheel
+	python -m pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel
 
 lint:
 	pre-commit run -a --hook-stage manual $(hook)
-
 test:
 	pytest --numprocesses 4 --dist loadfile
 
 test-no-spark:
 	pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile
 
+test-sequential:
+	pytest tests --cov-config pyproject.toml
+
+test-no-spark-sequential:
+	pytest tests --no-cov --ignore tests/extras/datasets/spark
+
 test-no-datasets:
 	pytest --no-cov --ignore tests/extras/datasets/ --numprocesses 4 --dist loadfile