Merge branch 'main' into chore/deprecate-abstract-data-set

.circleci/continue_config.yml

@@ -61,7 +61,7 @@ commands:
           command: conda install -c conda-forge pytables -y
       - run:
           name: Install requirements and test requirements
-          command: pip install --upgrade -r test_requirements.txt
+          command: pip install --upgrade .[test]
       - run:
           # this is needed to fix java cacerts so
           # spark can automatically download packages from mvn
@@ -146,7 +146,7 @@ commands:
     steps:
       - restore_cache:
           name: Restore package cache
-          key: kedro-deps-v1-win-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+          key: kedro-deps-v1-win-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
         # We don't restore the conda environment cache for python 3.10 as it conflicts with the
         # 'Install GDAL, Fiona and pytables' step breaking the conda environment (missing zlib.dll).
       - unless:
@@ -155,7 +155,7 @@ commands:
           steps:
             - restore_cache:
                 name: Restore conda environment cache
-                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
         # pytables and Fiona have a series of binary dependencies under Windows that
         # are best handled by conda-installing instead of pip-installing them.
         # Dependency resolution works best when installing these altogether in one
@@ -168,7 +168,7 @@ commands:
           command: conda activate kedro_builder; pip debug --verbose
       - run:
           name: Install all requirements
-          command: conda activate kedro_builder; pip install -v -r test_requirements.txt -U
+          command: conda activate kedro_builder; pip install -v -U .[test]
       - run:
           name: Print Python environment
           command: conda activate kedro_builder; make print-python-env
@@ -337,7 +337,7 @@ jobs:
           steps:
           - save_cache:
               name: Save Python package cache
-              key: kedro-deps-v1-win-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+              key: kedro-deps-v1-win-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
               paths:
                 # Cache pip cache and conda packages directories
                 - c:\tools\miniconda3\pkgs
@@ -350,7 +350,7 @@ jobs:
           steps:
             - save_cache:
                 name: Save conda environment cache
-                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
                 paths:
                   - c:\tools\miniconda3\envs\kedro_builder
       - run:

.github/ISSUE_TEMPLATE/design-doc.md

@@ -8,37 +8,29 @@ assignees: ''
 ---
 
 ## Introduction
-
-A high-level, short overview of the problem(s) you are designing a solution for.
+<!-- A high-level, short overview of the problem(s) you are designing a solution for. -->
 
 ## Background
-
-Provide the reader with the context surrounding the problem(s) you are trying to solve.
+<!-- Provide the reader with the context surrounding the problem(s) you are trying to solve. -->
 
 ## Problem
-
-Be as concrete as you can about:
+<!-- Be as concrete as you can about: -->
 
 ### What's in scope
 
 ### What's not in scope
 
 ## Design
-
-Explain your design to the solution here. Diagrams could help.
+<!-- Explain your design to the solution here. Diagrams could help. -->
 
 ### Alternatives considered
-
-Explain the trade off between different alternatives to your solution.
+<!-- Explain the trade off between different alternatives to your solution. -->
 
 ## Testing
-
-Explain the testing strategies to verify your design correctness (if possible).
+<!-- Explain the testing strategies to verify your design correctness (if possible). -->
 
 ## Rollout strategy
-
-Is the change backward compatible? If not, what is the migration strategy?
+<!-- Is the change backward compatible? If not, what is the migration strategy? -->
 
 ## Future iterations
-
-Will there be future iterations of this design?
+<!-- Will there be future iterations of this design? -->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -8,13 +8,13 @@ assignees: ''
 ---
 
 ## Description
-Is your feature request related to a problem? A clear and concise description of what the problem is: "I'm always frustrated when ..."
+<!-- Is your feature request related to a problem? A clear and concise description of what the problem is: "I'm always frustrated when ..." -->
 
 ## Context
-Why is this change important to you? How would you use it? How can it benefit other users?
+<!-- Why is this change important to you? How would you use it? How can it benefit other users? -->
 
 ## Possible Implementation
-(Optional) Suggest an idea for implementing the addition or change.
+<!-- (Optional) Suggest an idea for implementing the addition or change. -->
 
 ## Possible Alternatives
-(Optional) Describe any alternative solutions or features you've considered.
+<!-- (Optional) Describe any alternative solutions or features you've considered. -->

.github/dependabot.yml

@@ -6,7 +6,7 @@
 version: 2
 updates:
   - package-ecosystem: "pip" # See documentation for possible values
-    directory: "/dependency" # Location of package manifests
+    directory: "/" # Location of package manifests
     schedule:
       interval: "weekly"
     labels:

.github/workflows/all-checks.yml

@@ -0,0 +1,58 @@
+name: Run all checks on Kedro
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+    paths-ignore:
+      - "docs/**"
+      - '**.md'
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths-ignore:
+      - "docs/**"
+      - '**.md'
+
+jobs:
+  unit-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  lint:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.11" ]
+    uses: ./.github/workflows/lint.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  e2e-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/e2e-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  pip-compile:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/pip-compile.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}

.github/workflows/docs-only-checks.yml

@@ -0,0 +1,28 @@
+name: Run linter on Kedro Docs
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+    paths:
+      - "docs/**"
+      - '**.md'
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths:
+      - "docs/**"
+      - '**.md'
+
+jobs:
+  lint-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/lint.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}

.github/workflows/e2e-tests.yml

@@ -0,0 +1,48 @@
+name: Run e2e-tests on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+
+env:
+  COLUMNS: 120
+  LINES: 25
+
+jobs:
+  e2e-tests:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{inputs.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{inputs.python-version}}
+      - run: make install-pip-setuptools
+      - name: Cache python packages for Linux
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Cache python packages for Windows
+        if: inputs.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Add MSBuild to PATH
+        if: inputs.os == 'windows-latest'
+        uses: microsoft/setup-msbuild@v1
+      - name: Install dependencies
+        run: |
+          make install-test-requirements
+          make install-pre-commit
+      - name: pip freeze
+        run: pip freeze
+      - name: Run e2e tests
+        run: make e2e-tests

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Run linters on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+
+jobs:
+  lint:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Install dependencies
+        run: |
+            make install-test-requirements
+            make install-pre-commit
+      - name: pip freeze
+        run: pip freeze
+      - name: Run linter
+        run: make lint

.github/workflows/merge-gatekeeper.yml

@@ -0,0 +1,27 @@
+name: Merge Gatekeeper
+
+on:
+  pull_request:
+    branches:
+      - main
+      - develop
+
+jobs:
+  merge-gatekeeper:
+    runs-on: ubuntu-latest
+    # Restrict permissions of the GITHUB_TOKEN.
+    # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+    permissions:
+      checks: read
+      statuses: read
+    steps:
+      - name: Run Merge Gatekeeper
+        # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs:
+        #       https://github.com/upsidr/merge-gatekeeper/tags
+        #       https://github.com/upsidr/merge-gatekeeper/branches
+        uses: upsidr/merge-gatekeeper@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          timeout: 1800
+          interval: 30
+          ignored: 'ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circleci: check-updated-files,regular'

.github/workflows/pip-compile.yml

@@ -0,0 +1,42 @@
+name: Run pip-compile
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+
+jobs:
+  pip-compile:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{inputs.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{inputs.python-version}}
+      - run: make install-pip-setuptools
+      - name: Cache python packages for Linux
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Cache python packages for Windows
+        if: inputs.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Add MSBuild to PATH
+        if: inputs.os == 'windows-latest'
+        uses: microsoft/setup-msbuild@v1
+      - name: Install dependencies
+        run: |
+          make install-test-requirements
+          make install-pre-commit
+      - name: Run pip-compile
+        run: make pip-compile