Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pin pip < 23.2 and Port unit tests, lint and e2e tests to Github Actions unblock CI #2843

Merged
merged 14 commits into from
Jul 26, 2023
Merged
44 changes: 44 additions & 0 deletions .github/workflows/all-checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: Run all checks on Kedro

on:
push:
branches:
- main
paths-ignore:
- "docs/**"
pull_request:
branches:
- main
paths-ignore:
- "docs/**"

jobs:
unit-tests:
strategy:
matrix:
os: [ ubuntu-latest, windows-latest ]
python-version: [ "3.7", "3.8", "3.9", "3.10" ]
uses: ./.github/workflows/unit-tests.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}

lint:
strategy:
matrix:
os: [ ubuntu-latest ]
python-version: [ "3.10" ]
uses: ./.github/workflows/lint.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}

e2e-tests:
strategy:
matrix:
os: [ ubuntu-latest, windows-latest ]
python-version: [ "3.7", "3.8", "3.9", "3.10" ]
uses: ./.github/workflows/e2e-tests.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}
24 changes: 24 additions & 0 deletions .github/workflows/docs-only-checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: Run linter on Kedro Docs

on:
push:
branches:
- main
paths:
- "docs/**"
pull_request:
branches:
- main
paths:
- "docs/**"

jobs:
lint-tests:
strategy:
matrix:
os: [ ubuntu-latest ]
python-version: [ "3.7", "3.8", "3.9", "3.10" ]
uses: ./.github/workflows/lint.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}
46 changes: 46 additions & 0 deletions .github/workflows/e2e-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Run e2e-tests on Kedro

on:
workflow_call:
inputs:
os:
type: string
python-version:
type: string

env:
COLUMNS: 120
LINES: 25

jobs:
e2e-tests:
runs-on: ${{ inputs.os }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Python ${{inputs.python-version}}
uses: actions/setup-python@v3
with:
python-version: ${{inputs.python-version}}
- run: make install-pip-setuptools
- name: Cache python packages for Linux
if: inputs.os == 'ubuntu-latest'
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Cache python packages for Windows
if: inputs.os == 'windows-latest'
uses: actions/cache@v3
with:
path: ~\AppData\Local\pip\Cache
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Install dependencies
run: |
pip --version
make install-test-requirements
make install-pre-commit
- name: pip freeze
run: pip freeze
- name: Run e2e tests
run: make e2e-tests
32 changes: 32 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: Run linters on Kedro

on:
workflow_call:
inputs:
os:
type: string
python-version:
type: string

jobs:
lint:
runs-on: ${{ inputs.os }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Python ${{ inputs.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ inputs.python-version }}
- name: Cache python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Install dependencies
run: |
make install-test-requirements
make install-pre-commit
pip freeze
- name: Run linter
run: make lint
26 changes: 26 additions & 0 deletions .github/workflows/merge-gatekeeper.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
name: Merge Gatekeeper

on:
pull_request:
branches:
- main
- develop

jobs:
merge-gatekeeper:
runs-on: ubuntu-latest
# Restrict permissions of the GITHUB_TOKEN.
# Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
permissions:
checks: read
statuses: read
steps:
- name: Run Merge Gatekeeper
# NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs:
# https://github.com/upsidr/merge-gatekeeper/tags
# https://github.com/upsidr/merge-gatekeeper/branches
uses: upsidr/merge-gatekeeper@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
timeout: 1800
interval: 30
53 changes: 53 additions & 0 deletions .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: Run unit-tests on Kedro

on:
workflow_call:
inputs:
os:
type: string
python-version:
type: string
jobs:
unit-tests:
runs-on: ${{ inputs.os }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Python ${{inputs.python-version}}
uses: actions/setup-python@v3
with:
python-version: ${{inputs.python-version}}
- run: make install-pip-setuptools
- name: Cache python packages for Linux
if: inputs.os == 'ubuntu-latest'
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Cache python packages for Windows
if: inputs.os == 'windows-latest'
uses: actions/cache@v3
with:
path: ~\AppData\Local\pip\Cache
key: ${{inputs.os}}-python-${{inputs.python-version}}
- name: Install dependencies
run: |
make install-test-requirements
make install-pre-commit
- name: Install pytables (only for windows)
if: inputs.os == 'windows-latest'
run: pip install tables
- name: pip freeze
run: pip freeze
- name: Run unit tests
if: inputs.os == 'ubuntu-latest' && inputs.python-version == '3.10'
run: make test-sequential
- name: Run unit tests
if: inputs.os == 'ubuntu-latest' && inputs.python-version != '3.10'
run: make test
- name: Run unit tests (Windows)
if: inputs.os == 'windows-latest' && inputs.python-version == '3.10'
run: make test-no-spark-sequential
- name: Run unit tests (Windows)
if: inputs.os == 'windows-latest' && inputs.python-version != '3.10'
run: make test-no-spark
8 changes: 7 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ clean:
pre-commit clean || true

install-pip-setuptools:
pip install -U "pip>=21.2" "setuptools>=65.5.1" wheel
python -m pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel

lint:
pre-commit run -a --hook-stage manual $(hook)
Expand All @@ -19,6 +19,12 @@ test:
test-no-spark:
pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile

test-sequential:
pytest tests --cov-config pyproject.toml

test-no-spark-sequential:
pytest tests --no-cov --ignore tests/extras/datasets/spark

test-no-datasets:
pytest --no-cov --ignore tests/extras/datasets/ --numprocesses 4 --dist loadfile

Expand Down
4 changes: 3 additions & 1 deletion features/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ def _setup_minimal_env(context):
"pip",
"install",
"-U",
"pip>=21.2",
# pip==23.2 breaks pip-tools<7.0, and pip-tools>=7.0 does not support Python 3.7
"pip>=21.2,<23.2; python_version < '3.8'",
"pip>=21.2; python_version >= '3.8'",
"setuptools>=65.5.1",
"wheel",
],
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ dependencies = [
"more_itertools~=9.0",
"omegaconf~=2.3",
"parse~=1.19.0",
"pip-tools~=6.5",
"pip-tools>=6.5,<8",
"pluggy~=1.0",
"PyYAML>=4.2, <7.0",
"rich>=12.0, <14.0",
Expand Down
2 changes: 2 additions & 0 deletions trufflehog-ignore.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@ static/img/kedro_gitflow.svg
.coverage.*
.*\.log
.*\.iml
tests/extras/datasets/tensorflow/test_tensorflow_model_dataset.py
docs/source/meta/images/kedro_gitflow.svg