From 66dcd9389eb7a8f60fa5d65cb8544123c4204452 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 26 Jul 2023 18:31:47 +0100 Subject: [PATCH 1/3] Add pip compile step Signed-off-by: Ankita Katiyar --- .github/workflows/all-checks.yml | 10 ++++++++++ .github/workflows/merge-gatekeeper.yml | 2 +- .github/workflows/pip-compile.yml | 22 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pip-compile.yml diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml index 9f7abf3163..f2a6c1e231 100644 --- a/.github/workflows/all-checks.yml +++ b/.github/workflows/all-checks.yml @@ -42,3 +42,13 @@ jobs: with: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} + + pip-compile: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/pip-compile.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml index 4f5393a1e6..ff716a75fd 100644 --- a/.github/workflows/merge-gatekeeper.yml +++ b/.github/workflows/merge-gatekeeper.yml @@ -24,4 +24,4 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} timeout: 1800 interval: 30 - ignored: 'ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circlecici: check-updated-files,regular' + ignored: 'ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circleci: check-updated-files,regular' diff --git a/.github/workflows/pip-compile.yml b/.github/workflows/pip-compile.yml new file mode 100644 index 0000000000..b96895b4ff --- /dev/null +++ b/.github/workflows/pip-compile.yml @@ -0,0 +1,22 @@ +name: Run pip-compile on Kedro + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string + +jobs: + pip-compile: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ inputs.python-version }} + - name: Pip compile + run: make pip-compile From 67f4bf11028dbb666f2db5ee20e1f449ac423fb0 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 27 Jul 2023 10:18:03 +0100 Subject: [PATCH 2/3] Update pip compile step Signed-off-by: Ankita Katiyar --- .github/workflows/pip-compile.yml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pip-compile.yml b/.github/workflows/pip-compile.yml index b96895b4ff..796ba6049c 100644 --- a/.github/workflows/pip-compile.yml +++ b/.github/workflows/pip-compile.yml @@ -1,4 +1,4 @@ -name: Run pip-compile on Kedro +name: Run pip-compile on: workflow_call: @@ -14,9 +14,26 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python ${{ inputs.python-version }} + - name: Set up Python ${{inputs.python-version}} uses: actions/setup-python@v3 with: - python-version: ${{ inputs.python-version }} - - name: Pip compile + python-version: ${{inputs.python-version}} + - run: make install-pip-setuptools + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + - name: Install dependencies + run: | + make install-test-requirements + make install-pre-commit + - name: Run pip-compile run: make pip-compile From 02225d89f67c9a7421f327436a8b47f5d493af21 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 27 Jul 2023 11:05:52 +0100 Subject: [PATCH 3/3] Minor changes to other wfs + Add develop branch Signed-off-by: Ankita Katiyar --- .github/workflows/all-checks.yml | 2 ++ .github/workflows/docs-only-checks.yml | 2 ++ .github/workflows/e2e-tests.yml | 1 - .github/workflows/lint.yml | 3 ++- .github/workflows/unit-tests.yml | 6 +++--- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml index f2a6c1e231..767cc46378 100644 --- a/.github/workflows/all-checks.yml +++ b/.github/workflows/all-checks.yml @@ -4,11 +4,13 @@ on: push: branches: - main + - develop paths-ignore: - "docs/**" pull_request: branches: - main + - develop paths-ignore: - "docs/**" diff --git a/.github/workflows/docs-only-checks.yml b/.github/workflows/docs-only-checks.yml index aa53fdd604..4d6870d627 100644 --- a/.github/workflows/docs-only-checks.yml +++ b/.github/workflows/docs-only-checks.yml @@ -4,11 +4,13 @@ on: push: branches: - main + - develop paths: - "docs/**" pull_request: branches: - main + - develop paths: - "docs/**" diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 6c317c7c40..593af27299 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -37,7 +37,6 @@ jobs: key: ${{inputs.os}}-python-${{inputs.python-version}} - name: Install dependencies run: | - pip --version make install-test-requirements make install-pre-commit - name: pip freeze diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 81712415fd..a3222d1d9f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,6 +27,7 @@ jobs: run: | make install-test-requirements make install-pre-commit - pip freeze + - name: pip freeze + run: pip freeze - name: Run linter run: make lint diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 87ca8e0eed..2434f7811f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -39,15 +39,15 @@ jobs: run: pip install tables - name: pip freeze run: pip freeze - - name: Run unit tests + - name: Run unit tests sequentially if: inputs.os == 'ubuntu-latest' && inputs.python-version == '3.10' run: make test-sequential - name: Run unit tests if: inputs.os == 'ubuntu-latest' && inputs.python-version != '3.10' run: make test - - name: Run unit tests (Windows) + - name: Run unit tests without spark sequentially (Windows) if: inputs.os == 'windows-latest' && inputs.python-version == '3.10' run: make test-no-spark-sequential - - name: Run unit tests (Windows) + - name: Run unit tests without spark (Windows) if: inputs.os == 'windows-latest' && inputs.python-version != '3.10' run: make test-no-spark