diff --git a/.gitpod.yml b/.gitpod.yml index 70738f4c0..f01e52544 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,6 +1,4 @@ -# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.10:2023-04-20-16-32-37 - +image: gitpod/workspace-python-3.11 tasks: # We want packages installed during the pre-build init steps to go to /workspace @@ -12,22 +10,16 @@ tasks: echo PIP_USER=no >> ~/.bashrc && export PIP_USER=no init: | make sign-off + pip install uv + uv venv + echo source .venv/bin/activate >> ~/.bashrc + source ~/.bashrc + make install-test-requirements plugin=kedro-datasets command: | pre-commit install --install-hooks clear - -github: - prebuilds: - # enable for the master/default branch (defaults to true) - master: true - # enable for all branches in this repo (defaults to false) - branches: true - # enable for pull requests coming from this repo (defaults to true) - pullRequests: true - # enable for pull requests coming from forks (defaults to false) - pullRequestsFromForks: true - # add a "Review in Gitpod" button as a comment to pull requests (defaults to true) - addComment: false - # add a "Review in Gitpod" button to pull requests (defaults to false) - addBadge: true + - name: system + init: | + sudo apt-get update && sudo apt-get install -y --no-install-recommends libgl1 make + sudo apt-get install -y --no-install-recommends libatk-bridge2.0-0 libcups2 ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils diff --git a/Makefile b/Makefile index 22bc17816..324332972 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,6 @@ package: rm -Rf dist;\ python -m build -pypi: - python -m pip install twine -U - python -m twine upload $(plugin)/dist/* - -install: package - cd $(plugin) && pip install -U dist/*.whl - install-pip-setuptools: python -m pip install -U pip setuptools wheel @@ -25,46 +18,14 @@ mypy: test: cd $(plugin) && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile -# Run test_tensorflow_model_dataset separately, because these tests are flaky when run as part of the full test-suite -dataset-tests: dataset-doctests - cd kedro-datasets && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile --ignore tests/tensorflow - cd kedro-datasets && pytest tests/tensorflow/test_tensorflow_model_dataset.py --no-cov - -extra_pytest_args-no-spark=--ignore kedro_datasets/databricks --ignore kedro_datasets/spark -extra_pytest_args= -dataset-doctest%: - if [ "${*}" != 's-no-spark' ] && [ "${*}" != 's' ]; then \ - echo "make: *** No rule to make target \`${@}\`. Stop."; \ - exit 2; \ - fi; \ - \ - # The ignored datasets below require complicated setup with cloud/database clients which is overkill for the doctest examples. - cd kedro-datasets && pytest kedro_datasets --doctest-modules --doctest-continue-on-failure --no-cov \ - --ignore kedro_datasets/pandas/gbq_dataset.py \ - --ignore kedro_datasets/partitions/partitioned_dataset.py \ - --ignore kedro_datasets/redis/redis_dataset.py \ - --ignore kedro_datasets/snowflake/snowpark_dataset.py \ - --ignore kedro_datasets/spark/spark_hive_dataset.py \ - --ignore kedro_datasets/spark/spark_jdbc_dataset.py \ - $(extra_pytest_arg${*}) - -test-sequential: - cd $(plugin) && pytest tests --cov-config pyproject.toml - e2e-tests: cd $(plugin) && behave secret-scan: trufflehog --max_depth 1 --exclude_paths trufflehog-ignore.txt . -clean: - cd $(plugin);\ - rm -rf build dist pip-wheel-metadata .pytest_cache;\ - find . -regex ".*/__pycache__" -exec rm -rf {} +;\ - find . -regex ".*\.egg-info" -exec rm -rf {} +;\ - install-test-requirements: - cd $(plugin) && pip install ".[test]" + cd $(plugin) && uv pip install ".[test]" install-pre-commit: pre-commit install --install-hooks @@ -79,12 +40,12 @@ sign-off: echo '--in-place "$$1"' >> .git/hooks/commit-msg chmod +x .git/hooks/commit-msg +## kedro-datasets specific + # kedro-datasets related only test-no-spark: dataset-doctests-no-spark cd kedro-datasets && pytest tests --no-cov --ignore tests/spark --ignore tests/databricks --numprocesses 4 --dist loadfile -test-no-spark-sequential: dataset-doctests-no-spark - cd kedro-datasets && pytest tests --no-cov --ignore tests/spark --ignore tests/databricks # kedro-datasets/snowflake tests skipped from default scope test-snowflake-only: @@ -93,3 +54,26 @@ test-snowflake-only: check-datasets-docs: cd kedro-datasets && python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck + +# Run test_tensorflow_model_dataset separately, because these tests are flaky when run as part of the full test-suite +dataset-tests: dataset-doctests + cd kedro-datasets && pytest tests --cov-config pyproject.toml --numprocesses 4 --dist loadfile --ignore tests/tensorflow + cd kedro-datasets && pytest tests/tensorflow/test_tensorflow_model_dataset.py --no-cov + +extra_pytest_args-no-spark=--ignore kedro_datasets/databricks --ignore kedro_datasets/spark +extra_pytest_args= +dataset-doctest%: + if [ "${*}" != 's-no-spark' ] && [ "${*}" != 's' ]; then \ + echo "make: *** No rule to make target \`${@}\`. Stop."; \ + exit 2; \ + fi; \ + \ + # The ignored datasets below require complicated setup with cloud/database clients which is overkill for the doctest examples. + cd kedro-datasets && pytest kedro_datasets --doctest-modules --doctest-continue-on-failure --no-cov \ + --ignore kedro_datasets/pandas/gbq_dataset.py \ + --ignore kedro_datasets/partitions/partitioned_dataset.py \ + --ignore kedro_datasets/redis/redis_dataset.py \ + --ignore kedro_datasets/snowflake/snowpark_dataset.py \ + --ignore kedro_datasets/spark/spark_hive_dataset.py \ + --ignore kedro_datasets/spark/spark_jdbc_dataset.py \ + $(extra_pytest_arg${*})