From 25397670e0c3b6c3e21ad5dab21b91c2b0632b1a Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Thu, 19 Sep 2024 14:53:23 +0100 Subject: [PATCH 1/7] Updated lint section Signed-off-by: Elena Khaustova --- docs/source/development/linting.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/development/linting.md b/docs/source/development/linting.md index 61989cdf85..f47bad7b6e 100644 --- a/docs/source/development/linting.md +++ b/docs/source/development/linting.md @@ -18,17 +18,17 @@ There are a variety of Python tools available to use with your Kedro projects. T type. ### Install the tools -Install `ruff` by adding the following lines to your project's `requirements.txt` -file: -```text -ruff # Used for linting, formatting and sorting module imports +Install `ruff` by adding the following section to the `pyproject.toml` file located in the project root: +```toml +[project.optional-dependencies] +dev = ["ruff"] ``` To install all the project-specific dependencies, including the linting tools, navigate to the root directory of the project and run: ```bash -pip install -r requirements.txt +pip install ."[dev]" ``` Alternatively, you can individually install the linting tools using the following shell commands: From 0533b4167df843e10e85a3867d1ff78573dc376d Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Thu, 19 Sep 2024 15:16:13 +0100 Subject: [PATCH 2/7] Updated automated testing docs Signed-off-by: Elena Khaustova --- docs/source/development/automated_testing.md | 29 +++++++++++++++----- docs/source/development/linting.md | 4 +-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/docs/source/development/automated_testing.md b/docs/source/development/automated_testing.md index ed3efe3287..c4ca9a6538 100644 --- a/docs/source/development/automated_testing.md +++ b/docs/source/development/automated_testing.md @@ -19,21 +19,36 @@ There are many testing frameworks available for Python. One of the most popular Let's look at how you can start working with `pytest` in your Kedro project. -### Prerequisite: Install your Kedro project +### Install test requirements +Before getting started with test requirements, it is important to ensure you have installed your project locally. This allows you to test different parts of your project by importing them into your test files. + + +To install your project including all the project-specific dependencies and test requirements: +1. Add the following section to the `pyproject.toml` file located in the project root: +```toml +[project.optional-dependencies] +dev = [ + "pytest-cov", + "pytest-mock", + "pytest", +] +``` + +2. Navigate to the root directory of the project and run: +```bash +pip install ."[dev]" +``` -Before getting started with `pytest`, it is important to ensure you have installed your project locally. This allows you to test different parts of your project by importing them into your test files. +Alternatively, you can individually install test requirements as you would install other packages with `pip`, making sure you have installed your project locally and your [project's virtual environment is active](../get_started/install.md#create-a-virtual-environment-for-your-kedro-project). -To install your project, navigate to your project root and run the following command: +1. To install your project, navigate to your project root and run the following command: ```bash pip install -e . ``` - >**NOTE**: The option `-e` installs an editable version of your project, allowing you to make changes to the project files without needing to re-install them each time. -### Install `pytest` - -Install `pytest` as you would install other packages with `pip`, making sure your [project's virtual environment is active](../get_started/install.md#create-a-virtual-environment-for-your-kedro-project). +2. Install test requirements one by one: ```bash pip install pytest ``` diff --git a/docs/source/development/linting.md b/docs/source/development/linting.md index f47bad7b6e..fbc0b0147c 100644 --- a/docs/source/development/linting.md +++ b/docs/source/development/linting.md @@ -18,13 +18,13 @@ There are a variety of Python tools available to use with your Kedro projects. T type. ### Install the tools -Install `ruff` by adding the following section to the `pyproject.toml` file located in the project root: +To install `ruff` add the following section to the `pyproject.toml` file located in the project root: ```toml [project.optional-dependencies] dev = ["ruff"] ``` -To install all the project-specific dependencies, including the linting tools, navigate to the root directory of the +Then to install your project including all the project-specific dependencies and the linting tools, navigate to the root directory of the project and run: ```bash From f290e4441cedc175e231c7e4aad858469844ae1e Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Thu, 19 Sep 2024 23:27:41 +0100 Subject: [PATCH 3/7] Moved development requirements for the template Signed-off-by: Elena Khaustova --- .../project/{{ cookiecutter.repo_name }}/pyproject.toml | 6 ++++++ .../project/{{ cookiecutter.repo_name }}/requirements.txt | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml b/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml index f22a91242f..b2ab54c3bb 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml @@ -24,6 +24,12 @@ docs = [ "Jinja2<3.2.0", "myst-parser>=1.0,<2.1" ] +dev = [ + "pytest-cov~=3.0", + "pytest-mock>=1.7.1, <2.0", + "pytest~=7.2", + "ruff~=0.1.8" +] [tool.setuptools.dynamic] diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt b/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt index 9301f4e3f3..1be43016fb 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,7 +2,3 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version }} -pytest-cov~=3.0 -pytest-mock>=1.7.1, <2.0 -pytest~=7.2 -ruff~=0.1.8 From c729abf10f27b072268425e954bae2b139b7baef Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Thu, 19 Sep 2024 23:30:25 +0100 Subject: [PATCH 4/7] Updated release notes Signed-off-by: Elena Khaustova --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index f2f24f9a77..295d9d8e18 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,6 +1,7 @@ # Upcoming Release ## Major features and improvements +* Moved development requirements from `requirements.txt` to the dedicated section in `pyproject.toml` for project template. * Implemented `Protocol` abstraction for the current `DataCatalog` and adding new catalog implementations. * Refactored `kedro run` and `kedro catalog` commands. * Moved pattern resolution logic from `DataCatalog` to a separate component - `CatalogConfigResolver`. Updated `DataCatalog` to use `CatalogConfigResolver` internally. From e5a8b35cf487db22116c98db7ae5b3df8897d512 Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Fri, 20 Sep 2024 14:44:31 +0100 Subject: [PATCH 5/7] Updated _assert_requirements_ok starters test Signed-off-by: Elena Khaustova --- tests/framework/cli/test_starters.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/framework/cli/test_starters.py b/tests/framework/cli/test_starters.py index 32f618d68f..7f2641da10 100644 --- a/tests/framework/cli/test_starters.py +++ b/tests/framework/cli/test_starters.py @@ -147,17 +147,11 @@ def _assert_requirements_ok( assert "Congratulations!" in result.output assert f"has been created in the directory \n{root_path}" in result.output - requirements_file_path = root_path / "requirements.txt" pyproject_file_path = root_path / "pyproject.toml" tools_list = _parse_tools_input(tools) if "1" in tools_list: - with open(requirements_file_path) as requirements_file: - requirements = requirements_file.read() - - assert "ruff" in requirements - pyproject_config = toml.load(pyproject_file_path) expected = { "tool": { @@ -171,15 +165,11 @@ def _assert_requirements_ok( } } assert expected["tool"]["ruff"] == pyproject_config["tool"]["ruff"] + assert ( + "ruff~=0.1.8" in pyproject_config["project"]["optional-dependencies"]["dev"] + ) if "2" in tools_list: - with open(requirements_file_path) as requirements_file: - requirements = requirements_file.read() - - assert "pytest-cov~=3.0" in requirements - assert "pytest-mock>=1.7.1, <2.0" in requirements - assert "pytest~=7.2" in requirements - pyproject_config = toml.load(pyproject_file_path) expected = { "pytest": { @@ -198,6 +188,18 @@ def _assert_requirements_ok( assert expected["pytest"] == pyproject_config["tool"]["pytest"] assert expected["coverage"] == pyproject_config["tool"]["coverage"] + assert ( + "pytest-cov~=3.0" + in pyproject_config["project"]["optional-dependencies"]["dev"] + ) + assert ( + "pytest-mock>=1.7.1, <2.0" + in pyproject_config["project"]["optional-dependencies"]["dev"] + ) + assert ( + "pytest~=7.2" in pyproject_config["project"]["optional-dependencies"]["dev"] + ) + if "4" in tools_list: pyproject_config = toml.load(pyproject_file_path) expected = { From fc6cd8dbe7ac7a53fcd325b480814d5f487adb6b Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Fri, 20 Sep 2024 16:34:26 +0100 Subject: [PATCH 6/7] Test fix Signed-off-by: Elena Khaustova --- kedro/templates/project/hooks/utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kedro/templates/project/hooks/utils.py b/kedro/templates/project/hooks/utils.py index 947c234f19..c112d72bbb 100644 --- a/kedro/templates/project/hooks/utils.py +++ b/kedro/templates/project/hooks/utils.py @@ -20,7 +20,9 @@ ] # Configuration key for documentation dependencies -docs_pyproject_requirements = ["project.optional-dependencies"] # For pyproject.toml +docs_pyproject_requirements = ["project.optional-dependencies.docs"] # For pyproject.toml +# Configuration key for linting and testing dependencies +dev_pyproject_requirements = ["project.optional-dependencies.dev"] # For pyproject.toml # Requirements for example pipelines example_pipeline_requirements = "seaborn~=0.12.1\nscikit-learn~=1.0\n" @@ -191,12 +193,14 @@ def setup_template_tools( python_package_name (str): The name of the python package. example_pipeline (str): 'True' if example pipeline was selected """ + + if "Linting" not in selected_tools_list and "Testing" not in selected_tools_list: + _remove_from_toml(pyproject_file_path, dev_pyproject_requirements) + if "Linting" not in selected_tools_list: - _remove_from_file(requirements_file_path, lint_requirements) _remove_from_toml(pyproject_file_path, lint_pyproject_requirements) if "Testing" not in selected_tools_list: - _remove_from_file(requirements_file_path, test_requirements) _remove_from_toml(pyproject_file_path, test_pyproject_requirements) _remove_dir(current_dir / "tests") From 44a5f35bb4368ff01a4c06a7d64332f3fb69e163 Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Mon, 23 Sep 2024 10:32:03 +0100 Subject: [PATCH 7/7] Update starter template for e2e tests Signed-off-by: Elena Khaustova --- features/load_node.feature | 3 ++- features/steps/cli_steps.py | 10 ++++++++++ .../pyproject.toml | 20 ++++++++++++------- .../requirements.txt | 4 ---- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/features/load_node.feature b/features/load_node.feature index fbc5a65a07..e745378e22 100644 --- a/features/load_node.feature +++ b/features/load_node.feature @@ -5,5 +5,6 @@ Feature: load_node in new project And I have run a non-interactive kedro new with starter "default" Scenario: Execute ipython load_node magic - When I execute the load_node magic command + When I install project and its dev dependencies + And I execute the load_node magic command Then the logs should show that load_node executed successfully diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index 7ee2c153d8..62cda23001 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -755,3 +755,13 @@ def exec_magic_command(context): def change_dir(context, dir): """Execute Kedro target.""" util.chdir(dir) + + +@when("I install project and its dev dependencies") +def pip_install_project_and_dev_dependencies(context): + """Install project and its development dependencies using pip.""" + _ = run( + [context.pip, "install", ".[dev]"], + env=context.env, + cwd=str(context.root_project_dir), + ) diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/pyproject.toml b/features/steps/test_starter/{{ cookiecutter.repo_name }}/pyproject.toml index 462dd26eee..eb7cb5f113 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/pyproject.toml @@ -12,15 +12,21 @@ dynamic = ["dependencies", "version"] [project.optional-dependencies] docs = [ - "docutils<0.18.0", - "sphinx~=3.4.3", - "sphinx_rtd_theme==0.5.1", + "docutils<0.21", + "sphinx>=5.3,<7.3", + "sphinx_rtd_theme==2.0.0", "nbsphinx==0.8.1", - "sphinx-autodoc-typehints==1.11.1", - "sphinx_copybutton==0.3.1", + "sphinx-autodoc-typehints==1.20.2", + "sphinx_copybutton==0.5.2", "ipykernel>=5.3, <7.0", - "Jinja2<3.1.0", - "myst-parser~=0.17.2", + "Jinja2<3.2.0", + "myst-parser>=1.0,<2.1" +] +dev = [ + "pytest-cov~=3.0", + "pytest-mock>=1.7.1, <2.0", + "pytest~=7.2", + "ruff~=0.1.8" ] [tool.setuptools.dynamic] diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt b/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt index 8da5d60851..b07568d9da 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt @@ -1,10 +1,6 @@ -ruff==0.1.8 ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version}} kedro-datasets[pandas-csvdataset]; python_version >= "3.9" kedro-datasets[pandas.CSVDataset]<2.0.0; python_version < '3.9' -pytest-cov~=3.0 -pytest-mock>=1.7.1, <2.0 -pytest~=7.2