From 4fa311b113704759cf8d34789ea9881b33f859ae Mon Sep 17 00:00:00 2001
From: Yu-Sheng Li
Date: Tue, 8 Oct 2024 17:01:56 +0800
Subject: [PATCH] Remove support for Python 3.8 (#4212)
* Remove support for Python 3.8
Signed-off-by: kevin1kevin1k
Co-authored-by: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com>
---
.github/workflows/all-checks.yml | 6 +++---
.github/workflows/docs-only-checks.yml | 2 +-
README.md | 2 +-
RELEASE.md | 2 ++
docs/source/conf.py | 2 +-
docs/source/deployment/aws_step_functions.md | 2 +-
.../databricks_ide_development_workflow.md | 2 +-
...tabricks_notebooks_development_workflow.md | 2 +-
docs/source/get_started/install.md | 6 +++---
docs/source/index.rst | 4 ++--
docs/source/introduction/index.md | 2 +-
.../add_kedro_to_a_notebook.ipynb | 6 ++----
docs/source/tutorial/tutorial_template.md | 1 -
features/environment.py | 8 +------
.../requirements.txt | 3 +--
features/steps/util.py | 3 ++-
kedro/config/omegaconf_config.py | 4 ++--
kedro/framework/cli/cli.py | 5 ++++-
kedro/framework/cli/micropkg.py | 4 +++-
kedro/framework/cli/utils.py | 8 +++++--
kedro/framework/hooks/manager.py | 3 ++-
kedro/framework/session/session.py | 4 +++-
kedro/io/catalog_config_resolver.py | 4 ++--
kedro/io/core.py | 21 +++++++------------
kedro/ipython/__init__.py | 5 ++++-
kedro/pipeline/modular_pipeline.py | 12 ++++++-----
kedro/pipeline/node.py | 5 ++++-
kedro/pipeline/pipeline.py | 8 ++++---
kedro/pipeline/transcoding.py | 4 +---
kedro/runner/parallel_runner.py | 4 +++-
kedro/runner/runner.py | 7 +++++--
kedro/templates/project/hooks/utils.py | 6 +++---
pyproject.toml | 11 ++++------
tests/framework/cli/pipeline/test_pipeline.py | 10 ++++-----
tests/framework/session/test_session.py | 4 ++--
tests/io/test_data_catalog.py | 18 ----------------
tests/io/test_kedro_data_catalog.py | 18 ----------------
tests/runner/test_resume_logic.py | 4 ++--
38 files changed, 97 insertions(+), 125 deletions(-)
diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml
index 2dfb971e3d..afd9caced0 100644
--- a/.github/workflows/all-checks.yml
+++ b/.github/workflows/all-checks.yml
@@ -26,7 +26,7 @@ jobs:
strategy:
matrix:
os: [ windows-latest, ubuntu-latest ]
- python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+ python-version: [ "3.9", "3.10", "3.11", "3.12" ]
uses: ./.github/workflows/unit-tests.yml
with:
os: ${{ matrix.os }}
@@ -36,7 +36,7 @@ jobs:
strategy:
matrix:
os: [ windows-latest, ubuntu-latest ]
- python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+ python-version: [ "3.9", "3.10", "3.11", "3.12" ]
uses: ./.github/workflows/e2e-tests.yml
with:
os: ${{ matrix.os }}
@@ -59,7 +59,7 @@ jobs:
strategy:
matrix:
os: [ windows-latest, ubuntu-latest ]
- python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+ python-version: [ "3.9", "3.10", "3.11", "3.12" ]
uses: ./.github/workflows/pip-compile.yml
with:
os: ${{ matrix.os }}
diff --git a/.github/workflows/docs-only-checks.yml b/.github/workflows/docs-only-checks.yml
index d0cca88abd..edd95de234 100644
--- a/.github/workflows/docs-only-checks.yml
+++ b/.github/workflows/docs-only-checks.yml
@@ -21,7 +21,7 @@ jobs:
strategy:
matrix:
os: [ ubuntu-latest ]
- python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+ python-version: [ "3.9", "3.10", "3.11", "3.12" ]
uses: ./.github/workflows/lint.yml
with:
os: ${{ matrix.os }}
diff --git a/README.md b/README.md
index 5cc0bda930..5e82fcebc2 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@
-[![Python version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro/)
+[![Python version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro/)
[![PyPI version](https://badge.fury.io/py/kedro.svg)](https://pypi.org/project/kedro/)
[![Conda version](https://img.shields.io/conda/vn/conda-forge/kedro.svg)](https://anaconda.org/conda-forge/kedro)
[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/kedro-org/kedro/blob/main/LICENSE.md)
diff --git a/RELEASE.md b/RELEASE.md
index 61560acf87..da3423fab0 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,6 +1,7 @@
# Upcoming Release
## Major features and improvements
+* Dropped Python 3.8 support.
* Implemented `KedroDataCatalog` repeating `DataCatalog` functionality with a few API enhancements:
* Removed `_FrozenDatasets` and access datasets as properties;
* Added get dataset by name feature;
@@ -32,6 +33,7 @@ e `_get_config_credentials()`
* [Manezki](https://github.com/Manezki)
* [MigQ2](https://github.com/MigQ2)
* [Felix Scherz](https://github.com/felixscherz)
+* [Yu-Sheng Li](https://github.com/kevin1kevin1k)
# Release 0.19.8
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 50b719f117..a883f76bd6 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -70,7 +70,7 @@
intersphinx_mapping = {
"kedro-viz": ("https://docs.kedro.org/projects/kedro-viz/en/v6.6.1/", None),
"kedro-datasets": ("https://docs.kedro.org/projects/kedro-datasets/en/kedro-datasets-2.0.0/", None),
- "cpython": ("https://docs.python.org/3.8/", None),
+ "cpython": ("https://docs.python.org/3.9/", None),
"ipython": ("https://ipython.readthedocs.io/en/8.21.0/", None),
"mlflow": ("https://www.mlflow.org/docs/2.12.1/", None),
"kedro-mlflow": ("https://kedro-mlflow.readthedocs.io/en/0.12.2/", None),
diff --git a/docs/source/deployment/aws_step_functions.md b/docs/source/deployment/aws_step_functions.md
index 8a3d70da53..9802eb542b 100644
--- a/docs/source/deployment/aws_step_functions.md
+++ b/docs/source/deployment/aws_step_functions.md
@@ -156,7 +156,7 @@ This file acts as the handler for each Lambda function in our pipeline, receives
```Dockerfile
# Define global args
ARG FUNCTION_DIR="/home/app/"
-ARG RUNTIME_VERSION="3.8"
+ARG RUNTIME_VERSION="3.9"
# Stage 1 - bundle base image + runtime
# Grab a fresh copy of the image and install GCC
diff --git a/docs/source/deployment/databricks/databricks_ide_development_workflow.md b/docs/source/deployment/databricks/databricks_ide_development_workflow.md
index f85799272d..7146ec7927 100644
--- a/docs/source/deployment/databricks/databricks_ide_development_workflow.md
+++ b/docs/source/deployment/databricks/databricks_ide_development_workflow.md
@@ -31,7 +31,7 @@ The main steps in this tutorial are as follows:
- An active [Databricks deployment](https://docs.databricks.com/getting-started/index.html).
- A [Databricks cluster](https://docs.databricks.com/clusters/configure.html) configured with a recent version (>= 11.3 is recommended) of the Databricks runtime.
-- [Conda installed](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) on your local machine in order to create a virtual environment with a specific version of Python (>= 3.8 is required). If you have Python >= 3.8 installed, you can use other software to create a virtual environment.
+- [Conda installed](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) on your local machine in order to create a virtual environment with a specific version of Python (>= 3.9 is required). If you have Python >= 3.9 installed, you can use other software to create a virtual environment.
## Set up your project
diff --git a/docs/source/deployment/databricks/databricks_notebooks_development_workflow.md b/docs/source/deployment/databricks/databricks_notebooks_development_workflow.md
index 8fc4b76f4c..8a0616708b 100644
--- a/docs/source/deployment/databricks/databricks_notebooks_development_workflow.md
+++ b/docs/source/deployment/databricks/databricks_notebooks_development_workflow.md
@@ -19,7 +19,7 @@ This tutorial introduces a Kedro project development workflow using only the Dat
- An active [Databricks deployment](https://docs.databricks.com/getting-started/index.html).
- A [Databricks cluster](https://docs.databricks.com/clusters/configure.html) configured with a recent version (>= 11.3 is recommended) of the Databricks runtime.
-- Python >= 3.8 installed.
+- Python >= 3.9 installed.
- Git installed.
- A [GitHub](https://github.com/) account.
- A Python environment management system installed, [venv](https://docs.python.org/3/library/venv.html), [virtualenv](https://virtualenv.pypa.io/en/latest/) or [Conda](https://docs.conda.io/en/latest/) are popular choices.
diff --git a/docs/source/get_started/install.md b/docs/source/get_started/install.md
index d61d084a89..fe75decda2 100644
--- a/docs/source/get_started/install.md
+++ b/docs/source/get_started/install.md
@@ -1,7 +1,7 @@
# Set up Kedro
## Installation prerequisites
-* **Python**: Kedro supports macOS, Linux, and Windows and is built for Python 3.8+. You'll select a version of Python when you create a virtual environment for your Kedro project.
+* **Python**: Kedro supports macOS, Linux, and Windows and is built for Python 3.9+. You'll select a version of Python when you create a virtual environment for your Kedro project.
* **Virtual environment**: You should create a new virtual environment for *each* new Kedro project you work on to isolate its Python dependencies from those of other projects.
@@ -55,7 +55,7 @@ deactivate
conda create --name kedro-environment python=3.10 -y
```
-The example below uses Python 3.10, and creates a virtual environment called `kedro-environment`. You can opt for a different version of Python (any version >= 3.8 and <3.12) for your project, and you can name it anything you choose.
+The example below uses Python 3.10, and creates a virtual environment called `kedro-environment`. You can opt for a different version of Python (any version >= 3.9 and <3.12) for your project, and you can name it anything you choose.
The `conda` virtual environment is not dependent on your current working directory and can be activated from any directory:
@@ -136,7 +136,7 @@ When migrating an existing project to a newer Kedro version, make sure you also
## Summary
* Kedro can be used on Windows, macOS or Linux.
-* Installation prerequisites include a virtual environment manager like `conda`, Python 3.8+, and `git`.
+* Installation prerequisites include a virtual environment manager like `conda`, Python 3.9+, and `git`.
* You should install Kedro using `pip install kedro`.
If you encounter any problems as you set up Kedro, ask for help on Kedro's [Slack organisation](https://slack.kedro.org) or review the [searchable archive of Slack discussions](https://linen-slack.kedro.org/).
diff --git a/docs/source/index.rst b/docs/source/index.rst
index ce8d85a9a1..ecbdbbe381 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -23,9 +23,9 @@ Welcome to Kedro's award-winning documentation!
:target: https://opensource.org/license/apache2-0-php/
:alt: License is Apache 2.0
-.. image:: https://img.shields.io/badge/3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-blue.svg
+.. image:: https://img.shields.io/badge/3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg
:target: https://pypi.org/project/kedro/
- :alt: Python version 3.8, 3.9, 3.10, 3.11
+ :alt: Python version 3.9, 3.10, 3.11, 3.12
.. image:: https://badge.fury.io/py/kedro.svg
:target: https://pypi.org/project/kedro/
diff --git a/docs/source/introduction/index.md b/docs/source/introduction/index.md
index 38dab844a3..d9de994933 100644
--- a/docs/source/introduction/index.md
+++ b/docs/source/introduction/index.md
@@ -18,6 +18,6 @@ Use the left-hand table of contents to explore the documentation available for m
```{note}
We have designed the preliminary documentation and the [spaceflights tutorial](../tutorial/spaceflights_tutorial.md) for anyone new to Kedro. The more knowledge of Python you have, the easier you will find the learning curve.
-There are many excellent online resources for learning Python; you should choose those that reference Python 3, as Kedro is built for Python 3.8+. There are curated lists of online resources, such as the [official Python programming language website](https://www.python.org/) and this list of [free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python).
+There are many excellent online resources for learning Python; you should choose those that reference Python 3, as Kedro is built for Python 3.9+. There are curated lists of online resources, such as the [official Python programming language website](https://www.python.org/) and this list of [free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python).
```
diff --git a/docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb b/docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb
index 3c036386cc..1e91ef0d29 100644
--- a/docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb
+++ b/docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb
@@ -683,7 +683,6 @@
"####################\n",
"# Data processing #\n",
"####################\n",
- "from typing import Dict, Tuple\n",
"\n",
"import pandas as pd\n",
"from sklearn.linear_model import LinearRegression\n",
@@ -736,7 +735,7 @@
"##################################\n",
"\n",
"\n",
- "def split_data(data: pd.DataFrame, parameters: Dict) -> Tuple:\n",
+ "def split_data(data: pd.DataFrame, parameters: dict) -> tuple:\n",
" X = data[parameters[\"features\"]]\n",
" y = data[\"price\"]\n",
" X_train, X_test, y_train, y_test = train_test_split(\n",
@@ -796,7 +795,6 @@
"outputs": [],
"source": [
"# Kedro setup for data management and configuration\n",
- "from typing import Dict, Tuple\n",
"\n",
"import pandas as pd\n",
"from sklearn.linear_model import LinearRegression\n",
@@ -873,7 +871,7 @@
"##################################\n",
"\n",
"\n",
- "def split_data(data: pd.DataFrame, parameters: Dict) -> Tuple:\n",
+ "def split_data(data: pd.DataFrame, parameters: dict) -> tuple:\n",
" X = data[parameters[\"features\"]]\n",
" y = data[\"price\"]\n",
" X_train, X_test, y_train, y_test = train_test_split(\n",
diff --git a/docs/source/tutorial/tutorial_template.md b/docs/source/tutorial/tutorial_template.md
index d8462f1b20..1586487707 100644
--- a/docs/source/tutorial/tutorial_template.md
+++ b/docs/source/tutorial/tutorial_template.md
@@ -32,7 +32,6 @@ The spaceflights project dependencies are stored in `requirements.txt`(you may f
```text
# code quality packages
-ipython>=7.31.1, <8.0; python_version < '3.8'
ipython~=8.10; python_version >= '3.8'
ruff==0.1.8
diff --git a/features/environment.py b/features/environment.py
index 75bd183ed8..f1d45df926 100644
--- a/features/environment.py
+++ b/features/environment.py
@@ -5,7 +5,6 @@
import os
import shutil
import subprocess
-import sys
import tempfile
import venv
from pathlib import Path
@@ -131,11 +130,6 @@ def _install_project_requirements(context):
.splitlines()
)
install_reqs = [req for req in install_reqs if "{" not in req and "#" not in req]
- # For Python versions 3.9 and above we use the new dataset dependency format introduced in `kedro-datasets` 3.0.0
- if sys.version_info.minor > MINOR_PYTHON_38_VERSION:
- install_reqs.append("kedro-datasets[pandas-csvdataset]")
- # For Python 3.8 we use the older `kedro-datasets` dependency format
- else:
- install_reqs.append("kedro-datasets[pandas.CSVDataset]")
+ install_reqs.append("kedro-datasets[pandas-csvdataset]")
call([context.pip, "install", *install_reqs], env=context.env)
return context
diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt b/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt
index b07568d9da..014df14d12 100644
--- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt
+++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt
@@ -2,5 +2,4 @@ ipython>=8.10
jupyterlab>=3.0
notebook
kedro~={{ cookiecutter.kedro_version}}
-kedro-datasets[pandas-csvdataset]; python_version >= "3.9"
-kedro-datasets[pandas.CSVDataset]<2.0.0; python_version < '3.9'
+kedro-datasets[pandas-csvdataset]
diff --git a/features/steps/util.py b/features/steps/util.py
index 437b3f6f5e..588e4530e1 100644
--- a/features/steps/util.py
+++ b/features/steps/util.py
@@ -6,9 +6,10 @@
import re
from contextlib import contextmanager
from time import sleep, time
-from typing import TYPE_CHECKING, Any, Callable, Iterator
+from typing import TYPE_CHECKING, Any, Callable
if TYPE_CHECKING:
+ from collections.abc import Iterator
from pathlib import Path
diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py
index c4850159a1..8d82ebf360 100644
--- a/kedro/config/omegaconf_config.py
+++ b/kedro/config/omegaconf_config.py
@@ -8,10 +8,10 @@
import logging
import mimetypes
import typing
-from collections.abc import KeysView
+from collections.abc import Iterable, KeysView
from enum import Enum, auto
from pathlib import Path
-from typing import Any, Callable, Iterable
+from typing import Any, Callable
import fsspec
from omegaconf import DictConfig, OmegaConf
diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py
index b22fa70d00..f5917e1b87 100644
--- a/kedro/framework/cli/cli.py
+++ b/kedro/framework/cli/cli.py
@@ -10,10 +10,13 @@
import traceback
from collections import defaultdict
from pathlib import Path
-from typing import Any, Sequence
+from typing import TYPE_CHECKING, Any
import click
+if TYPE_CHECKING:
+ from collections.abc import Sequence
+
from kedro import __version__ as version
from kedro.framework.cli import BRIGHT_BLACK, ORANGE
from kedro.framework.cli.hooks import get_cli_hook_manager
diff --git a/kedro/framework/cli/micropkg.py b/kedro/framework/cli/micropkg.py
index d2733ff712..a80efd172a 100644
--- a/kedro/framework/cli/micropkg.py
+++ b/kedro/framework/cli/micropkg.py
@@ -12,7 +12,8 @@
import toml
from importlib import import_module
from pathlib import Path
-from typing import Any, Iterable, Iterator, TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
+
import click
from omegaconf import OmegaConf
@@ -42,6 +43,7 @@
if TYPE_CHECKING:
from kedro.framework.startup import ProjectMetadata
from importlib_metadata import PackageMetadata
+ from collections.abc import Iterable, Iterator
_PYPROJECT_TOML_TEMPLATE = """
[build-system]
diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py
index e165c15579..1b50408cc5 100644
--- a/kedro/framework/cli/utils.py
+++ b/kedro/framework/cli/utils.py
@@ -15,10 +15,14 @@
import typing
import warnings
from collections import defaultdict
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from collections.abc import Iterable, Sequence
from importlib import import_module
from itertools import chain
from pathlib import Path
-from typing import IO, Any, Callable, Iterable, Sequence
+from typing import IO, Any, Callable
import click
import importlib_metadata
@@ -55,7 +59,7 @@ def call(cmd: list[str], **kwargs: Any) -> None: # pragma: no cover
Raises:
click.exceptions.Exit: If `subprocess.run` returns non-zero code.
"""
- click.echo(" ".join(shlex.quote(c) for c in cmd))
+ click.echo(shlex.join(cmd))
code = subprocess.run(cmd, **kwargs).returncode # noqa: PLW1510, S603
if code:
raise click.exceptions.Exit(code=code)
diff --git a/kedro/framework/hooks/manager.py b/kedro/framework/hooks/manager.py
index 5cbbcf9f27..ceec064246 100644
--- a/kedro/framework/hooks/manager.py
+++ b/kedro/framework/hooks/manager.py
@@ -3,7 +3,8 @@
"""
import logging
-from typing import Any, Iterable
+from collections.abc import Iterable
+from typing import Any
from pluggy import PluginManager
diff --git a/kedro/framework/session/session.py b/kedro/framework/session/session.py
index caa3553954..ec0dc9bf4d 100644
--- a/kedro/framework/session/session.py
+++ b/kedro/framework/session/session.py
@@ -11,7 +11,7 @@
import traceback
from copy import deepcopy
from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterable
+from typing import TYPE_CHECKING, Any
import click
@@ -28,6 +28,8 @@
from kedro.utils import _find_kedro_project
if TYPE_CHECKING:
+ from collections.abc import Iterable
+
from kedro.config import AbstractConfigLoader
from kedro.framework.context import KedroContext
from kedro.framework.session.store import BaseSessionStore
diff --git a/kedro/io/catalog_config_resolver.py b/kedro/io/catalog_config_resolver.py
index aeafea81e3..dc55d18b3c 100644
--- a/kedro/io/catalog_config_resolver.py
+++ b/kedro/io/catalog_config_resolver.py
@@ -7,13 +7,13 @@
import copy
import logging
import re
-from typing import Any, Dict
+from typing import Any
from parse import parse
from kedro.io.core import DatasetError
-Patterns = Dict[str, Dict[str, Any]]
+Patterns = dict[str, dict[str, Any]]
CREDENTIALS_KEY = "credentials"
diff --git a/kedro/io/core.py b/kedro/io/core.py
index 036babc829..53b660835c 100644
--- a/kedro/io/core.py
+++ b/kedro/io/core.py
@@ -505,20 +505,13 @@ def parse_dataset_definition(
class_obj = tmp
break
else:
- hint = ""
- if "DataSet" in dataset_type:
- hint = ( # pragma: no cover # To remove when we drop support for python 3.8
- "Hint: If you are trying to use a dataset from `kedro-datasets`>=2.0.0, "
- "make sure that the dataset name uses the `Dataset` spelling instead of `DataSet`."
- )
- else:
- hint = (
- "Hint: If you are trying to use a dataset from `kedro-datasets`, "
- "make sure that the package is installed in your current environment. "
- "You can do so by running `pip install kedro-datasets` or "
- "`pip install kedro-datasets[]` to install `kedro-datasets` along with "
- "related dependencies for the specific dataset group."
- )
+ hint = (
+ "Hint: If you are trying to use a dataset from `kedro-datasets`, "
+ "make sure that the package is installed in your current environment. "
+ "You can do so by running `pip install kedro-datasets` or "
+ "`pip install kedro-datasets[]` to install `kedro-datasets` along with "
+ "related dependencies for the specific dataset group."
+ )
raise DatasetError(
f"Class '{dataset_type}' not found, is this a typo?" f"\n{hint}"
)
diff --git a/kedro/ipython/__init__.py b/kedro/ipython/__init__.py
index 7cdbf92138..0e479e8a68 100644
--- a/kedro/ipython/__init__.py
+++ b/kedro/ipython/__init__.py
@@ -14,7 +14,10 @@
import warnings
from pathlib import Path
from types import MappingProxyType
-from typing import Any, Callable, OrderedDict
+from typing import TYPE_CHECKING, Any, Callable
+
+if TYPE_CHECKING:
+ from collections import OrderedDict
from IPython.core.getipython import get_ipython
from IPython.core.magic import needs_local_scope, register_line_magic
diff --git a/kedro/pipeline/modular_pipeline.py b/kedro/pipeline/modular_pipeline.py
index 172a5f9593..c6372f37a4 100644
--- a/kedro/pipeline/modular_pipeline.py
+++ b/kedro/pipeline/modular_pipeline.py
@@ -4,13 +4,15 @@
import copy
import difflib
-from typing import TYPE_CHECKING, AbstractSet, Iterable
+from typing import TYPE_CHECKING
from kedro.pipeline.pipeline import Pipeline
from .transcoding import TRANSCODING_SEPARATOR, _strip_transcoding, _transcode_split
if TYPE_CHECKING:
+ from collections.abc import Iterable, Set
+
from kedro.pipeline.node import Node
@@ -35,7 +37,7 @@ def _is_parameter(name: str) -> bool:
def _validate_inputs_outputs(
- inputs: AbstractSet[str], outputs: AbstractSet[str], pipe: Pipeline
+ inputs: Set[str], outputs: Set[str], pipe: Pipeline
) -> None:
"""Safeguards to ensure that:
- parameters are not specified under inputs
@@ -64,9 +66,9 @@ def _validate_inputs_outputs(
def _validate_datasets_exist(
- inputs: AbstractSet[str],
- outputs: AbstractSet[str],
- parameters: AbstractSet[str],
+ inputs: Set[str],
+ outputs: Set[str],
+ parameters: Set[str],
pipe: Pipeline,
) -> None:
"""Validate that inputs, parameters and outputs map correctly onto the provided nodes."""
diff --git a/kedro/pipeline/node.py b/kedro/pipeline/node.py
index 09a83410aa..b382bee8cf 100644
--- a/kedro/pipeline/node.py
+++ b/kedro/pipeline/node.py
@@ -9,13 +9,16 @@
import logging
import re
from collections import Counter
-from typing import Any, Callable, Iterable
+from typing import TYPE_CHECKING, Any, Callable
from warnings import warn
from more_itertools import spy, unzip
from .transcoding import _strip_transcoding
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
class Node:
"""``Node`` is an auxiliary class facilitating the operations required to
diff --git a/kedro/pipeline/pipeline.py b/kedro/pipeline/pipeline.py
index 7810a98049..ab7365a154 100644
--- a/kedro/pipeline/pipeline.py
+++ b/kedro/pipeline/pipeline.py
@@ -8,16 +8,18 @@
import json
from collections import Counter, defaultdict
-from itertools import chain
-from typing import Any, Iterable
-
from graphlib import CycleError, TopologicalSorter
+from itertools import chain
+from typing import TYPE_CHECKING, Any
import kedro
from kedro.pipeline.node import Node, _to_list
from .transcoding import _strip_transcoding
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
def __getattr__(name: str) -> Any:
if name == "TRANSCODING_SEPARATOR":
diff --git a/kedro/pipeline/transcoding.py b/kedro/pipeline/transcoding.py
index 71f0dac342..eae9a10cf7 100644
--- a/kedro/pipeline/transcoding.py
+++ b/kedro/pipeline/transcoding.py
@@ -1,9 +1,7 @@
-from typing import Tuple
-
TRANSCODING_SEPARATOR = "@"
-def _transcode_split(element: str) -> Tuple[str, str]:
+def _transcode_split(element: str) -> tuple[str, str]:
"""Split the name by the transcoding separator.
If the transcoding part is missing, empty string will be put in.
diff --git a/kedro/runner/parallel_runner.py b/kedro/runner/parallel_runner.py
index d09601ff7e..7626bf8679 100644
--- a/kedro/runner/parallel_runner.py
+++ b/kedro/runner/parallel_runner.py
@@ -13,7 +13,7 @@
from multiprocessing.managers import BaseProxy, SyncManager
from multiprocessing.reduction import ForkingPickler
from pickle import PicklingError
-from typing import TYPE_CHECKING, Any, Iterable
+from typing import TYPE_CHECKING, Any
from kedro.framework.hooks.manager import (
_create_hook_manager,
@@ -30,6 +30,8 @@
from kedro.runner.runner import AbstractRunner, run_node
if TYPE_CHECKING:
+ from collections.abc import Iterable
+
from pluggy import PluginManager
from kedro.pipeline import Pipeline
diff --git a/kedro/runner/runner.py b/kedro/runner/runner.py
index f3a0889909..f6716e070f 100644
--- a/kedro/runner/runner.py
+++ b/kedro/runner/runner.py
@@ -9,6 +9,7 @@
import logging
from abc import ABC, abstractmethod
from collections import deque
+from collections.abc import Iterator
from concurrent.futures import (
ALL_COMPLETED,
Future,
@@ -16,7 +17,7 @@
as_completed,
wait,
)
-from typing import TYPE_CHECKING, Any, Collection, Iterable, Iterator
+from typing import TYPE_CHECKING, Any
from more_itertools import interleave
@@ -25,6 +26,8 @@
from kedro.pipeline import Pipeline
if TYPE_CHECKING:
+ from collections.abc import Collection, Iterable
+
from pluggy import PluginManager
from kedro.pipeline.node import Node
@@ -369,7 +372,7 @@ def _find_initial_node_group(pipeline: Pipeline, nodes: Iterable[Node]) -> list[
A list of initial ``Node``s to run given inputs (in topological order).
"""
- node_names = set(n.name for n in nodes)
+ node_names = {n.name for n in nodes}
if len(node_names) == 0:
return []
sub_pipeline = pipeline.only_nodes(*node_names)
diff --git a/kedro/templates/project/hooks/utils.py b/kedro/templates/project/hooks/utils.py
index c112d72bbb..595884248e 100644
--- a/kedro/templates/project/hooks/utils.py
+++ b/kedro/templates/project/hooks/utils.py
@@ -36,7 +36,7 @@ def _remove_from_file(file_path: Path, content_to_remove: str) -> None:
file_path (Path): The path of the file from which to remove content.
content_to_remove (str): The content to be removed from the file.
"""
- with open(file_path, "r") as file:
+ with open(file_path) as file:
lines = file.readlines()
# Split the content to remove into lines and remove trailing whitespaces/newlines
@@ -86,7 +86,7 @@ def _remove_from_toml(file_path: Path, sections_to_remove: list) -> None:
sections_to_remove (list): A list of section keys to remove from the TOML file.
"""
# Load the TOML file
- with open(file_path, "r") as file:
+ with open(file_path) as file:
data = toml.load(file)
# Remove the specified sections
@@ -162,7 +162,7 @@ def _remove_extras_from_kedro_datasets(file_path: Path) -> None:
Args:
file_path (Path): The path of the requirements file.
"""
- with open(file_path, "r") as file:
+ with open(file_path) as file:
lines = file.readlines()
for i, line in enumerate(lines):
diff --git a/pyproject.toml b/pyproject.toml
index d9ebbfd70b..23b60e9a61 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,7 @@ authors = [
{name = "Kedro"}
]
description = "Kedro helps you build production-ready data and analytics pipelines"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
dependencies = [
"attrs>=21.3",
"build>=0.7.0",
@@ -33,7 +33,6 @@ dependencies = [
"rope>=0.21,<2.0", # subject to LGPLv3 license
"toml>=0.10.0",
"typing_extensions>=4.0",
- "graphlib_backport>=1.0.0; python_version < '3.9'",
]
keywords = [
"pipelines",
@@ -45,10 +44,10 @@ keywords = [
license = {text = "Apache Software License (Apache 2.0)"}
classifiers = [
"Development Status :: 4 - Beta",
- "Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
]
dynamic = ["readme", "version"]
@@ -58,13 +57,11 @@ test = [
"coverage[toml]",
"import-linter==2.0",
"ipylab>=1.0.0",
- "ipython>=7.31.1, <8.0; python_version < '3.8'",
- "ipython~=8.10; python_version >= '3.8'",
+ "ipython~=8.10",
"jupyterlab_server>=2.11.1",
"jupyterlab>=3,<5",
"jupyter~=1.0",
- "kedro-datasets; python_version >= '3.9'",
- "kedro-datasets<2.0.0; python_version < '3.9'",
+ "kedro-datasets",
"mypy~=1.0",
"pandas~=2.0",
"pluggy>=1.0",
diff --git a/tests/framework/cli/pipeline/test_pipeline.py b/tests/framework/cli/pipeline/test_pipeline.py
index df01d685a6..099790221e 100644
--- a/tests/framework/cli/pipeline/test_pipeline.py
+++ b/tests/framework/cli/pipeline/test_pipeline.py
@@ -127,11 +127,11 @@ def test_create_pipeline_template_command_line_override(
assert not (pipelines_dir / PIPELINE_NAME).exists()
# Rename the local template dir to something else so we know the command line flag is taking precedence
- try:
- # Can skip if already there but copytree has a dirs_exist_ok flag in >python 3.8 only
- shutil.copytree(fake_local_template_dir, fake_repo_path / "local_templates")
- except FileExistsError:
- pass
+ shutil.copytree(
+ fake_local_template_dir,
+ fake_repo_path / "local_templates",
+ dirs_exist_ok=True,
+ )
cmd = ["pipeline", "create", PIPELINE_NAME]
cmd += ["-t", str(fake_repo_path / "local_templates/pipeline")]
diff --git a/tests/framework/session/test_session.py b/tests/framework/session/test_session.py
index 086d581045..1c67824d8d 100644
--- a/tests/framework/session/test_session.py
+++ b/tests/framework/session/test_session.py
@@ -5,7 +5,7 @@
import textwrap
from collections.abc import Mapping
from pathlib import Path
-from typing import Any, Type
+from typing import Any
from unittest.mock import create_autospec
import pytest
@@ -50,7 +50,7 @@ class BadConfigLoader:
NEW_TYPING = sys.version_info[:3] >= (3, 7, 0) # PEP 560
-def create_attrs_autospec(spec: Type, spec_set: bool = True) -> Any:
+def create_attrs_autospec(spec: type, spec_set: bool = True) -> Any:
"""Creates a mock of an attr class (creates mocks recursively on all attributes).
https://github.com/python-attrs/attrs/issues/462#issuecomment-1134656377
diff --git a/tests/io/test_data_catalog.py b/tests/io/test_data_catalog.py
index 39c0f77307..54cbdf340d 100644
--- a/tests/io/test_data_catalog.py
+++ b/tests/io/test_data_catalog.py
@@ -1,6 +1,5 @@
import logging
import re
-import sys
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
@@ -463,23 +462,6 @@ def test_config_missing_class(self, correct_config):
with pytest.raises(DatasetError, match=re.escape(pattern)):
DataCatalog.from_config(**correct_config)
- @pytest.mark.skipif(
- sys.version_info < (3, 9),
- reason="for python 3.8 kedro-datasets version 1.8 is used which has the old spelling",
- )
- def test_config_incorrect_spelling(self, correct_config):
- """Check hint if the type uses the old DataSet spelling"""
- correct_config["catalog"]["boats"]["type"] = "pandas.CSVDataSet"
-
- pattern = (
- "An exception occurred when parsing config for dataset 'boats':\n"
- "Class 'pandas.CSVDataSet' not found, is this a typo?"
- "\nHint: If you are trying to use a dataset from `kedro-datasets`>=2.0.0,"
- " make sure that the dataset name uses the `Dataset` spelling instead of `DataSet`."
- )
- with pytest.raises(DatasetError, match=re.escape(pattern)):
- DataCatalog.from_config(**correct_config)
-
def test_config_invalid_dataset(self, correct_config):
"""Check the error if the type points to invalid class"""
correct_config["catalog"]["boats"]["type"] = "DataCatalog"
diff --git a/tests/io/test_kedro_data_catalog.py b/tests/io/test_kedro_data_catalog.py
index b98e8fae83..5e0c463e7d 100644
--- a/tests/io/test_kedro_data_catalog.py
+++ b/tests/io/test_kedro_data_catalog.py
@@ -1,6 +1,5 @@
import logging
import re
-import sys
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
@@ -347,23 +346,6 @@ def test_config_missing_class(self, correct_config):
with pytest.raises(DatasetError, match=re.escape(pattern)):
KedroDataCatalog.from_config(**correct_config)
- @pytest.mark.skipif(
- sys.version_info < (3, 9),
- reason="for python 3.8 kedro-datasets version 1.8 is used which has the old spelling",
- )
- def test_config_incorrect_spelling(self, correct_config):
- """Check hint if the type uses the old DataSet spelling"""
- correct_config["catalog"]["boats"]["type"] = "pandas.CSVDataSet"
-
- pattern = (
- "An exception occurred when parsing config for dataset 'boats':\n"
- "Class 'pandas.CSVDataSet' not found, is this a typo?"
- "\nHint: If you are trying to use a dataset from `kedro-datasets`>=2.0.0,"
- " make sure that the dataset name uses the `Dataset` spelling instead of `DataSet`."
- )
- with pytest.raises(DatasetError, match=re.escape(pattern)):
- KedroDataCatalog.from_config(**correct_config)
-
def test_config_invalid_dataset(self, correct_config):
"""Check the error if the type points to invalid class"""
correct_config["catalog"]["boats"]["type"] = "KedroDataCatalog"
diff --git a/tests/runner/test_resume_logic.py b/tests/runner/test_resume_logic.py
index bd1f8e8acb..c733c504c5 100644
--- a/tests/runner/test_resume_logic.py
+++ b/tests/runner/test_resume_logic.py
@@ -153,6 +153,6 @@ def test_suggestion_consistency(
test_pipeline, remaining_nodes, persistent_dataset_catalog
)
- assert set(n.name for n in required_nodes) == set(
+ assert {n.name for n in required_nodes} == {
n.name for n in test_pipeline.from_nodes(*resume_node_names).nodes
- )
+ }