Merge branch 'main' into 0.19.8

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_stages: [commit, manual]
 
 repos:
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.1.8
+      rev: v0.6.1
       hooks:
         - id: ruff
           name: "ruff on kedro/, tests/ and docs/"
@@ -17,7 +17,7 @@ repos:
           exclude: "^features/steps/test_starter|^kedro/templates/"
 
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v2.2.3
+      rev: v4.6.0
       hooks:
           - id: trailing-whitespace
           - id: end-of-file-fixer

docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb

@@ -84,6 +84,7 @@
    "outputs": [],
    "source": [
     "# Model training\n",
+    "from sklearn.linear_model import LinearRegression\n",
     "from sklearn.model_selection import train_test_split\n",
     "\n",
     "X = model_input_table[\n",
@@ -102,8 +103,6 @@
     "\n",
     "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=3)\n",
     "\n",
-    "from sklearn.linear_model import LinearRegression\n",
-    "\n",
     "model = LinearRegression()\n",
     "model.fit(X_train, y_train)\n",
     "model.predict(X_test)"
@@ -163,10 +162,10 @@
    "source": [
     "# Using Kedro's DataCatalog\n",
     "\n",
-    "from kedro.io import DataCatalog\n",
-    "\n",
     "import yaml\n",
     "\n",
+    "from kedro.io import DataCatalog\n",
+    "\n",
     "# load the configuration file\n",
     "with open(\"catalog.yml\") as f:\n",
     "    conf_catalog = yaml.safe_load(f)\n",
@@ -311,11 +310,11 @@
    "outputs": [],
    "source": [
     "from sklearn.linear_model import LinearRegression\n",
+    "from sklearn.metrics import r2_score\n",
     "\n",
     "model = LinearRegression()\n",
     "model.fit(X_train, y_train)\n",
     "model.predict(X_test)\n",
-    "from sklearn.metrics import r2_score\n",
     "\n",
     "y_pred = model.predict(X_test)\n",
     "r2_score(y_test, y_pred)"
@@ -405,11 +404,11 @@
    "outputs": [],
    "source": [
     "from sklearn.linear_model import LinearRegression\n",
+    "from sklearn.metrics import r2_score\n",
     "\n",
     "model = LinearRegression()\n",
     "model.fit(X_train, y_train)\n",
     "model.predict(X_test)\n",
-    "from sklearn.metrics import r2_score\n",
     "\n",
     "y_pred = model.predict(X_test)\n",
     "r2_score(y_test, y_pred)"
@@ -483,11 +482,11 @@
    "outputs": [],
    "source": [
     "from sklearn.linear_model import LinearRegression\n",
+    "from sklearn.metrics import r2_score\n",
     "\n",
     "model = LinearRegression()\n",
     "model.fit(X_train, y_train)\n",
     "model.predict(X_test)\n",
-    "from sklearn.metrics import r2_score\n",
     "\n",
     "y_pred = model.predict(X_test)\n",
     "r2_score(y_test, y_pred)"
@@ -511,10 +510,10 @@
    "source": [
     "# Using Kedro's DataCatalog\n",
     "\n",
-    "from kedro.io import DataCatalog\n",
-    "\n",
     "import yaml\n",
     "\n",
+    "from kedro.io import DataCatalog\n",
+    "\n",
     "# load the configuration file\n",
     "with open(\"catalog.yml\") as f:\n",
     "    conf_catalog = yaml.safe_load(f)\n",
@@ -684,7 +683,12 @@
     "####################\n",
     "# Data processing  #\n",
     "####################\n",
+    "from typing import Dict, Tuple\n",
+    "\n",
     "import pandas as pd\n",
+    "from sklearn.linear_model import LinearRegression\n",
+    "from sklearn.metrics import r2_score\n",
+    "from sklearn.model_selection import train_test_split\n",
     "\n",
     "\n",
     "def _is_true(x: pd.Series) -> pd.Series:\n",
@@ -731,11 +735,6 @@
     "# Model training and evaluation  #\n",
     "##################################\n",
     "\n",
-    "from typing import Dict, Tuple\n",
-    "from sklearn.linear_model import LinearRegression\n",
-    "from sklearn.metrics import r2_score\n",
-    "from sklearn.model_selection import train_test_split\n",
-    "\n",
     "\n",
     "def split_data(data: pd.DataFrame, parameters: Dict) -> Tuple:\n",
     "    X = data[parameters[\"features\"]]\n",
@@ -797,6 +796,13 @@
    "outputs": [],
    "source": [
     "# Kedro setup for data management and configuration\n",
+    "from typing import Dict, Tuple\n",
+    "\n",
+    "import pandas as pd\n",
+    "from sklearn.linear_model import LinearRegression\n",
+    "from sklearn.metrics import r2_score\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
     "from kedro.config import OmegaConfigLoader\n",
     "from kedro.io import DataCatalog\n",
     "\n",
@@ -820,7 +826,6 @@
     "####################\n",
     "# Data processing  #\n",
     "####################\n",
-    "import pandas as pd\n",
     "\n",
     "\n",
     "def _is_true(x: pd.Series) -> pd.Series:\n",
@@ -867,11 +872,6 @@
     "# Model training and evaluation  #\n",
     "##################################\n",
     "\n",
-    "from typing import Dict, Tuple\n",
-    "from sklearn.linear_model import LinearRegression\n",
-    "from sklearn.metrics import r2_score\n",
-    "from sklearn.model_selection import train_test_split\n",
-    "\n",
     "\n",
     "def split_data(data: pd.DataFrame, parameters: Dict) -> Tuple:\n",
     "    X = data[parameters[\"features\"]]\n",

features/environment.py

@@ -1,4 +1,5 @@
 """Behave environment setup commands."""
+
 from __future__ import annotations
 
 import os
@@ -96,9 +97,8 @@ def _create_tmp_dir() -> Path:
 
 def _setup_minimal_env(context):
     if os.environ.get("BEHAVE_LOCAL_ENV"):
-        output = subprocess.check_output(
-            ["which", "kedro"]  # noqa: S603, S607
-        )  # equivalent run "which kedro"
+        # equivalent run "which kedro"
+        output = subprocess.check_output(["which", "kedro"])  # noqa: S603, S607
         output = output.strip().decode("utf8")
         kedro_install_venv_dir = Path(output).parent.parent
         context.kedro_install_venv_dir = kedro_install_venv_dir

features/steps/sh_run.py

@@ -27,6 +27,7 @@ def run(
     Example:
     ::
         "ls"
+
         "ls -la"
         "chmod 754 local/file"
 

features/steps/test_plugin/plugin.py

@@ -1,4 +1,5 @@
 """Dummy plugin with simple hook implementations."""
+
 import logging
 from pathlib import Path
 

features/steps/util.py

@@ -1,5 +1,5 @@
-"""Common functions for e2e testing.
-"""
+"""Common functions for e2e testing."""
+
 from __future__ import annotations
 
 import os

kedro/config/abstract_config.py

@@ -1,6 +1,7 @@
 """This module provides ``kedro.abstract_config`` with the baseline
 class model for a `ConfigLoader` implementation.
 """
+
 from __future__ import annotations
 
 from collections import UserDict

kedro/config/omegaconf_config.py

@@ -1,6 +1,7 @@
 """This module provides ``kedro.config`` with the functionality to load one
 or more configuration files of yaml or json type from specified paths through OmegaConf.
 """
+
 from __future__ import annotations
 
 import io
@@ -269,7 +270,7 @@ def keys(self) -> KeysView:
         return KeysView(self.config_patterns)
 
     @typing.no_type_check
-    def load_and_merge_dir_config(  # noqa: PLR0913
+    def load_and_merge_dir_config(
         self,
         conf_path: str,
         patterns: Iterable[str],

kedro/framework/__init__.py

@@ -1 +1 @@
-"""``kedro.framework`` provides Kedro's framework components """
+"""``kedro.framework`` provides Kedro's framework components"""

kedro/framework/cli/__init__.py

@@ -1,5 +1,4 @@
-"""``kedro.framework.cli`` implements commands available from Kedro's CLI.
-"""
+"""``kedro.framework.cli`` implements commands available from Kedro's CLI."""
 
 # The constant need to be defined first otherwise it causes circular depdencies
 ORANGE = (255, 175, 0)

kedro/framework/cli/catalog.py

@@ -1,4 +1,5 @@
 """A collection of CLI commands for working with Kedro catalog."""
+
 from __future__ import annotations
 
 import copy

kedro/framework/cli/cli.py

@@ -2,6 +2,7 @@
 
 This module implements commands available from the kedro CLI.
 """
+
 from __future__ import annotations
 
 import importlib

kedro/framework/cli/hooks/__init__.py

@@ -1,4 +1,5 @@
 """``kedro.framework.cli.hooks`` provides primitives to use hooks to extend KedroCLI's behaviour"""
+
 from .manager import CLIHooksManager, get_cli_hook_manager
 from .markers import cli_hook_impl
 

kedro/framework/cli/hooks/manager.py

@@ -1,4 +1,5 @@
 """This module defines a dedicated hook manager for hooks that extends Kedro CLI behaviour."""
+
 import logging
 
 from pluggy import PluginManager

kedro/framework/cli/hooks/markers.py

@@ -2,6 +2,7 @@
 For more information, please see
 [Pluggy's documentation](https://pluggy.readthedocs.io/en/stable/#marking-hooks).
 """
+
 import pluggy
 
 CLI_HOOK_NAMESPACE = "kedro_cli"

kedro/framework/cli/hooks/specs.py

@@ -2,6 +2,7 @@
 For more information about these specifications, please visit
 [Pluggy's documentation](https://pluggy.readthedocs.io/en/stable/#specs)
 """
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING

kedro/framework/cli/jupyter.py

@@ -1,6 +1,7 @@
 """A collection of helper functions to integrate with Jupyter/IPython
 and CLI commands for working with Kedro catalog.
 """
+
 from __future__ import annotations
 
 import json

kedro/framework/cli/micropkg.py

@@ -1,4 +1,5 @@
 """A collection of CLI commands for working with Kedro micro-packages."""
+
 # ruff: noqa: I001
 from __future__ import annotations
 
@@ -410,7 +411,7 @@ def safe_extract(tar: tarfile.TarFile, path: Path) -> None:
         if not _is_within_directory(path, member_path):
             raise Exception("Failed to safely extract tar file.")
         safe_members.append(member)
-    tar.extractall(path, members=safe_members)  # nosec B202
+    tar.extractall(path, members=safe_members)  # noqa S202
     # The nosec is still required because bandit still flags this.
     # Related issue: https://github.com/PyCQA/bandit/issues/1038
 

kedro/framework/cli/pipeline.py

@@ -1,4 +1,5 @@
 """A collection of CLI commands for working with Kedro pipelines."""
+
 from __future__ import annotations
 
 import re

kedro/framework/cli/project.py

@@ -1,4 +1,5 @@
 """A collection of CLI commands for working with Kedro project."""
+
 from __future__ import annotations
 
 import os

kedro/framework/cli/registry.py

@@ -1,4 +1,5 @@
 """A collection of CLI commands for working with registered Kedro pipelines."""
+
 from typing import Any
 
 import click

kedro/framework/cli/starters.py

@@ -3,6 +3,7 @@
 This module implements commands available from the kedro CLI for creating
 projects.
 """
+
 from __future__ import annotations
 
 import logging

kedro/framework/cli/utils.py

@@ -1,4 +1,5 @@
 """Utilities for use with click."""
+
 from __future__ import annotations
 
 import difflib
@@ -442,7 +443,7 @@ def _config_file_callback(ctx: click.Context, param: Any, value: Any) -> Any:
 
     if value:
         config = OmegaConf.to_container(OmegaConf.load(value))[section]
-        for key, value in config.items():
+        for key, value in config.items():  # noqa: PLR1704
             _validate_config_file(key)
         ctx.default_map.update(config)
 

kedro/framework/context/context.py

@@ -1,4 +1,5 @@
 """This module provides context for Kedro project."""
+
 from __future__ import annotations
 
 import logging

kedro/framework/hooks/__init__.py

@@ -1,4 +1,5 @@
 """``kedro.framework.hooks`` provides primitives to use hooks to extend KedroContext's behaviour"""
+
 from .manager import _create_hook_manager
 from .markers import hook_impl
 

kedro/framework/hooks/manager.py

@@ -1,6 +1,7 @@
 """This module provides an utility function to retrieve the global hook_manager singleton
 in a Kedro's execution process.
 """
+
 import logging
 from typing import Any, Iterable
 

kedro/framework/hooks/specs.py

@@ -2,6 +2,7 @@
 For more information about these specifications, please visit
 [Pluggy's documentation](https://pluggy.readthedocs.io/en/stable/#specs)
 """
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
@@ -49,7 +50,7 @@ class NodeSpecs:
     """Namespace that defines all specifications for a node's lifecycle hooks."""
 
     @hook_spec
-    def before_node_run(  # noqa: PLR0913
+    def before_node_run(
         self,
         node: Node,
         catalog: DataCatalog,

kedro/framework/project/__init__.py

@@ -1,5 +1,6 @@
 """``kedro.framework.project`` module provides utility to
 configure a Kedro project and access its settings."""
+
 from __future__ import annotations
 
 import importlib

kedro/framework/session/__init__.py

@@ -1,6 +1,7 @@
 """``kedro.framework.session`` provides access to KedroSession responsible for
 project lifecycle.
 """
+
 from .session import KedroSession
 
 __all__ = ["KedroSession"]