From c35443ad88d264e35a78263ecd798960543b93c8 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 16 Aug 2023 14:37:28 +0100 Subject: [PATCH 1/4] Enable overriding keys in config with runtime params Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 45 +++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 4d2ace59d4..c4284f545a 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -7,7 +7,7 @@ import logging import mimetypes from pathlib import Path -from typing import Any, Callable, Iterable +from typing import Any, Callable, Iterable, Mapping import fsspec from omegaconf import OmegaConf @@ -240,7 +240,8 @@ def load_and_merge_dir_config( # noqa: too-many-arguments """ # noqa: too-many-locals - + print(conf_path) + print(key) if not self._fs.isdir(Path(conf_path).as_posix()): raise MissingConfigException( f"Given configuration path either does not exist " @@ -282,22 +283,40 @@ def load_and_merge_dir_config( # noqa: too-many-arguments } aggregate_config = config_per_file.values() self._check_duplicates(seen_file_to_keys) - + print(self.runtime_params) if not aggregate_config: return {} - if key == "parameters": - # Merge with runtime parameters only for "parameters" - return OmegaConf.to_container( + merged_conf = OmegaConf.to_container( OmegaConf.merge(*aggregate_config, self.runtime_params), resolve=True ) - return { - k: v - for k, v in OmegaConf.to_container( - OmegaConf.merge(*aggregate_config), resolve=True - ).items() - if not k.startswith("_") - } + else: + merged_conf = { + k: v + for k, v in OmegaConf.to_container( + self.update_nested_dict( + OmegaConf.merge(*aggregate_config), self.runtime_params + ), + resolve=True, + ).items() + if not k.startswith("_") + } + return merged_conf + + @classmethod + def update_nested_dict(cls, dict1, dict2): + for key, value in dict1.items(): + if ( + isinstance(value, Mapping) + and key in dict2 + and isinstance(dict2[key], Mapping) + ): + cls.update_nested_dict( + value, dict2[key] + ) # Recurse into nested dictionaries + elif key in dict2: + dict1[key] = dict2[key] + return dict1 def _is_valid_config_path(self, path): """Check if given path is a file path and file type is yaml or json.""" From 989fa8c3d7b5061f32abf8427ef798838e6b8b74 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 16 Aug 2023 14:39:30 +0100 Subject: [PATCH 2/4] Cleanup Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index c4284f545a..43d5139cfd 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -240,8 +240,6 @@ def load_and_merge_dir_config( # noqa: too-many-arguments """ # noqa: too-many-locals - print(conf_path) - print(key) if not self._fs.isdir(Path(conf_path).as_posix()): raise MissingConfigException( f"Given configuration path either does not exist " @@ -283,7 +281,6 @@ def load_and_merge_dir_config( # noqa: too-many-arguments } aggregate_config = config_per_file.values() self._check_duplicates(seen_file_to_keys) - print(self.runtime_params) if not aggregate_config: return {} if key == "parameters": From 392f878faedd2974a79c46b949a9921bca507492 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 16 Aug 2023 14:42:38 +0100 Subject: [PATCH 3/4] Cleanup Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 43d5139cfd..2df86d246b 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -240,6 +240,7 @@ def load_and_merge_dir_config( # noqa: too-many-arguments """ # noqa: too-many-locals + if not self._fs.isdir(Path(conf_path).as_posix()): raise MissingConfigException( f"Given configuration path either does not exist " @@ -281,8 +282,10 @@ def load_and_merge_dir_config( # noqa: too-many-arguments } aggregate_config = config_per_file.values() self._check_duplicates(seen_file_to_keys) + if not aggregate_config: return {} + if key == "parameters": merged_conf = OmegaConf.to_container( OmegaConf.merge(*aggregate_config, self.runtime_params), resolve=True @@ -291,7 +294,7 @@ def load_and_merge_dir_config( # noqa: too-many-arguments merged_conf = { k: v for k, v in OmegaConf.to_container( - self.update_nested_dict( + self._update_nested_dict( OmegaConf.merge(*aggregate_config), self.runtime_params ), resolve=True, @@ -301,7 +304,7 @@ def load_and_merge_dir_config( # noqa: too-many-arguments return merged_conf @classmethod - def update_nested_dict(cls, dict1, dict2): + def _update_nested_dict(cls, dict1, dict2): for key, value in dict1.items(): if ( isinstance(value, Mapping) From 5cb6bce767efaba1d43907ef0f374d32a2a9da5e Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 16 Aug 2023 17:59:15 +0100 Subject: [PATCH 4/4] Remove update_nested_dict Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 2df86d246b..37e92f211c 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -7,7 +7,7 @@ import logging import mimetypes from pathlib import Path -from typing import Any, Callable, Iterable, Mapping +from typing import Any, Callable, Iterable import fsspec from omegaconf import OmegaConf @@ -291,33 +291,17 @@ def load_and_merge_dir_config( # noqa: too-many-arguments OmegaConf.merge(*aggregate_config, self.runtime_params), resolve=True ) else: + original_keys = OmegaConf.merge(*aggregate_config).keys() merged_conf = { k: v for k, v in OmegaConf.to_container( - self._update_nested_dict( - OmegaConf.merge(*aggregate_config), self.runtime_params - ), + OmegaConf.merge(*aggregate_config, self.runtime_params), resolve=True, ).items() - if not k.startswith("_") + if not k.startswith("_") and k in original_keys } return merged_conf - @classmethod - def _update_nested_dict(cls, dict1, dict2): - for key, value in dict1.items(): - if ( - isinstance(value, Mapping) - and key in dict2 - and isinstance(dict2[key], Mapping) - ): - cls.update_nested_dict( - value, dict2[key] - ) # Recurse into nested dictionaries - elif key in dict2: - dict1[key] = dict2[key] - return dict1 - def _is_valid_config_path(self, path): """Check if given path is a file path and file type is yaml or json.""" posix_path = path.as_posix()