From da5a750d9789ff60ebe08d0595fcb37ffc657679 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Tue, 16 Jul 2024 10:43:13 +0200 Subject: [PATCH 01/12] change output component type Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/data_handling.py | 57 ++++++++++-- src/power_grid_model/core/power_grid_model.py | 10 +- src/power_grid_model/typing.py | 14 +++ tests/unit/test_data_handling.py | 93 +++++++++++++++++++ 4 files changed, 160 insertions(+), 14 deletions(-) create mode 100644 src/power_grid_model/typing.py create mode 100644 tests/unit/test_data_handling.py diff --git a/src/power_grid_model/core/data_handling.py b/src/power_grid_model/core/data_handling.py index 0dcf33b3b..6a0acfb66 100644 --- a/src/power_grid_model/core/data_handling.py +++ b/src/power_grid_model/core/data_handling.py @@ -8,7 +8,7 @@ from enum import Enum -from typing import Dict, List, Mapping, Set, Tuple, Union +from typing import Dict, Mapping, Tuple, Union import numpy as np @@ -16,6 +16,7 @@ from power_grid_model.core.power_grid_dataset import CConstDataset, CMutableDataset from power_grid_model.core.power_grid_meta import initialize_array, power_grid_meta_data from power_grid_model.enum import CalculationType +from power_grid_model.typing import OutputComponentNamesType, _OutputComponentTypeDict class OutputType(Enum): @@ -101,7 +102,7 @@ def prepare_output_view(output_data: Mapping[ComponentType, np.ndarray], output_ def create_output_data( - output_component_types: Union[Set[ComponentType], List[ComponentType]], + output_component_types: OutputComponentNamesType, output_type: OutputType, all_component_count: Dict[ComponentType, int], is_batch: bool, @@ -135,10 +136,14 @@ def create_output_data( Dimension 0: each batch Dimension 1: the result of each element for this component type """ - # raise error if some specified components are unknown - unknown_components = [x for x in output_component_types if x not in power_grid_meta_data[output_type.value]] - if unknown_components: - raise KeyError(f"You have specified some unknown component types: {unknown_components}") + + # limit all component count to user specified component types in output + if output_component_types is None: + output_component_types = {k: None for k in all_component_count} + elif isinstance(output_component_types, (list, set)): + output_component_types = {k: None for k in output_component_types} + + validate_output_component_types(output_type, output_component_types) all_component_count = {k: v for k, v in all_component_count.items() if k in output_component_types} @@ -151,6 +156,44 @@ def create_output_data( shape: Union[Tuple[int], Tuple[int, int]] = (batch_size, count) else: shape = (count,) - result_dict[name] = initialize_array(output_type.value, name, shape=shape, empty=True) + attributes = output_component_types[name] + if attributes is None: + result_dict[name] = initialize_array(output_type.value, name, shape=shape, empty=True) + elif attributes in [[], set()]: + result_dict[name] = {} + else: + raise NotImplementedError( + "Columnar data types are not implemented yet." + "output_component_types must be provided with a list or set" + ) return result_dict + + +def validate_output_component_types(output_type: OutputType, dict_output_types: _OutputComponentTypeDict): + """Checks dict_output_types for any invalid component names and attribute names + + Args: + output_type (OutputType): the type of output that the user will see (as per the calculation options) + dict_output_types (Dict[ComponentType, Optional[str]]): output_component_types converted to dictionary + + Raises: + KeyError: with "unknown component" for any unknown components + KeyError: with "unknown attributes" for any unknown attributes for a known component + """ + # raise error if some specified components are unknown + output_meta = power_grid_meta_data[output_type.value] + unknown_components = [x for x in dict_output_types if x not in output_meta] + if unknown_components: + raise KeyError(f"You have specified some unknown component types: {unknown_components}") + + unknown_attributes = {} + for comp_name, attrs in dict_output_types.items(): + if attrs is None: + continue + diff = set(attrs).difference(output_meta[comp_name].dtype.names) + if diff != set(): + unknown_attributes[comp_name] = diff + + if unknown_attributes: + raise KeyError(f"You have specified some unknown attributes: {unknown_attributes}") diff --git a/src/power_grid_model/core/power_grid_model.py b/src/power_grid_model/core/power_grid_model.py index fd41c5e54..505fe934c 100644 --- a/src/power_grid_model/core/power_grid_model.py +++ b/src/power_grid_model/core/power_grid_model.py @@ -30,6 +30,7 @@ TapChangingStrategy, _ExperimentalFeatures, ) +from power_grid_model.typing import OutputComponentNamesType class PowerGridModel: @@ -187,18 +188,13 @@ def include_type(component_type: ComponentType): # pylint: disable=too-many-arguments def _construct_output( self, - output_component_types: Optional[Union[Set[ComponentType], List[ComponentType]]], + output_component_types: OutputComponentNamesType, calculation_type: CalculationType, symmetric: bool, is_batch: bool, batch_size: int, ) -> Dict[ComponentType, np.ndarray]: all_component_count = self._get_output_component_count(calculation_type=calculation_type) - - # limit all component count to user specified component types in output - if output_component_types is None: - output_component_types = set(all_component_count.keys()) - return create_output_data( output_component_types=output_component_types, output_type=get_output_type(calculation_type=calculation_type, symmetric=symmetric), @@ -236,7 +232,7 @@ def _calculate_impl( calculation_type: CalculationType, symmetric: bool, update_data: Optional[Dataset], - output_component_types: Optional[Union[Set[ComponentType], List[ComponentType]]], + output_component_types: OutputComponentNamesType, options: Options, continue_on_batch_error: bool, decode_error: bool, diff --git a/src/power_grid_model/typing.py b/src/power_grid_model/typing.py new file mode 100644 index 000000000..d666af264 --- /dev/null +++ b/src/power_grid_model/typing.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + +""" +Type hints for PGM. This includes all miscellaneous type hints not under dataset or dataset_definitions categories +""" +from typing import Dict, List, Optional, Set, Union + +from power_grid_model.core.dataset_definitions import ComponentType + +_OutputComponentTypeDict = Dict[ComponentType, Optional[Union[Set[str], List[str]]]] + +OutputComponentNamesType = Optional[Union[Set[ComponentType], List[ComponentType], _OutputComponentTypeDict]] diff --git a/tests/unit/test_data_handling.py b/tests/unit/test_data_handling.py new file mode 100644 index 000000000..b097f8ffd --- /dev/null +++ b/tests/unit/test_data_handling.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Contributors to the Power Grid Model project +# +# SPDX-License-Identifier: MPL-2.0 + +import numpy as np +import pytest + +from power_grid_model.core.data_handling import OutputType, create_output_data +from power_grid_model.core.dataset_definitions import ComponentType as CT, DatasetType as DT +from power_grid_model.core.power_grid_meta import initialize_array + + +@pytest.mark.parametrize( + ("output_component_types", "is_batch", "expected"), + [ + ( + None, + False, + { + CT.node: initialize_array(DT.sym_output, CT.node, 4), + CT.sym_load: initialize_array(DT.sym_output, CT.sym_load, 3), + CT.source: initialize_array(DT.sym_output, CT.source, 1), + }, + ), + ( + [CT.node, CT.sym_load], + False, + { + CT.node: initialize_array(DT.sym_output, CT.node, 4), + CT.sym_load: initialize_array(DT.sym_output, CT.sym_load, 3), + }, + ), + ( + {CT.node, CT.sym_load}, + False, + { + CT.node: initialize_array(DT.sym_output, CT.node, 4), + CT.sym_load: initialize_array(DT.sym_output, CT.sym_load, 3), + }, + ), + ({CT.node: [], CT.sym_load: []}, True, {CT.node: dict(), CT.sym_load: dict()}), + pytest.param({CT.node: [], CT.sym_load: ["p"]}, True, {}, marks=pytest.mark.xfail), + pytest.param({CT.node: ["u"], CT.sym_load: ["p"]}, True, {}, marks=pytest.mark.xfail), + pytest.param({CT.node: None, CT.sym_load: ["p"]}, True, {}, marks=pytest.mark.xfail), + ], +) +def test_create_output_data(output_component_types, is_batch, expected): + # TODO use is_batch and shorten parameterization after columnar data implementation + all_component_count = {CT.node: 4, CT.sym_load: 3, CT.source: 1} + batch_size = 15 if is_batch else 1 + actual = create_output_data( + output_component_types=output_component_types, + output_type=OutputType.SYM_OUTPUT, + all_component_count=all_component_count, + is_batch=is_batch, + batch_size=batch_size, + ) + + assert actual.keys() == expected.keys() + for k in expected: + if isinstance(expected[k], np.ndarray): + # Row based + assert actual[k].dtype == expected[k].dtype + elif expected[k] == dict(): + # Empty atrtibutes + assert actual[k] == expected[k] + else: + # Columnar data + assert "data" in actual[k] and "indptr" in actual[k] + assert actual[k]["data"].dtype == expected[k]["data"].dtype + assert actual[k]["indptr"].dtype == expected[k]["indptr"].dtype + + +@pytest.mark.parametrize( + ("output_component_types", "match"), + [ + ({"abc": None, "def": None}, "unknown component"), + ({"abc": None, CT.sym_load: None}, "unknown component"), + ({"abc": ["xyz"], CT.sym_load: None}, "unknown component"), + ({CT.node: ["xyz"], CT.sym_load: None}, "unknown attributes"), + ({CT.node: ["xyz1"], CT.sym_load: ["xyz2"]}, "unknown attributes"), + ], +) +def test_create_output_data__errors(output_component_types, match): + all_component_count = {CT.node: 4, CT.sym_load: 3, CT.source: 1} + with pytest.raises(KeyError, match=match): + create_output_data( + output_component_types=output_component_types, + output_type=OutputType.SYM_OUTPUT, + all_component_count=all_component_count, + is_batch=False, + batch_size=15, + ) From f0dfd164f71a09ed43b676b84a1bf48c4c88e28b Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Thu, 18 Jul 2024 09:56:04 +0200 Subject: [PATCH 02/12] revise per coming steps Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/data_handling.py | 71 ++++++++++--------- src/power_grid_model/core/power_grid_model.py | 6 +- src/power_grid_model/typing.py | 8 +-- tests/unit/test_data_handling.py | 29 ++++---- 4 files changed, 60 insertions(+), 54 deletions(-) diff --git a/src/power_grid_model/core/data_handling.py b/src/power_grid_model/core/data_handling.py index 6a0acfb66..d8360e0e5 100644 --- a/src/power_grid_model/core/data_handling.py +++ b/src/power_grid_model/core/data_handling.py @@ -8,15 +8,16 @@ from enum import Enum -from typing import Dict, Mapping, Tuple, Union +from typing import Mapping, Tuple, Union import numpy as np from power_grid_model.core.dataset_definitions import ComponentType, DatasetType from power_grid_model.core.power_grid_dataset import CConstDataset, CMutableDataset from power_grid_model.core.power_grid_meta import initialize_array, power_grid_meta_data +from power_grid_model.data_types import Dataset from power_grid_model.enum import CalculationType -from power_grid_model.typing import OutputComponentNamesType, _OutputComponentTypeDict +from power_grid_model.typing import ComponentAttributeMapping, _ComponentAttributeMappingDict class OutputType(Enum): @@ -102,15 +103,14 @@ def prepare_output_view(output_data: Mapping[ComponentType, np.ndarray], output_ def create_output_data( - output_component_types: OutputComponentNamesType, + output_component_types: ComponentAttributeMapping, output_type: OutputType, - all_component_count: Dict[ComponentType, int], + all_component_count: dict[ComponentType, int], is_batch: bool, batch_size: int, -) -> Dict[ComponentType, np.ndarray]: +) -> Dataset: """ - Create the output data that the user can use. always returns batch type output data. - Use reduce_output_data to flatten to single scenario output if applicable. + Create the output dataset based on component and batch size from the model; and output attribtues requested by user. Args: output_component_types: @@ -124,28 +124,14 @@ def create_output_data( batch_size: the batch size - Raises: - KeyError: if some specified components are unknown. - Returns: - dictionary of results of all components - key: component type name to be updated in batch - value: - for single calculation: 1D numpy structured array for the results of this component type - for batch calculation: 2D numpy structured array for the results of this component type - Dimension 0: each batch - Dimension 1: the result of each element for this component type + Dataset: output dataset """ + processed_output_types = process_output_component_types( + output_type, output_component_types, list(all_component_count.keys()) + ) - # limit all component count to user specified component types in output - if output_component_types is None: - output_component_types = {k: None for k in all_component_count} - elif isinstance(output_component_types, (list, set)): - output_component_types = {k: None for k in output_component_types} - - validate_output_component_types(output_type, output_component_types) - - all_component_count = {k: v for k, v in all_component_count.items() if k in output_component_types} + all_component_count = {k: v for k, v in all_component_count.items() if k in processed_output_types} # create result dataset result_dict = {} @@ -156,7 +142,7 @@ def create_output_data( shape: Union[Tuple[int], Tuple[int, int]] = (batch_size, count) else: shape = (count,) - attributes = output_component_types[name] + attributes = processed_output_types[name] if attributes is None: result_dict[name] = initialize_array(output_type.value, name, shape=shape, empty=True) elif attributes in [[], set()]: @@ -170,25 +156,44 @@ def create_output_data( return result_dict -def validate_output_component_types(output_type: OutputType, dict_output_types: _OutputComponentTypeDict): - """Checks dict_output_types for any invalid component names and attribute names +def process_output_component_types( + output_type: OutputType, + output_component_types: ComponentAttributeMapping, + available_components: list[ComponentType], +) -> _ComponentAttributeMappingDict: + """Checks valid type for output_component_types. Also checks for any invalid component names and attribute names Args: output_type (OutputType): the type of output that the user will see (as per the calculation options) - dict_output_types (Dict[ComponentType, Optional[str]]): output_component_types converted to dictionary + output_component_types (OutputComponentNamesType): output_component_types provided by user + available_components (list[ComponentType]): all components available in model instance Raises: + ValueError: when the type for output_comoponent_types is incorrect KeyError: with "unknown component" for any unknown components KeyError: with "unknown attributes" for any unknown attributes for a known component + + Returns: + _OutputComponentTypeDict: processed output_component_types in a dictionary """ + # limit all component count to user specified component types in output and convert to a dict + if output_component_types is None: + output_component_types = {k: None for k in available_components} + elif isinstance(output_component_types, (list, set)): + output_component_types = {k: None for k in output_component_types} + elif not isinstance(output_component_types, dict) or not all( + attrs is None or isinstance(attrs, (set, list)) for attrs in output_component_types.values() + ): + raise ValueError(f"Invalid output_component_types provided: {output_component_types}") + # raise error if some specified components are unknown output_meta = power_grid_meta_data[output_type.value] - unknown_components = [x for x in dict_output_types if x not in output_meta] + unknown_components = [x for x in output_component_types if x not in output_meta] if unknown_components: raise KeyError(f"You have specified some unknown component types: {unknown_components}") unknown_attributes = {} - for comp_name, attrs in dict_output_types.items(): + for comp_name, attrs in output_component_types.items(): if attrs is None: continue diff = set(attrs).difference(output_meta[comp_name].dtype.names) @@ -197,3 +202,5 @@ def validate_output_component_types(output_type: OutputType, dict_output_types: if unknown_attributes: raise KeyError(f"You have specified some unknown attributes: {unknown_attributes}") + + return output_component_types diff --git a/src/power_grid_model/core/power_grid_model.py b/src/power_grid_model/core/power_grid_model.py index 505fe934c..1409e83a5 100644 --- a/src/power_grid_model/core/power_grid_model.py +++ b/src/power_grid_model/core/power_grid_model.py @@ -30,7 +30,7 @@ TapChangingStrategy, _ExperimentalFeatures, ) -from power_grid_model.typing import OutputComponentNamesType +from power_grid_model.typing import ComponentAttributeMapping class PowerGridModel: @@ -188,7 +188,7 @@ def include_type(component_type: ComponentType): # pylint: disable=too-many-arguments def _construct_output( self, - output_component_types: OutputComponentNamesType, + output_component_types: ComponentAttributeMapping, calculation_type: CalculationType, symmetric: bool, is_batch: bool, @@ -232,7 +232,7 @@ def _calculate_impl( calculation_type: CalculationType, symmetric: bool, update_data: Optional[Dataset], - output_component_types: OutputComponentNamesType, + output_component_types: ComponentAttributeMapping, options: Options, continue_on_batch_error: bool, decode_error: bool, diff --git a/src/power_grid_model/typing.py b/src/power_grid_model/typing.py index d666af264..7a20b3046 100644 --- a/src/power_grid_model/typing.py +++ b/src/power_grid_model/typing.py @@ -5,10 +5,10 @@ """ Type hints for PGM. This includes all miscellaneous type hints not under dataset or dataset_definitions categories """ -from typing import Dict, List, Optional, Set, Union - from power_grid_model.core.dataset_definitions import ComponentType -_OutputComponentTypeDict = Dict[ComponentType, Optional[Union[Set[str], List[str]]]] +_ComponentAttributeMappingDict = dict[ComponentType, set[str] | list[str] | None] -OutputComponentNamesType = Optional[Union[Set[ComponentType], List[ComponentType], _OutputComponentTypeDict]] +ComponentAttributeMapping = ( + set[ComponentType] | list[ComponentType] | None | dict[ComponentType, set[str] | list[str] | None] +) diff --git a/tests/unit/test_data_handling.py b/tests/unit/test_data_handling.py index b097f8ffd..134da766c 100644 --- a/tests/unit/test_data_handling.py +++ b/tests/unit/test_data_handling.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from power_grid_model.core.data_handling import OutputType, create_output_data +from power_grid_model.core.data_handling import OutputType, create_output_data, process_output_component_types from power_grid_model.core.dataset_definitions import ComponentType as CT, DatasetType as DT from power_grid_model.core.power_grid_meta import initialize_array @@ -72,22 +72,21 @@ def test_create_output_data(output_component_types, is_batch, expected): @pytest.mark.parametrize( - ("output_component_types", "match"), + ("output_component_types", "error", "match"), [ - ({"abc": None, "def": None}, "unknown component"), - ({"abc": None, CT.sym_load: None}, "unknown component"), - ({"abc": ["xyz"], CT.sym_load: None}, "unknown component"), - ({CT.node: ["xyz"], CT.sym_load: None}, "unknown attributes"), - ({CT.node: ["xyz1"], CT.sym_load: ["xyz2"]}, "unknown attributes"), + ({"abc": 3, "def": None}, ValueError, "Invalid output_component_types"), + ({"abc": None, "def": None}, KeyError, "unknown component"), + ({"abc": None, CT.sym_load: None}, KeyError, "unknown component"), + ({"abc": ["xyz"], CT.sym_load: None}, KeyError, "unknown component"), + ({CT.node: ["xyz"], CT.sym_load: None}, KeyError, "unknown attributes"), + ({CT.node: ["xyz1"], CT.sym_load: ["xyz2"]}, KeyError, "unknown attributes"), ], ) -def test_create_output_data__errors(output_component_types, match): - all_component_count = {CT.node: 4, CT.sym_load: 3, CT.source: 1} - with pytest.raises(KeyError, match=match): - create_output_data( - output_component_types=output_component_types, +def test_create_output_data__errors(output_component_types, error, match): + available_components = [CT.node, CT.sym_load, CT.source] + with pytest.raises(error, match=match): + process_output_component_types( output_type=OutputType.SYM_OUTPUT, - all_component_count=all_component_count, - is_batch=False, - batch_size=15, + output_component_types=output_component_types, + available_components=available_components, ) From 59593ff9b517d5ac670ef1e82cf28e19bc43c772 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Thu, 18 Jul 2024 14:18:12 +0200 Subject: [PATCH 03/12] change to dense output data Signed-off-by: Nitish Bharambe --- tests/unit/test_data_handling.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_data_handling.py b/tests/unit/test_data_handling.py index 134da766c..8daf8cbf6 100644 --- a/tests/unit/test_data_handling.py +++ b/tests/unit/test_data_handling.py @@ -57,18 +57,17 @@ def test_create_output_data(output_component_types, is_batch, expected): ) assert actual.keys() == expected.keys() - for k in expected: - if isinstance(expected[k], np.ndarray): + for comp in expected: + if isinstance(expected[comp], np.ndarray): # Row based - assert actual[k].dtype == expected[k].dtype - elif expected[k] == dict(): + assert actual[comp].dtype == expected[comp].dtype + elif expected[comp] == dict(): # Empty atrtibutes - assert actual[k] == expected[k] + assert actual[comp] == expected[comp] else: # Columnar data - assert "data" in actual[k] and "indptr" in actual[k] - assert actual[k]["data"].dtype == expected[k]["data"].dtype - assert actual[k]["indptr"].dtype == expected[k]["indptr"].dtype + assert actual[comp].keys() == expected[comp].keys() + assert all(actual[comp][attr].dtype == expected[comp][attr].dtype for attr in expected[comp]) @pytest.mark.parametrize( From 2d5c41f89a79e7ec4ce77276a76de3d1772e00e5 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 19 Jul 2024 10:37:23 +0200 Subject: [PATCH 04/12] change to sequence Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/data_handling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/power_grid_model/core/data_handling.py b/src/power_grid_model/core/data_handling.py index d8360e0e5..9990c9409 100644 --- a/src/power_grid_model/core/data_handling.py +++ b/src/power_grid_model/core/data_handling.py @@ -8,7 +8,7 @@ from enum import Enum -from typing import Mapping, Tuple, Union +from typing import Mapping, Sequence, Tuple, Union import numpy as np @@ -145,7 +145,7 @@ def create_output_data( attributes = processed_output_types[name] if attributes is None: result_dict[name] = initialize_array(output_type.value, name, shape=shape, empty=True) - elif attributes in [[], set()]: + elif isinstance(attributes, Sequence) and len(attributes) == 0: result_dict[name] = {} else: raise NotImplementedError( From ddab634e7d7b4121e07f61eaebd51f31b5b58258 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 19 Jul 2024 10:51:13 +0200 Subject: [PATCH 05/12] Revert "change to sequence" This reverts commit 2d5c41f89a79e7ec4ce77276a76de3d1772e00e5. Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/data_handling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/power_grid_model/core/data_handling.py b/src/power_grid_model/core/data_handling.py index 9990c9409..d8360e0e5 100644 --- a/src/power_grid_model/core/data_handling.py +++ b/src/power_grid_model/core/data_handling.py @@ -8,7 +8,7 @@ from enum import Enum -from typing import Mapping, Sequence, Tuple, Union +from typing import Mapping, Tuple, Union import numpy as np @@ -145,7 +145,7 @@ def create_output_data( attributes = processed_output_types[name] if attributes is None: result_dict[name] = initialize_array(output_type.value, name, shape=shape, empty=True) - elif isinstance(attributes, Sequence) and len(attributes) == 0: + elif attributes in [[], set()]: result_dict[name] = {} else: raise NotImplementedError( From b769eb7d52f1df9598d5dd9307e533782895d7c6 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 19 Jul 2024 14:12:40 +0200 Subject: [PATCH 06/12] convert to columnar Signed-off-by: Nitish Bharambe --- src/power_grid_model/_utils.py | 39 +++++++++++++++++++ src/power_grid_model/core/data_handling.py | 12 +----- src/power_grid_model/core/power_grid_model.py | 8 ++++ src/power_grid_model/typing.py | 4 +- tests/unit/test_data_handling.py | 4 +- 5 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/power_grid_model/_utils.py b/src/power_grid_model/_utils.py index 2d4e21607..d3836745f 100644 --- a/src/power_grid_model/_utils.py +++ b/src/power_grid_model/_utils.py @@ -10,10 +10,12 @@ We do not officially support this functionality and may remove features in this library at any given time! """ +from copy import deepcopy from typing import List, Optional, Union, cast import numpy as np +from power_grid_model.core.data_handling import OutputType, process_output_component_types from power_grid_model.core.dataset_definitions import ComponentType from power_grid_model.data_types import ( BatchArray, @@ -27,6 +29,7 @@ SinglePythonDataset, SparseBatchArray, ) +from power_grid_model.typing import ComponentAttributeMapping def is_nan(data) -> bool: @@ -284,3 +287,39 @@ def convert_single_dataset_to_python_single_dataset(data: SingleDataset) -> Sing ] for component, objects in data.items() } + + +def copy_output_to_columnar_dataset( + output_data: Dataset, + output_component_types: ComponentAttributeMapping, + output_type: OutputType, + available_components: list[ComponentType], +) -> Dataset: + """Temporary function to copy row based dataset to a column based dataset as per output_component_types. + + Args: + data (Dataset): + component_types (_ComponentAttributeMappingDict): + + Returns: + Dataset: converted to + Args: + output_data (Dataset): dataset to convert + output_component_types (ComponentAttributeMapping): desired component and attribute mapping + output_type (OutputType): output type sym or asym + available_components (list[ComponentType]): available components in model + + Returns: + Dataset: converted dataset + """ + processed_output_types = process_output_component_types(output_type, output_component_types, available_components) + + result_data = {} + for comp_name, attrs in processed_output_types.items(): + if attrs is None: + result_data[comp_name] = output_data[comp_name] + elif isinstance(attrs, (list, set)) and len(attrs) == 0: + result_data[comp_name] = {} + else: + result_data[comp_name] = {attr: deepcopy(output_data[comp_name][attr]) for attr in attrs} + return result_data diff --git a/src/power_grid_model/core/data_handling.py b/src/power_grid_model/core/data_handling.py index d8360e0e5..bb6cb45b0 100644 --- a/src/power_grid_model/core/data_handling.py +++ b/src/power_grid_model/core/data_handling.py @@ -142,17 +142,7 @@ def create_output_data( shape: Union[Tuple[int], Tuple[int, int]] = (batch_size, count) else: shape = (count,) - attributes = processed_output_types[name] - if attributes is None: - result_dict[name] = initialize_array(output_type.value, name, shape=shape, empty=True) - elif attributes in [[], set()]: - result_dict[name] = {} - else: - raise NotImplementedError( - "Columnar data types are not implemented yet." - "output_component_types must be provided with a list or set" - ) - + result_dict[name] = initialize_array(output_type.value, name, shape=shape, empty=True) return result_dict diff --git a/src/power_grid_model/core/power_grid_model.py b/src/power_grid_model/core/power_grid_model.py index 1409e83a5..5b9471ee0 100644 --- a/src/power_grid_model/core/power_grid_model.py +++ b/src/power_grid_model/core/power_grid_model.py @@ -10,6 +10,7 @@ import numpy as np +from power_grid_model._utils import copy_output_to_columnar_dataset from power_grid_model.core.data_handling import ( create_output_data, get_output_type, @@ -287,6 +288,13 @@ def _calculate_impl( continue_on_batch_error=continue_on_batch_error, batch_size=batch_size, decode_error=decode_error ) + available_components = list(self._get_output_component_count(calculation_type=calculation_type).keys()) + output_data = copy_output_to_columnar_dataset( + output_data=output_data, + output_type=get_output_type(calculation_type=calculation_type, symmetric=symmetric), + available_components=available_components, + output_component_types=output_component_types, + ) return output_data def _calculate_power_flow( diff --git a/src/power_grid_model/typing.py b/src/power_grid_model/typing.py index 7a20b3046..455ce2be3 100644 --- a/src/power_grid_model/typing.py +++ b/src/power_grid_model/typing.py @@ -9,6 +9,4 @@ _ComponentAttributeMappingDict = dict[ComponentType, set[str] | list[str] | None] -ComponentAttributeMapping = ( - set[ComponentType] | list[ComponentType] | None | dict[ComponentType, set[str] | list[str] | None] -) +ComponentAttributeMapping = set[ComponentType] | list[ComponentType] | None | _ComponentAttributeMappingDict diff --git a/tests/unit/test_data_handling.py b/tests/unit/test_data_handling.py index 8daf8cbf6..93aedccf2 100644 --- a/tests/unit/test_data_handling.py +++ b/tests/unit/test_data_handling.py @@ -38,7 +38,9 @@ CT.sym_load: initialize_array(DT.sym_output, CT.sym_load, 3), }, ), - ({CT.node: [], CT.sym_load: []}, True, {CT.node: dict(), CT.sym_load: dict()}), + pytest.param( + {CT.node: [], CT.sym_load: []}, True, {CT.node: dict(), CT.sym_load: dict()}, marks=pytest.mark.xfail + ), pytest.param({CT.node: [], CT.sym_load: ["p"]}, True, {}, marks=pytest.mark.xfail), pytest.param({CT.node: ["u"], CT.sym_load: ["p"]}, True, {}, marks=pytest.mark.xfail), pytest.param({CT.node: None, CT.sym_load: ["p"]}, True, {}, marks=pytest.mark.xfail), From 3ec562a0bc3793c1eda9a53623c0ef6c84b42d34 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Mon, 22 Jul 2024 09:33:18 +0200 Subject: [PATCH 07/12] change comments Signed-off-by: Nitish Bharambe --- src/power_grid_model/_utils.py | 1 + src/power_grid_model/core/data_handling.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/power_grid_model/_utils.py b/src/power_grid_model/_utils.py index d3836745f..729785c30 100644 --- a/src/power_grid_model/_utils.py +++ b/src/power_grid_model/_utils.py @@ -296,6 +296,7 @@ def copy_output_to_columnar_dataset( available_components: list[ComponentType], ) -> Dataset: """Temporary function to copy row based dataset to a column based dataset as per output_component_types. + The purpose of this function is to mimic columnar data without any memory overhead benefits. Args: data (Dataset): diff --git a/src/power_grid_model/core/data_handling.py b/src/power_grid_model/core/data_handling.py index bb6cb45b0..9d9509806 100644 --- a/src/power_grid_model/core/data_handling.py +++ b/src/power_grid_model/core/data_handling.py @@ -110,7 +110,7 @@ def create_output_data( batch_size: int, ) -> Dataset: """ - Create the output dataset based on component and batch size from the model; and output attribtues requested by user. + Create the output dataset based on component and batch size from the model; and output attributes requested by user. Args: output_component_types: From ba7ce89aa1cfee503e4eb4d6cf1dc4b29ce0e364 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Mon, 22 Jul 2024 11:44:07 +0200 Subject: [PATCH 08/12] change type hint Signed-off-by: Nitish Bharambe --- src/power_grid_model/_utils.py | 2 +- src/power_grid_model/core/power_grid_model.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/power_grid_model/_utils.py b/src/power_grid_model/_utils.py index 729785c30..393085f8b 100644 --- a/src/power_grid_model/_utils.py +++ b/src/power_grid_model/_utils.py @@ -294,7 +294,7 @@ def copy_output_to_columnar_dataset( output_component_types: ComponentAttributeMapping, output_type: OutputType, available_components: list[ComponentType], -) -> Dataset: +) -> dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: """Temporary function to copy row based dataset to a column based dataset as per output_component_types. The purpose of this function is to mimic columnar data without any memory overhead benefits. diff --git a/src/power_grid_model/core/power_grid_model.py b/src/power_grid_model/core/power_grid_model.py index 5b9471ee0..efb7a9cec 100644 --- a/src/power_grid_model/core/power_grid_model.py +++ b/src/power_grid_model/core/power_grid_model.py @@ -311,7 +311,7 @@ def _calculate_power_flow( decode_error: bool = True, tap_changing_strategy: Union[TapChangingStrategy, str] = TapChangingStrategy.disabled, experimental_features: Union[_ExperimentalFeatures, str] = _ExperimentalFeatures.disabled, - ): + ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: calculation_type = CalculationType.power_flow options = self._options( calculation_type=calculation_type, @@ -346,7 +346,7 @@ def _calculate_state_estimation( continue_on_batch_error: bool = False, decode_error: bool = True, experimental_features: Union[_ExperimentalFeatures, str] = _ExperimentalFeatures.disabled, - ) -> Dict[ComponentType, np.ndarray]: + ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: calculation_type = CalculationType.state_estimation options = self._options( calculation_type=calculation_type, @@ -378,7 +378,7 @@ def _calculate_short_circuit( decode_error: bool = True, short_circuit_voltage_scaling: Union[ShortCircuitVoltageScaling, str] = ShortCircuitVoltageScaling.maximum, experimental_features: Union[_ExperimentalFeatures, str] = _ExperimentalFeatures.disabled, - ) -> Dict[ComponentType, np.ndarray]: + ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: calculation_type = CalculationType.short_circuit symmetric = False @@ -418,7 +418,7 @@ def calculate_power_flow( continue_on_batch_error: bool = False, decode_error: bool = True, tap_changing_strategy: Union[TapChangingStrategy, str] = TapChangingStrategy.disabled, - ) -> Dict[ComponentType, np.ndarray]: + ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: """ Calculate power flow once with the current model attributes. Or calculate in batch with the given update dataset in batch. @@ -511,7 +511,7 @@ def calculate_state_estimation( output_component_types: Optional[Union[Set[ComponentType], List[ComponentType]]] = None, continue_on_batch_error: bool = False, decode_error: bool = True, - ) -> Dict[ComponentType, np.ndarray]: + ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: """ Calculate state estimation once with the current model attributes. Or calculate in batch with the given update dataset in batch. @@ -598,7 +598,7 @@ def calculate_short_circuit( continue_on_batch_error: bool = False, decode_error: bool = True, short_circuit_voltage_scaling: Union[ShortCircuitVoltageScaling, str] = ShortCircuitVoltageScaling.maximum, - ) -> Dict[ComponentType, np.ndarray]: + ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: """ Calculate a short circuit once with the current model attributes. Or calculate in batch with the given update dataset in batch From 196690e32b20c42e505fe56ee84ce21c5384cf70 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Mon, 22 Jul 2024 14:19:25 +0200 Subject: [PATCH 09/12] Revert "change type hint" This reverts commit ba7ce89aa1cfee503e4eb4d6cf1dc4b29ce0e364. Signed-off-by: Nitish Bharambe --- src/power_grid_model/_utils.py | 2 +- src/power_grid_model/core/power_grid_model.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/power_grid_model/_utils.py b/src/power_grid_model/_utils.py index 393085f8b..729785c30 100644 --- a/src/power_grid_model/_utils.py +++ b/src/power_grid_model/_utils.py @@ -294,7 +294,7 @@ def copy_output_to_columnar_dataset( output_component_types: ComponentAttributeMapping, output_type: OutputType, available_components: list[ComponentType], -) -> dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: +) -> Dataset: """Temporary function to copy row based dataset to a column based dataset as per output_component_types. The purpose of this function is to mimic columnar data without any memory overhead benefits. diff --git a/src/power_grid_model/core/power_grid_model.py b/src/power_grid_model/core/power_grid_model.py index efb7a9cec..5b9471ee0 100644 --- a/src/power_grid_model/core/power_grid_model.py +++ b/src/power_grid_model/core/power_grid_model.py @@ -311,7 +311,7 @@ def _calculate_power_flow( decode_error: bool = True, tap_changing_strategy: Union[TapChangingStrategy, str] = TapChangingStrategy.disabled, experimental_features: Union[_ExperimentalFeatures, str] = _ExperimentalFeatures.disabled, - ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: + ): calculation_type = CalculationType.power_flow options = self._options( calculation_type=calculation_type, @@ -346,7 +346,7 @@ def _calculate_state_estimation( continue_on_batch_error: bool = False, decode_error: bool = True, experimental_features: Union[_ExperimentalFeatures, str] = _ExperimentalFeatures.disabled, - ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: + ) -> Dict[ComponentType, np.ndarray]: calculation_type = CalculationType.state_estimation options = self._options( calculation_type=calculation_type, @@ -378,7 +378,7 @@ def _calculate_short_circuit( decode_error: bool = True, short_circuit_voltage_scaling: Union[ShortCircuitVoltageScaling, str] = ShortCircuitVoltageScaling.maximum, experimental_features: Union[_ExperimentalFeatures, str] = _ExperimentalFeatures.disabled, - ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: + ) -> Dict[ComponentType, np.ndarray]: calculation_type = CalculationType.short_circuit symmetric = False @@ -418,7 +418,7 @@ def calculate_power_flow( continue_on_batch_error: bool = False, decode_error: bool = True, tap_changing_strategy: Union[TapChangingStrategy, str] = TapChangingStrategy.disabled, - ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: + ) -> Dict[ComponentType, np.ndarray]: """ Calculate power flow once with the current model attributes. Or calculate in batch with the given update dataset in batch. @@ -511,7 +511,7 @@ def calculate_state_estimation( output_component_types: Optional[Union[Set[ComponentType], List[ComponentType]]] = None, continue_on_batch_error: bool = False, decode_error: bool = True, - ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: + ) -> Dict[ComponentType, np.ndarray]: """ Calculate state estimation once with the current model attributes. Or calculate in batch with the given update dataset in batch. @@ -598,7 +598,7 @@ def calculate_short_circuit( continue_on_batch_error: bool = False, decode_error: bool = True, short_circuit_voltage_scaling: Union[ShortCircuitVoltageScaling, str] = ShortCircuitVoltageScaling.maximum, - ) -> Dict[ComponentType, np.ndarray] | dict[ComponentType, dict[str, np.ndarray]]: + ) -> Dict[ComponentType, np.ndarray]: """ Calculate a short circuit once with the current model attributes. Or calculate in batch with the given update dataset in batch From 4d4aebb66fa9ad5397d48dfbbb9af9ecc204c043 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Mon, 22 Jul 2024 14:21:54 +0200 Subject: [PATCH 10/12] add experimental flag Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/power_grid_model.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/power_grid_model/core/power_grid_model.py b/src/power_grid_model/core/power_grid_model.py index 5b9471ee0..1978002b4 100644 --- a/src/power_grid_model/core/power_grid_model.py +++ b/src/power_grid_model/core/power_grid_model.py @@ -31,6 +31,7 @@ TapChangingStrategy, _ExperimentalFeatures, ) +from power_grid_model.errors import PowerGridError from power_grid_model.typing import ComponentAttributeMapping @@ -237,6 +238,7 @@ def _calculate_impl( options: Options, continue_on_batch_error: bool, decode_error: bool, + experimental_features: _ExperimentalFeatures, ): """ Core calculation routine @@ -263,6 +265,11 @@ def _calculate_impl( update_ptr = ConstDatasetPtr() batch_size = 1 + if experimental_features == _ExperimentalFeatures.disabled and isinstance(output_component_types, dict): + raise PowerGridError( + "Experimental features flag must be enabled when providing a dict for output_component_types" + ) + output_data = self._construct_output( output_component_types=output_component_types, calculation_type=calculation_type, @@ -331,6 +338,7 @@ def _calculate_power_flow( options=options, continue_on_batch_error=continue_on_batch_error, decode_error=decode_error, + experimental_features=experimental_features, ) def _calculate_state_estimation( @@ -365,6 +373,7 @@ def _calculate_state_estimation( options=options, continue_on_batch_error=continue_on_batch_error, decode_error=decode_error, + experimental_features=experimental_features, ) def _calculate_short_circuit( @@ -398,6 +407,7 @@ def _calculate_short_circuit( options=options, continue_on_batch_error=continue_on_batch_error, decode_error=decode_error, + experimental_features=experimental_features, ) def calculate_power_flow( From ca59ea9426ff09e0ce0c16b273af42f807e6144d Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Mon, 22 Jul 2024 14:31:10 +0200 Subject: [PATCH 11/12] remove local Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/power_grid_model.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/power_grid_model/core/power_grid_model.py b/src/power_grid_model/core/power_grid_model.py index 1978002b4..d2c4cc29f 100644 --- a/src/power_grid_model/core/power_grid_model.py +++ b/src/power_grid_model/core/power_grid_model.py @@ -238,7 +238,7 @@ def _calculate_impl( options: Options, continue_on_batch_error: bool, decode_error: bool, - experimental_features: _ExperimentalFeatures, + experimental_features: Union[_ExperimentalFeatures, str], # pylint: disable=too-many-arguments ): """ Core calculation routine @@ -265,7 +265,10 @@ def _calculate_impl( update_ptr = ConstDatasetPtr() batch_size = 1 - if experimental_features == _ExperimentalFeatures.disabled and isinstance(output_component_types, dict): + if experimental_features in [ + _ExperimentalFeatures.disabled, + _ExperimentalFeatures.disabled.name, + ] and isinstance(output_component_types, dict): raise PowerGridError( "Experimental features flag must be enabled when providing a dict for output_component_types" ) @@ -295,11 +298,10 @@ def _calculate_impl( continue_on_batch_error=continue_on_batch_error, batch_size=batch_size, decode_error=decode_error ) - available_components = list(self._get_output_component_count(calculation_type=calculation_type).keys()) output_data = copy_output_to_columnar_dataset( output_data=output_data, output_type=get_output_type(calculation_type=calculation_type, symmetric=symmetric), - available_components=available_components, + available_components=list(self._get_output_component_count(calculation_type=calculation_type).keys()), output_component_types=output_component_types, ) return output_data From a6e1e6b781b792acd2caac027317fbeeac8befc1 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:23:40 +0200 Subject: [PATCH 12/12] Update src/power_grid_model/_utils.py Signed-off-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> --- src/power_grid_model/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/power_grid_model/_utils.py b/src/power_grid_model/_utils.py index 729785c30..26f035b22 100644 --- a/src/power_grid_model/_utils.py +++ b/src/power_grid_model/_utils.py @@ -296,7 +296,7 @@ def copy_output_to_columnar_dataset( available_components: list[ComponentType], ) -> Dataset: """Temporary function to copy row based dataset to a column based dataset as per output_component_types. - The purpose of this function is to mimic columnar data without any memory overhead benefits. + The purpose of this function is to mimic columnar data without any memory footprint benefits. Args: data (Dataset):