From 39a9bf0e55aa01231bf64e12d0fe44ba687bd634 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 11 Oct 2024 11:10:38 +0200 Subject: [PATCH 1/7] add compatibility check Signed-off-by: Nitish Bharambe --- .../core/power_grid_dataset.py | 21 ++++++++++- tests/unit/test_data_handling.py | 36 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index 05db79d4b..3ccee3c1b 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -8,7 +8,13 @@ from typing import Any, Mapping, Optional -from power_grid_model._utils import get_dataset_type, is_columnar, is_nan_or_equivalent, process_data_filter +from power_grid_model._utils import ( + _extract_data_from_component_data, + get_dataset_type, + is_columnar, + is_nan_or_equivalent, + process_data_filter, +) from power_grid_model.core.buffer_handling import ( BufferProperties, CAttributeBuffer, @@ -231,6 +237,7 @@ def _add_component_data(self, component: ComponentType, data: ComponentData, all return self._validate_properties(data, self._schema[component]) + self._validate_dtypes_compatibility(data, self._schema[component]) c_buffer = get_buffer_view(data, self._schema[component], self._is_batch, self._batch_size) self._buffer_views.append(c_buffer) self._register_buffer(component, c_buffer) @@ -266,6 +273,18 @@ def _validate_properties(self, data: ComponentData, schema: ComponentMetaData): if properties.batch_size != self._batch_size: raise ValueError(f"Dataset must have a consistent batch size across all components. {VALIDATOR_MSG}") + def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentMetaData): + sub_data = _extract_data_from_component_data(data) + if is_columnar(data): + for attr, array in sub_data.items(): + if schema.dtype.names is None or attr not in schema.dtype.names: + raise Warning(f"Attribute {attr} is not in schema. {VALIDATOR_MSG}") + if array.dtype != schema.dtype[attr]: + raise Warning(f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}") + else: + if sub_data.dtype != schema.dtype: + raise Warning("Data type does not match schema. {VALIDATOR_MSG}") + def __del__(self): pgc.destroy_dataset_mutable(self._mutable_dataset) diff --git a/tests/unit/test_data_handling.py b/tests/unit/test_data_handling.py index 0fe8006a0..e6aceff35 100644 --- a/tests/unit/test_data_handling.py +++ b/tests/unit/test_data_handling.py @@ -2,12 +2,16 @@ # # SPDX-License-Identifier: MPL-2.0 +import warnings + import numpy as np import pytest from power_grid_model._utils import is_columnar from power_grid_model.core.data_handling import create_output_data from power_grid_model.core.dataset_definitions import ComponentType as CT, DatasetType as DT +from power_grid_model.core.power_grid_core import VoidPtr +from power_grid_model.core.power_grid_dataset import CMutableDataset from power_grid_model.core.power_grid_meta import initialize_array @@ -69,3 +73,35 @@ def test_create_output_data(output_component_types, is_batch, expected): else: assert actual[comp].keys() == expected[comp].keys() assert all(actual[comp][attr].dtype == expected[comp][attr].dtype for attr in expected[comp]) + + +def test_dtype_compatibility_check_normal(): + nodes = initialize_array(DT.sym_output, CT.node, (1, 2)) + nodes_ptr = nodes.ctypes.data_as(VoidPtr) + + data = {CT.node: nodes} + mutable_dataset = CMutableDataset(data, DT.sym_output) + buffer_views = mutable_dataset.get_buffer_views() + + assert buffer_views[0].data.value == nodes_ptr.value + + +def test_dtype_compatibility_check_compatible(): + nodes = initialize_array(DT.sym_output, CT.node, 4) + nodes = nodes[::2] + nodes_ptr = nodes.ctypes.data_as(VoidPtr) + + data = {CT.node: nodes} + with warnings.catch_warnings(): + warnings.simplefilter("error") + mutable_dataset = CMutableDataset(data, DT.sym_output) + buffer_views = mutable_dataset.get_buffer_views() + + assert buffer_views[0].data.value != nodes_ptr.value + + +def test_dtype_compatibility_check__error(): + nodes = initialize_array(DT.sym_output, CT.node, (1, 2)) + data = {CT.node: nodes.astype(nodes.dtype.newbyteorder("S"))} + with pytest.raises(Warning): + CMutableDataset(data, DT.sym_output) From 5a0763612fcc44908d1c69f0dfda8f7f42443100 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 11 Oct 2024 15:32:23 +0200 Subject: [PATCH 2/7] change shape and warning check Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/power_grid_dataset.py | 17 ++++++++++++----- tests/unit/test_data_handling.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index 3ccee3c1b..84125b29d 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -6,6 +6,7 @@ Power grid model raw dataset handler """ +import warnings from typing import Any, Mapping, Optional from power_grid_model._utils import ( @@ -277,13 +278,19 @@ def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentM sub_data = _extract_data_from_component_data(data) if is_columnar(data): for attr, array in sub_data.items(): - if schema.dtype.names is None or attr not in schema.dtype.names: - raise Warning(f"Attribute {attr} is not in schema. {VALIDATOR_MSG}") - if array.dtype != schema.dtype[attr]: - raise Warning(f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}") + if ( + schema.dtype.names is None + or attr not in schema.dtype.names + or (schema.dtype[attr].shape == (3,) and array.shape[-1] != 3) + ): + raise TypeError("Given data has a different schema than supported.") + if array.dtype.base != schema.dtype[attr].base: + warnings.warn( + f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}", DeprecationWarning + ) else: if sub_data.dtype != schema.dtype: - raise Warning("Data type does not match schema. {VALIDATOR_MSG}") + warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) def __del__(self): pgc.destroy_dataset_mutable(self._mutable_dataset) diff --git a/tests/unit/test_data_handling.py b/tests/unit/test_data_handling.py index e6aceff35..108e02445 100644 --- a/tests/unit/test_data_handling.py +++ b/tests/unit/test_data_handling.py @@ -103,5 +103,5 @@ def test_dtype_compatibility_check_compatible(): def test_dtype_compatibility_check__error(): nodes = initialize_array(DT.sym_output, CT.node, (1, 2)) data = {CT.node: nodes.astype(nodes.dtype.newbyteorder("S"))} - with pytest.raises(Warning): + with pytest.warns(DeprecationWarning): CMutableDataset(data, DT.sym_output) From 7ad224891c4b7e3a64878c9e8a3f117ffad09d52 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:49:01 +0200 Subject: [PATCH 3/7] Update src/power_grid_model/core/power_grid_dataset.py Co-authored-by: Martijn Govers Signed-off-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> --- src/power_grid_model/core/power_grid_dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index 84125b29d..46e12c861 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -288,8 +288,7 @@ def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentM warnings.warn( f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}", DeprecationWarning ) - else: - if sub_data.dtype != schema.dtype: + elif sub_data.dtype != schema.dtype: warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) def __del__(self): From f90cf37da600762552b6b028e5bf5cf1beb5aab0 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 11 Oct 2024 16:09:03 +0200 Subject: [PATCH 4/7] format changes and doc Signed-off-by: Nitish Bharambe --- docs/api_reference/python-api-reference.md | 1 + src/power_grid_model/core/power_grid_dataset.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api_reference/python-api-reference.md b/docs/api_reference/python-api-reference.md index 6257888d7..3a97a8865 100644 --- a/docs/api_reference/python-api-reference.md +++ b/docs/api_reference/python-api-reference.md @@ -11,6 +11,7 @@ SPDX-License-Identifier: MPL-2.0 .. autoclass:: power_grid_model.PowerGridModel :show-inheritance: .. autofunction:: power_grid_model.initialize_array +.. autoclass:: power_grid_model.power_grid_meta_data ``` ## enum diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index 46e12c861..cd6278c10 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -289,7 +289,7 @@ def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentM f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}", DeprecationWarning ) elif sub_data.dtype != schema.dtype: - warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) + warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) def __del__(self): pgc.destroy_dataset_mutable(self._mutable_dataset) From a4ef4018462ff2a40dec25f014548fee4166067e Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Sat, 12 Oct 2024 15:22:05 +0200 Subject: [PATCH 5/7] move to buffer handling Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/buffer_handling.py | 5 ++++ .../core/power_grid_dataset.py | 27 +------------------ 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/src/power_grid_model/core/buffer_handling.py b/src/power_grid_model/core/buffer_handling.py index b012c2333..5d6fe2700 100644 --- a/src/power_grid_model/core/buffer_handling.py +++ b/src/power_grid_model/core/buffer_handling.py @@ -6,6 +6,7 @@ Power grid model buffer handler """ +import warnings from dataclasses import dataclass from typing import cast @@ -83,6 +84,8 @@ def _get_raw_data_view(data: np.ndarray, dtype: np.dtype) -> VoidPtr: Returns: a raw view on the data set. """ + if data.dtype != dtype: + warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) return np.ascontiguousarray(data, dtype=dtype).ctypes.data_as(VoidPtr) @@ -115,6 +118,8 @@ def _get_raw_attribute_data_view(data: np.ndarray, schema: ComponentMetaData, at Returns: a raw view on the data set. """ + if schema.dtype[attribute].shape == (3,) and data.shape[-1] != 3: + raise TypeError("Given data has a different schema than supported.") return _get_raw_data_view(data, dtype=schema.dtype[attribute].base) diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index cd6278c10..05db79d4b 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -6,16 +6,9 @@ Power grid model raw dataset handler """ -import warnings from typing import Any, Mapping, Optional -from power_grid_model._utils import ( - _extract_data_from_component_data, - get_dataset_type, - is_columnar, - is_nan_or_equivalent, - process_data_filter, -) +from power_grid_model._utils import get_dataset_type, is_columnar, is_nan_or_equivalent, process_data_filter from power_grid_model.core.buffer_handling import ( BufferProperties, CAttributeBuffer, @@ -238,7 +231,6 @@ def _add_component_data(self, component: ComponentType, data: ComponentData, all return self._validate_properties(data, self._schema[component]) - self._validate_dtypes_compatibility(data, self._schema[component]) c_buffer = get_buffer_view(data, self._schema[component], self._is_batch, self._batch_size) self._buffer_views.append(c_buffer) self._register_buffer(component, c_buffer) @@ -274,23 +266,6 @@ def _validate_properties(self, data: ComponentData, schema: ComponentMetaData): if properties.batch_size != self._batch_size: raise ValueError(f"Dataset must have a consistent batch size across all components. {VALIDATOR_MSG}") - def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentMetaData): - sub_data = _extract_data_from_component_data(data) - if is_columnar(data): - for attr, array in sub_data.items(): - if ( - schema.dtype.names is None - or attr not in schema.dtype.names - or (schema.dtype[attr].shape == (3,) and array.shape[-1] != 3) - ): - raise TypeError("Given data has a different schema than supported.") - if array.dtype.base != schema.dtype[attr].base: - warnings.warn( - f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}", DeprecationWarning - ) - elif sub_data.dtype != schema.dtype: - warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) - def __del__(self): pgc.destroy_dataset_mutable(self._mutable_dataset) From 4fcb6a1c7a13a4b56a724bbdb2541ea097bb0db8 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:20:50 +0200 Subject: [PATCH 6/7] Update src/power_grid_model/core/buffer_handling.py Signed-off-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> --- src/power_grid_model/core/buffer_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/power_grid_model/core/buffer_handling.py b/src/power_grid_model/core/buffer_handling.py index 5d6fe2700..9a7c7e32a 100644 --- a/src/power_grid_model/core/buffer_handling.py +++ b/src/power_grid_model/core/buffer_handling.py @@ -119,7 +119,7 @@ def _get_raw_attribute_data_view(data: np.ndarray, schema: ComponentMetaData, at a raw view on the data set. """ if schema.dtype[attribute].shape == (3,) and data.shape[-1] != 3: - raise TypeError("Given data has a different schema than supported.") + raise ValueError("Given data has a different schema than supported.") return _get_raw_data_view(data, dtype=schema.dtype[attribute].base) From 2737a55e981c58a31ce8be75a54477b5870248e5 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:41:42 +0200 Subject: [PATCH 7/7] Update docs/api_reference/python-api-reference.md Signed-off-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> --- docs/api_reference/python-api-reference.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/api_reference/python-api-reference.md b/docs/api_reference/python-api-reference.md index 3a97a8865..6257888d7 100644 --- a/docs/api_reference/python-api-reference.md +++ b/docs/api_reference/python-api-reference.md @@ -11,7 +11,6 @@ SPDX-License-Identifier: MPL-2.0 .. autoclass:: power_grid_model.PowerGridModel :show-inheritance: .. autofunction:: power_grid_model.initialize_array -.. autoclass:: power_grid_model.power_grid_meta_data ``` ## enum