From 157f23d6d5daf195dd2592ff636bb4970cff945b Mon Sep 17 00:00:00 2001 From: Santiago Figueroa Manrique Date: Wed, 9 Oct 2024 09:34:26 +0200 Subject: [PATCH 01/15] updated dataset and component types Signed-off-by: Santiago Figueroa Manrique --- tests/unit/test_power_grid_model.py | 69 ++++++++++++++++------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/tests/unit/test_power_grid_model.py b/tests/unit/test_power_grid_model.py index eb9de80b9..a5c045c65 100644 --- a/tests/unit/test_power_grid_model.py +++ b/tests/unit/test_power_grid_model.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from power_grid_model import ComponentType, PowerGridModel, initialize_array +from power_grid_model import ComponentType, DatasetType, PowerGridModel, initialize_array from power_grid_model.errors import InvalidCalculationMethod, IterationDiverge, PowerGridBatchError, PowerGridError from power_grid_model.validation import assert_valid_input_data @@ -34,31 +34,31 @@ # test data INPUT = { - "node": [{"id": 0, "u_rated": 100.0}], - "source": [{"id": 1, "node": 0, "status": 1, "u_ref": 1.0, "sk": 1000.0, "rx_ratio": 0.0}], - "sym_load": [{"id": 2, "node": 0, "status": 1, "type": 2, "p_specified": 0.0, "q_specified": 500.0}], + ComponentType.node: [{"id": 0, "u_rated": 100.0}], + ComponentType.source: [{"id": 1, "node": 0, "status": 1, "u_ref": 1.0, "sk": 1000.0, "rx_ratio": 0.0}], + ComponentType.sym_load: [{"id": 2, "node": 0, "status": 1, "type": 2, "p_specified": 0.0, "q_specified": 500.0}], } -SYM_OUTPUT = {"node": [{"id": 0, "u": 50.0, "u_pu": 0.5, "u_angle": 0.0}]} +SYM_OUTPUT = {ComponentType.node: [{"id": 0, "u": 50.0, "u_pu": 0.5, "u_angle": 0.0}]} SYM_OUTPUT_BATCH = [ - {"node": [{"id": 0, "u": 40.0, "u_pu": 0.4, "u_angle": 0.0}]}, - {"node": [{"id": 0, "u": 70.0, "u_pu": 0.7, "u_angle": 0.0}]}, + {ComponentType.node: [{"id": 0, "u": 40.0, "u_pu": 0.4, "u_angle": 0.0}]}, + {ComponentType.node: [{"id": 0, "u": 70.0, "u_pu": 0.7, "u_angle": 0.0}]}, ] UPDATE_BATCH = [ - {"source": [{"id": 1, "u_ref": 0.5}], "sym_load": [{"id": 2, "q_specified": 100.0}]}, - {"sym_load": [{"id": 2, "q_specified": 300.0}]}, + {ComponentType.source: [{"id": 1, "u_ref": 0.5}], ComponentType.sym_load: [{"id": 2, "q_specified": 100.0}]}, + {ComponentType.sym_load: [{"id": 2, "q_specified": 300.0}]}, ] @pytest.fixture def case_data(): return { - "input": convert_python_to_numpy(INPUT, "input"), - "output": convert_python_to_numpy(SYM_OUTPUT, "sym_output"), - "update_batch": convert_python_to_numpy(UPDATE_BATCH, "update"), - "output_batch": convert_python_to_numpy(SYM_OUTPUT_BATCH, "sym_output"), + "input": convert_python_to_numpy(INPUT, DatasetType.input), + "output": convert_python_to_numpy(SYM_OUTPUT, DatasetType.sym_output), + "update_batch": convert_python_to_numpy(UPDATE_BATCH, DatasetType.update), + "output_batch": convert_python_to_numpy(SYM_OUTPUT_BATCH, DatasetType.sym_output), } @@ -74,22 +74,22 @@ def test_simple_power_flow(model: PowerGridModel, case_data): def test_simple_update(model: PowerGridModel, case_data): update_batch = case_data["update_batch"] - source_indptr = update_batch["source"]["indptr"] - source_update = update_batch["source"]["data"] + source_indptr = update_batch[ComponentType.source]["indptr"] + source_update = update_batch[ComponentType.source]["data"] update_data = { - "source": source_update[source_indptr[0] : source_indptr[1]], - "sym_load": update_batch["sym_load"][0, :], + ComponentType.source: source_update[source_indptr[0] : source_indptr[1]], + ComponentType.sym_load: update_batch[ComponentType.sym_load][0, :], } model.update(update_data=update_data) - expected_result = {ComponentType.node: case_data["output_batch"]["node"][0, :]} + expected_result = {ComponentType.node: case_data["output_batch"][ComponentType.node][0, :]} result = model.calculate_power_flow() compare_result(result, expected_result, rtol=0.0, atol=1e-8) def test_update_error(model: PowerGridModel): - load_update = initialize_array("update", "sym_load", 1) + load_update = initialize_array(DatasetType.update, ComponentType.sym_load, 1) load_update["id"] = 5 - update_data = {"sym_load": load_update} + update_data = {ComponentType.sym_load: load_update} with pytest.raises(PowerGridError, match="The id cannot be found:"): model.update(update_data=update_data) @@ -103,7 +103,7 @@ def test_copy_model(model: PowerGridModel, case_data): def test_get_indexer(model: PowerGridModel): ids = np.array([2, 2]) expected_indexer = np.array([0, 0]) - indexer = model.get_indexer("sym_load", ids) + indexer = model.get_indexer(ComponentType.sym_load, ids) assert np.allclose(expected_indexer, indexer) @@ -113,7 +113,7 @@ def test_batch_power_flow(model: PowerGridModel, case_data): def test_construction_error(case_data): - case_data["input"]["sym_load"]["id"] = 0 + case_data["input"][ComponentType.sym_load]["id"] = 0 with pytest.raises(PowerGridError, match="Conflicting id detected:"): PowerGridModel(case_data["input"]) @@ -131,7 +131,7 @@ def test_single_calculation_error(model: PowerGridModel): def test_batch_calculation_error(model: PowerGridModel, case_data): # wrong id - case_data["update_batch"]["sym_load"]["id"][1, 0] = 5 + case_data["update_batch"][ComponentType.sym_load]["id"][1, 0] = 5 # with error with pytest.raises(PowerGridBatchError) as e: model.calculate_power_flow(update_data=case_data["update_batch"]) @@ -143,7 +143,7 @@ def test_batch_calculation_error(model: PowerGridModel, case_data): def test_batch_calculation_error_continue(model: PowerGridModel, case_data): # wrong id - case_data["update_batch"]["sym_load"]["id"][1, 0] = 5 + case_data["update_batch"][ComponentType.sym_load]["id"][1, 0] = 5 result = model.calculate_power_flow(update_data=case_data["update_batch"], continue_on_batch_error=True) # assert error error = model.batch_error @@ -159,18 +159,25 @@ def test_batch_calculation_error_continue(model: PowerGridModel, case_data): with pytest.raises(PowerGridError, match="The calculation method is invalid for this calculation!"): model.calculate_state_estimation( calculation_method="iterative_current", - update_data={"source": initialize_array("update", "source", shape=(5, 0))}, + update_data={ + ComponentType.source: initialize_array(DatasetType.update, ComponentType.source, shape=(5, 0)) + }, continue_on_batch_error=True, ) def test_empty_input(): - node = initialize_array("input", "node", 0) - line = initialize_array("input", "line", 0) - sym_load = initialize_array("input", "sym_load", 0) - source = initialize_array("input", "source", 0) - - input_data = {"node": node, "line": line, "sym_load": sym_load, "source": source} + node = initialize_array(DatasetType.input, ComponentType.node, 0) + line = initialize_array(DatasetType.input, ComponentType.line, 0) + sym_load = initialize_array(DatasetType.input, ComponentType.sym_load, 0) + source = initialize_array(DatasetType.input, ComponentType.source, 0) + + input_data = { + ComponentType.node: node, + ComponentType.line: line, + ComponentType.sym_load: sym_load, + ComponentType.source: source, + } assert_valid_input_data(input_data) model = PowerGridModel(input_data, system_frequency=50.0) From 379451508ead0ee49ddc43bd11e7f615a48a1519 Mon Sep 17 00:00:00 2001 From: Santiago Figueroa Manrique Date: Wed, 9 Oct 2024 16:35:30 +0200 Subject: [PATCH 02/15] minor bug fix Signed-off-by: Santiago Figueroa Manrique --- src/power_grid_model/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/power_grid_model/_utils.py b/src/power_grid_model/_utils.py index f396967ba..23e230d4b 100644 --- a/src/power_grid_model/_utils.py +++ b/src/power_grid_model/_utils.py @@ -418,7 +418,7 @@ def compatibility_convert_row_columnar_dataset( ) if is_sparse(data[comp_name]): - result_data[comp_name] = {"indptr": _extract_indptr(data), "data": converted_sub_data} + result_data[comp_name] = {"indptr": _extract_indptr(data[comp_name]), "data": converted_sub_data} else: result_data[comp_name] = converted_sub_data return result_data From 39a9bf0e55aa01231bf64e12d0fe44ba687bd634 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 11 Oct 2024 11:10:38 +0200 Subject: [PATCH 03/15] add compatibility check Signed-off-by: Nitish Bharambe --- .../core/power_grid_dataset.py | 21 ++++++++++- tests/unit/test_data_handling.py | 36 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index 05db79d4b..3ccee3c1b 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -8,7 +8,13 @@ from typing import Any, Mapping, Optional -from power_grid_model._utils import get_dataset_type, is_columnar, is_nan_or_equivalent, process_data_filter +from power_grid_model._utils import ( + _extract_data_from_component_data, + get_dataset_type, + is_columnar, + is_nan_or_equivalent, + process_data_filter, +) from power_grid_model.core.buffer_handling import ( BufferProperties, CAttributeBuffer, @@ -231,6 +237,7 @@ def _add_component_data(self, component: ComponentType, data: ComponentData, all return self._validate_properties(data, self._schema[component]) + self._validate_dtypes_compatibility(data, self._schema[component]) c_buffer = get_buffer_view(data, self._schema[component], self._is_batch, self._batch_size) self._buffer_views.append(c_buffer) self._register_buffer(component, c_buffer) @@ -266,6 +273,18 @@ def _validate_properties(self, data: ComponentData, schema: ComponentMetaData): if properties.batch_size != self._batch_size: raise ValueError(f"Dataset must have a consistent batch size across all components. {VALIDATOR_MSG}") + def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentMetaData): + sub_data = _extract_data_from_component_data(data) + if is_columnar(data): + for attr, array in sub_data.items(): + if schema.dtype.names is None or attr not in schema.dtype.names: + raise Warning(f"Attribute {attr} is not in schema. {VALIDATOR_MSG}") + if array.dtype != schema.dtype[attr]: + raise Warning(f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}") + else: + if sub_data.dtype != schema.dtype: + raise Warning("Data type does not match schema. {VALIDATOR_MSG}") + def __del__(self): pgc.destroy_dataset_mutable(self._mutable_dataset) diff --git a/tests/unit/test_data_handling.py b/tests/unit/test_data_handling.py index 0fe8006a0..e6aceff35 100644 --- a/tests/unit/test_data_handling.py +++ b/tests/unit/test_data_handling.py @@ -2,12 +2,16 @@ # # SPDX-License-Identifier: MPL-2.0 +import warnings + import numpy as np import pytest from power_grid_model._utils import is_columnar from power_grid_model.core.data_handling import create_output_data from power_grid_model.core.dataset_definitions import ComponentType as CT, DatasetType as DT +from power_grid_model.core.power_grid_core import VoidPtr +from power_grid_model.core.power_grid_dataset import CMutableDataset from power_grid_model.core.power_grid_meta import initialize_array @@ -69,3 +73,35 @@ def test_create_output_data(output_component_types, is_batch, expected): else: assert actual[comp].keys() == expected[comp].keys() assert all(actual[comp][attr].dtype == expected[comp][attr].dtype for attr in expected[comp]) + + +def test_dtype_compatibility_check_normal(): + nodes = initialize_array(DT.sym_output, CT.node, (1, 2)) + nodes_ptr = nodes.ctypes.data_as(VoidPtr) + + data = {CT.node: nodes} + mutable_dataset = CMutableDataset(data, DT.sym_output) + buffer_views = mutable_dataset.get_buffer_views() + + assert buffer_views[0].data.value == nodes_ptr.value + + +def test_dtype_compatibility_check_compatible(): + nodes = initialize_array(DT.sym_output, CT.node, 4) + nodes = nodes[::2] + nodes_ptr = nodes.ctypes.data_as(VoidPtr) + + data = {CT.node: nodes} + with warnings.catch_warnings(): + warnings.simplefilter("error") + mutable_dataset = CMutableDataset(data, DT.sym_output) + buffer_views = mutable_dataset.get_buffer_views() + + assert buffer_views[0].data.value != nodes_ptr.value + + +def test_dtype_compatibility_check__error(): + nodes = initialize_array(DT.sym_output, CT.node, (1, 2)) + data = {CT.node: nodes.astype(nodes.dtype.newbyteorder("S"))} + with pytest.raises(Warning): + CMutableDataset(data, DT.sym_output) From e65dccf733a2f95d6e7a842f9a4c528f7359a484 Mon Sep 17 00:00:00 2001 From: Santiago Figueroa Manrique Date: Fri, 11 Oct 2024 13:56:41 +0200 Subject: [PATCH 04/15] columnar data integration tests Signed-off-by: Santiago Figueroa Manrique --- tests/unit/test_power_grid_model.py | 169 +++++++++++++++++++--------- 1 file changed, 115 insertions(+), 54 deletions(-) diff --git a/tests/unit/test_power_grid_model.py b/tests/unit/test_power_grid_model.py index a5c045c65..b4d260886 100644 --- a/tests/unit/test_power_grid_model.py +++ b/tests/unit/test_power_grid_model.py @@ -7,8 +7,10 @@ import numpy as np import pytest -from power_grid_model import ComponentType, DatasetType, PowerGridModel, initialize_array +from power_grid_model._utils import compatibility_convert_row_columnar_dataset +from power_grid_model import ComponentAttributeFilterOptions, ComponentType, DatasetType, PowerGridModel, initialize_array from power_grid_model.errors import InvalidCalculationMethod, IterationDiverge, PowerGridBatchError, PowerGridError +from power_grid_model.utils import get_dataset_scenario from power_grid_model.validation import assert_valid_input_data from .utils import compare_result, convert_python_to_numpy @@ -32,57 +34,113 @@ """ -# test data -INPUT = { - ComponentType.node: [{"id": 0, "u_rated": 100.0}], - ComponentType.source: [{"id": 1, "node": 0, "status": 1, "u_ref": 1.0, "sk": 1000.0, "rx_ratio": 0.0}], - ComponentType.sym_load: [{"id": 2, "node": 0, "status": 1, "type": 2, "p_specified": 0.0, "q_specified": 500.0}], -} +@pytest.fixture +def input_row(): + node = initialize_array(DatasetType.input, ComponentType.node, 1) + node["id"] = 0 + node["u_rated"] = 100.0 + + source = initialize_array(DatasetType.input, ComponentType.source, 1) + source["id"] = 1 + source["node"] = 0 + source["status"] = 1 + source["u_ref"] = 1.0 + source["sk"] = 1000.0 + source["rx_ratio"] = 0.0 + + sym_load = initialize_array(DatasetType.input, ComponentType.sym_load, 1) + sym_load["id"] = 2 + sym_load["node"] = 0 + sym_load["status"] = 1 + sym_load["type"] = 2 + sym_load["p_specified"] = 0.0 + sym_load["q_specified"] = 500.0 -SYM_OUTPUT = {ComponentType.node: [{"id": 0, "u": 50.0, "u_pu": 0.5, "u_angle": 0.0}]} + return { + ComponentType.node: node, + ComponentType.source: source, + ComponentType.sym_load: sym_load, + } -SYM_OUTPUT_BATCH = [ - {ComponentType.node: [{"id": 0, "u": 40.0, "u_pu": 0.4, "u_angle": 0.0}]}, - {ComponentType.node: [{"id": 0, "u": 70.0, "u_pu": 0.7, "u_angle": 0.0}]}, -] -UPDATE_BATCH = [ - {ComponentType.source: [{"id": 1, "u_ref": 0.5}], ComponentType.sym_load: [{"id": 2, "q_specified": 100.0}]}, - {ComponentType.sym_load: [{"id": 2, "q_specified": 300.0}]}, -] +@pytest.fixture +def input_col(input_row): + return compatibility_convert_row_columnar_dataset(input_row, ComponentAttributeFilterOptions.RELEVANT, DatasetType.input) +@pytest.fixture(params=["input_row", "input_col"]) +def input(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sym_output(): + node = initialize_array(DatasetType.sym_output, ComponentType.node, 1) + node["id"] = 0 + node["u"] = 50.0 + node["u_pu"] = 0.5 + node["u_angle"] = 0.0 + + return {ComponentType.node: node} + + @pytest.fixture -def case_data(): +def update_batch_row(): + source = initialize_array(DatasetType.update, ComponentType.source, 1) + source["id"] = 1 + source["u_ref"] = 0.5 + + sym_load = initialize_array(DatasetType.update, ComponentType.sym_load, 2) + sym_load["id"] = [2, 2] + sym_load["q_specified"] = [100.0, 300.0] + return { - "input": convert_python_to_numpy(INPUT, DatasetType.input), - "output": convert_python_to_numpy(SYM_OUTPUT, DatasetType.sym_output), - "update_batch": convert_python_to_numpy(UPDATE_BATCH, DatasetType.update), - "output_batch": convert_python_to_numpy(SYM_OUTPUT_BATCH, DatasetType.sym_output), + ComponentType.source: { + "data": source, + "indptr": np.array([0, 1, 1]), + }, + ComponentType.sym_load: { + "data": sym_load, + "indptr": np.array([0, 1, 2]), + } } @pytest.fixture -def model(case_data): - return PowerGridModel(input_data=case_data["input"]) +def update_batch_col(update_batch_row): + return compatibility_convert_row_columnar_dataset(update_batch_row, ComponentAttributeFilterOptions.RELEVANT, DatasetType.update) + + +@pytest.fixture(params=["update_batch_row", "update_batch_col"]) +def update_batch(request): + return request.getfixturevalue(request.param) + + +@pytest.fixture +def sym_output_batch(): + node = initialize_array(DatasetType.sym_output, ComponentType.node, (2, 1)) + node["id"] = [[0], [0]] + node["u"] = [[40.0], [70.0]] + node["u_pu"] = [[0.4], [0.7]] + node["u_angle"] = [[0.0], [0.0]] + return { ComponentType.node: node, } -def test_simple_power_flow(model: PowerGridModel, case_data): + +@pytest.fixture +def model(input): + return PowerGridModel(input) + + +def test_simple_power_flow(model: PowerGridModel, sym_output): result = model.calculate_power_flow() - compare_result(result, case_data["output"], rtol=0.0, atol=1e-8) + compare_result(result, sym_output, rtol=0.0, atol=1e-8) -def test_simple_update(model: PowerGridModel, case_data): - update_batch = case_data["update_batch"] - source_indptr = update_batch[ComponentType.source]["indptr"] - source_update = update_batch[ComponentType.source]["data"] - update_data = { - ComponentType.source: source_update[source_indptr[0] : source_indptr[1]], - ComponentType.sym_load: update_batch[ComponentType.sym_load][0, :], - } - model.update(update_data=update_data) - expected_result = {ComponentType.node: case_data["output_batch"][ComponentType.node][0, :]} +def test_simple_permanent_update(model: PowerGridModel, update_batch, sym_output_batch): # error if permanent update is not single scenario + model.update(update_data=update_batch) # single permanent model update result = model.calculate_power_flow() + expected_result = get_dataset_scenario(sym_output_batch, 0) compare_result(result, expected_result, rtol=0.0, atol=1e-8) @@ -92,12 +150,15 @@ def test_update_error(model: PowerGridModel): update_data = {ComponentType.sym_load: load_update} with pytest.raises(PowerGridError, match="The id cannot be found:"): model.update(update_data=update_data) + update_data_col = compatibility_convert_row_columnar_dataset(update_data, ComponentAttributeFilterOptions.RELEVANT, DatasetType.update) + with pytest.raises(PowerGridError, match="The id cannot be found:"): + model.update(update_data=update_data_col) -def test_copy_model(model: PowerGridModel, case_data): +def test_copy_model(model: PowerGridModel, sym_output): model_2 = copy(model) result = model_2.calculate_power_flow() - compare_result(result, case_data["output"], rtol=0.0, atol=1e-8) + compare_result(result, sym_output, rtol=0.0, atol=1e-8) def test_get_indexer(model: PowerGridModel): @@ -107,15 +168,15 @@ def test_get_indexer(model: PowerGridModel): assert np.allclose(expected_indexer, indexer) -def test_batch_power_flow(model: PowerGridModel, case_data): - result = model.calculate_power_flow(update_data=case_data["update_batch"]) - compare_result(result, case_data["output_batch"], rtol=0.0, atol=1e-8) +def test_batch_power_flow(model: PowerGridModel, update_batch, sym_output_batch): + result = model.calculate_power_flow(update_data=update_batch) + compare_result(result, sym_output_batch, rtol=0.0, atol=1e-8) -def test_construction_error(case_data): - case_data["input"][ComponentType.sym_load]["id"] = 0 +def test_construction_error(input): + input[ComponentType.sym_load]["id"][0] = 0 with pytest.raises(PowerGridError, match="Conflicting id detected:"): - PowerGridModel(case_data["input"]) + PowerGridModel(input) def test_single_calculation_error(model: PowerGridModel): @@ -129,31 +190,31 @@ def test_single_calculation_error(model: PowerGridModel): model.calculate_short_circuit(calculation_method=calculation_method) -def test_batch_calculation_error(model: PowerGridModel, case_data): +def test_batch_calculation_error(model: PowerGridModel, update_batch): # wrong id - case_data["update_batch"][ComponentType.sym_load]["id"][1, 0] = 5 + update_batch[ComponentType.sym_load]["data"]["id"][1] = 5 # with error with pytest.raises(PowerGridBatchError) as e: - model.calculate_power_flow(update_data=case_data["update_batch"]) + model.calculate_power_flow(update_data=update_batch) error = e.value - np.allclose(error.failed_scenarios, [1]) - np.allclose(error.succeeded_scenarios, [0]) + assert np.allclose(error.failed_scenarios, [1]) + assert np.allclose(error.succeeded_scenarios, [0]) assert "The id cannot be found:" in error.error_messages[0] -def test_batch_calculation_error_continue(model: PowerGridModel, case_data): +def test_batch_calculation_error_continue(model: PowerGridModel, update_batch, sym_output_batch): # wrong id - case_data["update_batch"][ComponentType.sym_load]["id"][1, 0] = 5 - result = model.calculate_power_flow(update_data=case_data["update_batch"], continue_on_batch_error=True) + update_batch[ComponentType.sym_load]["data"]["id"][1] = 5 + result = model.calculate_power_flow(update_data=update_batch, continue_on_batch_error=True) # assert error error = model.batch_error assert error is not None - np.allclose(error.failed_scenarios, [1]) - np.allclose(error.succeeded_scenarios, [0]) + assert np.allclose(error.failed_scenarios, [1]) + assert np.allclose(error.succeeded_scenarios, [0]) assert "The id cannot be found:" in error.error_messages[0] # assert value result for scenario 0 result = {ComponentType.node: result[ComponentType.node][error.succeeded_scenarios, :]} - expected_result = {ComponentType.node: case_data["output_batch"][ComponentType.node][error.succeeded_scenarios, :]} + expected_result = {ComponentType.node: sym_output_batch[ComponentType.node][error.succeeded_scenarios, :]} compare_result(result, expected_result, rtol=0.0, atol=1e-8) # general error before the batch with pytest.raises(PowerGridError, match="The calculation method is invalid for this calculation!"): From 52b23202809601aa563de9eb0af501846b29c37a Mon Sep 17 00:00:00 2001 From: Santiago Figueroa Manrique Date: Fri, 11 Oct 2024 14:00:15 +0200 Subject: [PATCH 05/15] formatting Signed-off-by: Santiago Figueroa Manrique --- tests/unit/test_power_grid_model.py | 36 +++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_power_grid_model.py b/tests/unit/test_power_grid_model.py index b4d260886..28fec9ca4 100644 --- a/tests/unit/test_power_grid_model.py +++ b/tests/unit/test_power_grid_model.py @@ -7,13 +7,19 @@ import numpy as np import pytest +from power_grid_model import ( + ComponentAttributeFilterOptions, + ComponentType, + DatasetType, + PowerGridModel, + initialize_array, +) from power_grid_model._utils import compatibility_convert_row_columnar_dataset -from power_grid_model import ComponentAttributeFilterOptions, ComponentType, DatasetType, PowerGridModel, initialize_array from power_grid_model.errors import InvalidCalculationMethod, IterationDiverge, PowerGridBatchError, PowerGridError from power_grid_model.utils import get_dataset_scenario from power_grid_model.validation import assert_valid_input_data -from .utils import compare_result, convert_python_to_numpy +from .utils import compare_result """ Testing network @@ -65,7 +71,9 @@ def input_row(): @pytest.fixture def input_col(input_row): - return compatibility_convert_row_columnar_dataset(input_row, ComponentAttributeFilterOptions.RELEVANT, DatasetType.input) + return compatibility_convert_row_columnar_dataset( + input_row, ComponentAttributeFilterOptions.RELEVANT, DatasetType.input + ) @pytest.fixture(params=["input_row", "input_col"]) @@ -82,7 +90,7 @@ def sym_output(): node["u_angle"] = 0.0 return {ComponentType.node: node} - + @pytest.fixture def update_batch_row(): @@ -102,13 +110,15 @@ def update_batch_row(): ComponentType.sym_load: { "data": sym_load, "indptr": np.array([0, 1, 2]), - } + }, } @pytest.fixture def update_batch_col(update_batch_row): - return compatibility_convert_row_columnar_dataset(update_batch_row, ComponentAttributeFilterOptions.RELEVANT, DatasetType.update) + return compatibility_convert_row_columnar_dataset( + update_batch_row, ComponentAttributeFilterOptions.RELEVANT, DatasetType.update + ) @pytest.fixture(params=["update_batch_row", "update_batch_col"]) @@ -124,7 +134,9 @@ def sym_output_batch(): node["u_pu"] = [[0.4], [0.7]] node["u_angle"] = [[0.0], [0.0]] - return { ComponentType.node: node, } + return { + ComponentType.node: node, + } @pytest.fixture @@ -137,8 +149,10 @@ def test_simple_power_flow(model: PowerGridModel, sym_output): compare_result(result, sym_output, rtol=0.0, atol=1e-8) -def test_simple_permanent_update(model: PowerGridModel, update_batch, sym_output_batch): # error if permanent update is not single scenario - model.update(update_data=update_batch) # single permanent model update +def test_simple_permanent_update( + model: PowerGridModel, update_batch, sym_output_batch +): # error if permanent update is not single scenario + model.update(update_data=update_batch) # single permanent model update result = model.calculate_power_flow() expected_result = get_dataset_scenario(sym_output_batch, 0) compare_result(result, expected_result, rtol=0.0, atol=1e-8) @@ -150,7 +164,9 @@ def test_update_error(model: PowerGridModel): update_data = {ComponentType.sym_load: load_update} with pytest.raises(PowerGridError, match="The id cannot be found:"): model.update(update_data=update_data) - update_data_col = compatibility_convert_row_columnar_dataset(update_data, ComponentAttributeFilterOptions.RELEVANT, DatasetType.update) + update_data_col = compatibility_convert_row_columnar_dataset( + update_data, ComponentAttributeFilterOptions.RELEVANT, DatasetType.update + ) with pytest.raises(PowerGridError, match="The id cannot be found:"): model.update(update_data=update_data_col) From 5a0763612fcc44908d1c69f0dfda8f7f42443100 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 11 Oct 2024 15:32:23 +0200 Subject: [PATCH 06/15] change shape and warning check Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/power_grid_dataset.py | 17 ++++++++++++----- tests/unit/test_data_handling.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index 3ccee3c1b..84125b29d 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -6,6 +6,7 @@ Power grid model raw dataset handler """ +import warnings from typing import Any, Mapping, Optional from power_grid_model._utils import ( @@ -277,13 +278,19 @@ def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentM sub_data = _extract_data_from_component_data(data) if is_columnar(data): for attr, array in sub_data.items(): - if schema.dtype.names is None or attr not in schema.dtype.names: - raise Warning(f"Attribute {attr} is not in schema. {VALIDATOR_MSG}") - if array.dtype != schema.dtype[attr]: - raise Warning(f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}") + if ( + schema.dtype.names is None + or attr not in schema.dtype.names + or (schema.dtype[attr].shape == (3,) and array.shape[-1] != 3) + ): + raise TypeError("Given data has a different schema than supported.") + if array.dtype.base != schema.dtype[attr].base: + warnings.warn( + f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}", DeprecationWarning + ) else: if sub_data.dtype != schema.dtype: - raise Warning("Data type does not match schema. {VALIDATOR_MSG}") + warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) def __del__(self): pgc.destroy_dataset_mutable(self._mutable_dataset) diff --git a/tests/unit/test_data_handling.py b/tests/unit/test_data_handling.py index e6aceff35..108e02445 100644 --- a/tests/unit/test_data_handling.py +++ b/tests/unit/test_data_handling.py @@ -103,5 +103,5 @@ def test_dtype_compatibility_check_compatible(): def test_dtype_compatibility_check__error(): nodes = initialize_array(DT.sym_output, CT.node, (1, 2)) data = {CT.node: nodes.astype(nodes.dtype.newbyteorder("S"))} - with pytest.raises(Warning): + with pytest.warns(DeprecationWarning): CMutableDataset(data, DT.sym_output) From 7ad224891c4b7e3a64878c9e8a3f117ffad09d52 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:49:01 +0200 Subject: [PATCH 07/15] Update src/power_grid_model/core/power_grid_dataset.py Co-authored-by: Martijn Govers Signed-off-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> --- src/power_grid_model/core/power_grid_dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index 84125b29d..46e12c861 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -288,8 +288,7 @@ def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentM warnings.warn( f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}", DeprecationWarning ) - else: - if sub_data.dtype != schema.dtype: + elif sub_data.dtype != schema.dtype: warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) def __del__(self): From f90cf37da600762552b6b028e5bf5cf1beb5aab0 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Fri, 11 Oct 2024 16:09:03 +0200 Subject: [PATCH 08/15] format changes and doc Signed-off-by: Nitish Bharambe --- docs/api_reference/python-api-reference.md | 1 + src/power_grid_model/core/power_grid_dataset.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api_reference/python-api-reference.md b/docs/api_reference/python-api-reference.md index 6257888d7..3a97a8865 100644 --- a/docs/api_reference/python-api-reference.md +++ b/docs/api_reference/python-api-reference.md @@ -11,6 +11,7 @@ SPDX-License-Identifier: MPL-2.0 .. autoclass:: power_grid_model.PowerGridModel :show-inheritance: .. autofunction:: power_grid_model.initialize_array +.. autoclass:: power_grid_model.power_grid_meta_data ``` ## enum diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index 46e12c861..cd6278c10 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -289,7 +289,7 @@ def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentM f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}", DeprecationWarning ) elif sub_data.dtype != schema.dtype: - warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) + warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) def __del__(self): pgc.destroy_dataset_mutable(self._mutable_dataset) From a4ef4018462ff2a40dec25f014548fee4166067e Mon Sep 17 00:00:00 2001 From: Nitish Bharambe Date: Sat, 12 Oct 2024 15:22:05 +0200 Subject: [PATCH 09/15] move to buffer handling Signed-off-by: Nitish Bharambe --- src/power_grid_model/core/buffer_handling.py | 5 ++++ .../core/power_grid_dataset.py | 27 +------------------ 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/src/power_grid_model/core/buffer_handling.py b/src/power_grid_model/core/buffer_handling.py index b012c2333..5d6fe2700 100644 --- a/src/power_grid_model/core/buffer_handling.py +++ b/src/power_grid_model/core/buffer_handling.py @@ -6,6 +6,7 @@ Power grid model buffer handler """ +import warnings from dataclasses import dataclass from typing import cast @@ -83,6 +84,8 @@ def _get_raw_data_view(data: np.ndarray, dtype: np.dtype) -> VoidPtr: Returns: a raw view on the data set. """ + if data.dtype != dtype: + warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) return np.ascontiguousarray(data, dtype=dtype).ctypes.data_as(VoidPtr) @@ -115,6 +118,8 @@ def _get_raw_attribute_data_view(data: np.ndarray, schema: ComponentMetaData, at Returns: a raw view on the data set. """ + if schema.dtype[attribute].shape == (3,) and data.shape[-1] != 3: + raise TypeError("Given data has a different schema than supported.") return _get_raw_data_view(data, dtype=schema.dtype[attribute].base) diff --git a/src/power_grid_model/core/power_grid_dataset.py b/src/power_grid_model/core/power_grid_dataset.py index cd6278c10..05db79d4b 100644 --- a/src/power_grid_model/core/power_grid_dataset.py +++ b/src/power_grid_model/core/power_grid_dataset.py @@ -6,16 +6,9 @@ Power grid model raw dataset handler """ -import warnings from typing import Any, Mapping, Optional -from power_grid_model._utils import ( - _extract_data_from_component_data, - get_dataset_type, - is_columnar, - is_nan_or_equivalent, - process_data_filter, -) +from power_grid_model._utils import get_dataset_type, is_columnar, is_nan_or_equivalent, process_data_filter from power_grid_model.core.buffer_handling import ( BufferProperties, CAttributeBuffer, @@ -238,7 +231,6 @@ def _add_component_data(self, component: ComponentType, data: ComponentData, all return self._validate_properties(data, self._schema[component]) - self._validate_dtypes_compatibility(data, self._schema[component]) c_buffer = get_buffer_view(data, self._schema[component], self._is_batch, self._batch_size) self._buffer_views.append(c_buffer) self._register_buffer(component, c_buffer) @@ -274,23 +266,6 @@ def _validate_properties(self, data: ComponentData, schema: ComponentMetaData): if properties.batch_size != self._batch_size: raise ValueError(f"Dataset must have a consistent batch size across all components. {VALIDATOR_MSG}") - def _validate_dtypes_compatibility(self, data: ComponentData, schema: ComponentMetaData): - sub_data = _extract_data_from_component_data(data) - if is_columnar(data): - for attr, array in sub_data.items(): - if ( - schema.dtype.names is None - or attr not in schema.dtype.names - or (schema.dtype[attr].shape == (3,) and array.shape[-1] != 3) - ): - raise TypeError("Given data has a different schema than supported.") - if array.dtype.base != schema.dtype[attr].base: - warnings.warn( - f"Data type for attribute {attr} does not match schema. {VALIDATOR_MSG}", DeprecationWarning - ) - elif sub_data.dtype != schema.dtype: - warnings.warn("Data type does not match schema. {VALIDATOR_MSG}", DeprecationWarning) - def __del__(self): pgc.destroy_dataset_mutable(self._mutable_dataset) From 4fcb6a1c7a13a4b56a724bbdb2541ea097bb0db8 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:20:50 +0200 Subject: [PATCH 10/15] Update src/power_grid_model/core/buffer_handling.py Signed-off-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> --- src/power_grid_model/core/buffer_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/power_grid_model/core/buffer_handling.py b/src/power_grid_model/core/buffer_handling.py index 5d6fe2700..9a7c7e32a 100644 --- a/src/power_grid_model/core/buffer_handling.py +++ b/src/power_grid_model/core/buffer_handling.py @@ -119,7 +119,7 @@ def _get_raw_attribute_data_view(data: np.ndarray, schema: ComponentMetaData, at a raw view on the data set. """ if schema.dtype[attribute].shape == (3,) and data.shape[-1] != 3: - raise TypeError("Given data has a different schema than supported.") + raise ValueError("Given data has a different schema than supported.") return _get_raw_data_view(data, dtype=schema.dtype[attribute].base) From 443f10f1b0d9b23f829a4971cc7ccd01afcf28a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 07:36:08 +0000 Subject: [PATCH 11/15] Bump pypa/cibuildwheel from 2.21.2 to 2.21.3 Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.21.2 to 2.21.3. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.21.2...v2.21.3) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d736c4885..53764bab0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -219,7 +219,7 @@ jobs: xcode-version: latest-stable - name: Build wheels - uses: pypa/cibuildwheel@v2.21.2 + uses: pypa/cibuildwheel@v2.21.3 # GitHub Actions specific build parameters env: # pass GitHub runner info into Linux container From 2737a55e981c58a31ce8be75a54477b5870248e5 Mon Sep 17 00:00:00 2001 From: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:41:42 +0200 Subject: [PATCH 12/15] Update docs/api_reference/python-api-reference.md Signed-off-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> --- docs/api_reference/python-api-reference.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/api_reference/python-api-reference.md b/docs/api_reference/python-api-reference.md index 3a97a8865..6257888d7 100644 --- a/docs/api_reference/python-api-reference.md +++ b/docs/api_reference/python-api-reference.md @@ -11,7 +11,6 @@ SPDX-License-Identifier: MPL-2.0 .. autoclass:: power_grid_model.PowerGridModel :show-inheritance: .. autofunction:: power_grid_model.initialize_array -.. autoclass:: power_grid_model.power_grid_meta_data ``` ## enum From 3f2b6f3a2d3ac6d072d3c54f0fb804d48c11a593 Mon Sep 17 00:00:00 2001 From: Santiago Figueroa Manrique Date: Mon, 14 Oct 2024 11:23:11 +0200 Subject: [PATCH 13/15] Addressed comments Signed-off-by: Santiago Figueroa Manrique --- tests/unit/test_power_grid_model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_power_grid_model.py b/tests/unit/test_power_grid_model.py index 28fec9ca4..bbe197a8f 100644 --- a/tests/unit/test_power_grid_model.py +++ b/tests/unit/test_power_grid_model.py @@ -181,7 +181,7 @@ def test_get_indexer(model: PowerGridModel): ids = np.array([2, 2]) expected_indexer = np.array([0, 0]) indexer = model.get_indexer(ComponentType.sym_load, ids) - assert np.allclose(expected_indexer, indexer) + np.testing.assert_allclose(expected_indexer, indexer) def test_batch_power_flow(model: PowerGridModel, update_batch, sym_output_batch): @@ -213,8 +213,8 @@ def test_batch_calculation_error(model: PowerGridModel, update_batch): with pytest.raises(PowerGridBatchError) as e: model.calculate_power_flow(update_data=update_batch) error = e.value - assert np.allclose(error.failed_scenarios, [1]) - assert np.allclose(error.succeeded_scenarios, [0]) + np.testing.assert_allclose(error.failed_scenarios, [1]) + np.testing.assert_allclose(error.succeeded_scenarios, [0]) assert "The id cannot be found:" in error.error_messages[0] @@ -225,8 +225,8 @@ def test_batch_calculation_error_continue(model: PowerGridModel, update_batch, s # assert error error = model.batch_error assert error is not None - assert np.allclose(error.failed_scenarios, [1]) - assert np.allclose(error.succeeded_scenarios, [0]) + np.testing.assert_allclose(error.failed_scenarios, [1]) + np.testing.assert_allclose(error.succeeded_scenarios, [0]) assert "The id cannot be found:" in error.error_messages[0] # assert value result for scenario 0 result = {ComponentType.node: result[ComponentType.node][error.succeeded_scenarios, :]} From 4f7e4a4a916326372afc80a3868c40f8de24c993 Mon Sep 17 00:00:00 2001 From: Martijn Govers Date: Tue, 15 Oct 2024 08:08:04 +0200 Subject: [PATCH 14/15] Update .gitignore Signed-off-by: Martijn Govers Co-authored-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Signed-off-by: Martijn Govers --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index fa6330893..79cfd9f96 100644 --- a/.gitignore +++ b/.gitignore @@ -513,7 +513,6 @@ PYPI_VERSION # docs builds docs/make.bat docs/Makefile -docs/jupyter_execute/ # VSCode configurations # it is ignored by default so the user does not accidentally change the configuration From 4bb4bdff659a1ba4003527fe849b502129a84192 Mon Sep 17 00:00:00 2001 From: Martijn Govers Date: Tue, 15 Oct 2024 08:09:46 +0200 Subject: [PATCH 15/15] Update src/power_grid_model/typing.py Signed-off-by: Martijn Govers Co-authored-by: Nitish Bharambe <78108900+nitbharambe@users.noreply.github.com> Signed-off-by: Martijn Govers --- src/power_grid_model/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/power_grid_model/typing.py b/src/power_grid_model/typing.py index 763554c30..06e58443a 100644 --- a/src/power_grid_model/typing.py +++ b/src/power_grid_model/typing.py @@ -37,7 +37,7 @@ - A set of :class:`ComponentType` or `str` -- A list of :class:`ComponentType` +- A list of :class:`ComponentType` or `str` - A :class:`ComponentAttributeFilterOptions ` value