diff --git a/src/power_grid_model/validation/errors.py b/src/power_grid_model/validation/errors.py index 695c4b904..64c4a8d2e 100644 --- a/src/power_grid_model/validation/errors.py +++ b/src/power_grid_model/validation/errors.py @@ -467,3 +467,55 @@ class FaultPhaseError(MultiFieldValidationError): """ _message = "The fault phase is not applicable to the corresponding fault type for {n} {objects}." + + +class InvalidAssociatedEnumValueError(MultiFieldValidationError): + """ + The value is not a valid value in combination with the other specified attributes. + E.g. When a transformer tap regulator has a branch3 control side but regulates a transformer. + """ + + _message = "The combination of fields {field} results in invalid {enum} values for {n} {objects}." + enum: Union[Type[Enum], List[Type[Enum]]] + + def __init__( + self, + component: str, + fields: List[str], + ids: List[int], + enum: Union[Type[Enum], List[Type[Enum]]], + ): + """ + Args: + component: Component name + fields: List of field names + ids: List of component IDs (not row indices) + enum: The supported enum values + """ + super().__init__(component, fields, ids) + self.enum = enum + + @property + def enum_str(self) -> str: + """ + A string representation of the field to which this error applies. + """ + if isinstance(self.enum, list): + return ",".join(e.__name__ for e in self.enum) + + return self.enum.__name__ + + def __eq__(self, other): + return super().__eq__(other) and self.enum == other.enum + + +class UnsupportedTransformerRegulationError(MultiFieldValidationError): + """ + The control side of a branch regulator is not supported for the regulated object + """ + + _message = ( + "Unsupported control side for {n} {objects}. " + "The control side cannot be on the same side as the tap side. " + "The node at the control side must have the same or lower u_rated as the node at the tap side." + ) diff --git a/src/power_grid_model/validation/rules.py b/src/power_grid_model/validation/rules.py index f61298df7..8033d660b 100644 --- a/src/power_grid_model/validation/rules.py +++ b/src/power_grid_model/validation/rules.py @@ -46,6 +46,7 @@ FaultPhaseError, IdNotInDatasetError, InfinityError, + InvalidAssociatedEnumValueError, InvalidEnumValueError, InvalidIdError, MissingValueError, @@ -63,9 +64,17 @@ SameValueError, TransformerClockError, TwoValuesZeroError, + UnsupportedTransformerRegulationError, ValidationError, ) -from power_grid_model.validation.utils import eval_expression, nan_type, set_default_value +from power_grid_model.validation.utils import ( + eval_expression, + get_indexer, + get_mask, + get_valid_ids, + nan_type, + set_default_value, +) Error = TypeVar("Error", bound=ValidationError) CompError = TypeVar("CompError", bound=ComparisonError) @@ -481,7 +490,7 @@ def all_valid_enum_values( data (SingleDataset): The input/update data set for all components component (str): The component of interest field (str): The field of interest - enum (Type[Enum]): The enum type to validate against, or a list of such enum types + enum (Type[Enum] | List[Type[Enum]]): The enum type to validate against, or a list of such enum types Returns: A list containing zero or one InvalidEnumValueError, listing all ids where the value in the field of interest @@ -500,6 +509,48 @@ def all_valid_enum_values( return [] +def all_valid_associated_enum_values( # pylint: disable=too-many-arguments + data: SingleDataset, + component: str, + field: str, + ref_object_id_field: str, + ref_components: List[str], + enum: Union[Type[Enum], List[Type[Enum]]], + **filters: Any, +) -> List[InvalidAssociatedEnumValueError]: + """ + Args: + data (SingleDataset): The input/update data set for all components + component (str): The component of interest + field (str): The field of interest + ref_object_id_field (str): The field that contains the referenced component ids + ref_components (List[str]): The component or components in which we want to look for ids + enum (Type[Enum] | List[Type[Enum]]): The enum type to validate against, or a list of such enum types + **filters: One or more filters on the dataset. E.g. regulated_object="transformer". + + Returns: + A list containing zero or one InvalidAssociatedEnumValueError, listing all ids where the value in the field + of interest was not a valid value in the supplied enum type. + """ + enums: List[Type[Enum]] = enum if isinstance(enum, list) else [enum] + + valid_ids = get_valid_ids(data=data, ref_components=ref_components) + mask = np.logical_and( + get_mask(data=data, component=component, field=field, **filters), + np.isin(data[component][ref_object_id_field], valid_ids), + ) + + valid = {nan_type(component, field)} + for enum_type in enums: + valid.update(list(enum_type)) + + invalid = np.isin(data[component][field][mask], np.array(list(valid), dtype=np.int8), invert=True) + if invalid.any(): + ids = data[component]["id"][mask][invalid].flatten().tolist() + return [InvalidAssociatedEnumValueError(component, [field, ref_object_id_field], ids, enum)] + return [] + + def all_valid_ids( data: SingleDataset, component: str, field: str, ref_components: Union[str, List[str]], **filters: Any ) -> List[InvalidIdError]: @@ -518,30 +569,11 @@ def all_valid_ids( A list containing zero or one InvalidIdError, listing all ids where the value in the field of interest was not a valid object identifier. """ - # For convenience, ref_component may be a string and we'll convert it to a 'list' containing that string as it's - # single element. - if isinstance(ref_components, str): - ref_components = [ref_components] - - # Create a set of ids by chaining the ids of all ref_components - valid_ids = set() - for ref_component in ref_components: - if ref_component in data: - nan = nan_type(ref_component, "id") - if np.isnan(nan): - mask = ~np.isnan(data[ref_component]["id"]) - else: - mask = np.not_equal(data[ref_component]["id"], nan) - valid_ids.update(data[ref_component]["id"][mask]) - - # Apply the filters (e.g. to select only records with a certain MeasuredTerminalType) - values = data[component][field] - mask = np.ones(shape=values.shape, dtype=bool) - for filter_field, filter_value in filters.items(): - mask = np.logical_and(mask, data[component][filter_field] == filter_value) + valid_ids = get_valid_ids(data=data, ref_components=ref_components) + mask = get_mask(data=data, component=component, field=field, **filters) # Find any values that can't be found in the set of ids - invalid = np.logical_and(mask, np.isin(values, list(valid_ids), invert=True)) + invalid = np.logical_and(mask, np.isin(data[component][field], valid_ids, invert=True)) if invalid.any(): ids = data[component]["id"][invalid].flatten().tolist() return [InvalidIdError(component, field, ids, ref_components, filters)] @@ -830,3 +862,47 @@ def _fault_phase_supported(fault_type: FaultType, fault_phase: FaultPhase): ) ] return [] + + +def all_supported_tap_control_side( # pylint: disable=too-many-arguments + data: SingleDataset, + component: str, + control_side_field: str, + regulated_object_field: str, + tap_side_fields: List[Tuple[str, str]], + **filters: Any, +) -> List[UnsupportedTransformerRegulationError]: + """ + Args: + data (SingleDataset): The input/update data set for all components + component (str): The component of interest + control_side_field (str): The field of interest + regulated_object_field (str): The field that contains the regulated component ids + tap_side_fields (List[Tuple[str, str]]): The fields of interest per regulated component, + formatted as [(component_1, field_1), (component_2, field_2)] + **filters: One or more filters on the dataset. E.g. regulated_object="transformer". + + Returns: + A list containing zero or more InvalidAssociatedEnumValueErrors; listing all the ids + of components where the field of interest was invalid, given the referenced object's field. + """ + mask = get_mask(data=data, component=component, field=control_side_field, **filters) + values = data[component][control_side_field][mask] + + invalid = np.zeros_like(mask) + + for ref_component, ref_field in tap_side_fields: + indices = get_indexer(data[ref_component]["id"], data[component][regulated_object_field], default_value=-1) + found = indices != -1 + ref_comp_values = data[ref_component][ref_field][indices[found]] + invalid[found] = np.logical_or(invalid[found], values[found] == ref_comp_values) + + if invalid.any(): + return [ + UnsupportedTransformerRegulationError( + component=component, + fields=[control_side_field, regulated_object_field], + ids=data[component]["id"][invalid].flatten().tolist(), + ) + ] + return [] diff --git a/src/power_grid_model/validation/utils.py b/src/power_grid_model/validation/utils.py index 9d93767f6..3087aca75 100644 --- a/src/power_grid_model/validation/utils.py +++ b/src/power_grid_model/validation/utils.py @@ -6,7 +6,7 @@ Utilities used for validation. Only errors_to_string() is intended for end users. """ import re -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union import numpy as np @@ -168,26 +168,40 @@ def nan_type(component: str, field: str, data_type="input"): return power_grid_meta_data[data_type][component].nans[field] -def get_indexer(input_ids: np.ndarray, update_ids: np.ndarray) -> np.ndarray: +def get_indexer(source: np.ndarray, target: np.ndarray, default_value: Optional[int] = None) -> np.ndarray: """ - Given array of ids from input and update dataset. - Find the posision of each id in the update dataset in the context of input dataset. + Given array of values from a source and a target dataset. + Find the position of each value in the target dataset in the context of the source dataset. This is needed to update values in the dataset by id lookup. Internally this is done by sorting the input ids, then using binary search lookup. + E.g.: Find the position of each id in an update (target) dataset in the input (source) dataset + + >>> input_ids = [1, 2, 3, 4, 5] + >>> update_ids = [3] + >>> assert get_indexer(input_ids, update_ids) == np.array([2]) + Args: - input_ids: array of ids in the input dataset - update_ids: array of ids in the update dataset + source: array of values in the source dataset + target: array of values in the target dataset + default_value: Optional. the default index to provide for target values not in source Returns: - np.ndarray: array of positions of the ids from update dataset in the input dataset - the following should hold - input_ids[result] == update_ids + np.ndarray: array of positions of the values from target dataset in the source dataset + if default_value is None, (source[result] == target) + else, ((source[result] == target) | (source[result] == default_value)) + + Raises: + IndexError: if default_value is None and there were values in target that were not in source """ - permutation_sort = np.argsort(input_ids) # complexity O(N_input * logN_input) - return permutation_sort[ - np.searchsorted(input_ids, update_ids, sorter=permutation_sort) - ] # complexity O(N_update * logN_input) + permutation_sort = np.argsort(source) # complexity O(N_input * logN_input) + indices = np.searchsorted(source, target, sorter=permutation_sort) # complexity O(N_update * logN_input) + + if default_value is None: + return permutation_sort[indices] + + clipped_indices = np.take(permutation_sort, indices, mode="clip") + return np.where(source[clipped_indices] == target, permutation_sort[clipped_indices], default_value) def set_default_value(data: SingleDataset, component: str, field: str, default_value: Union[int, float, np.ndarray]): @@ -214,3 +228,55 @@ def set_default_value(data: SingleDataset, component: str, field: str, default_v data[component][field][mask] = default_value[mask] else: data[component][field][mask] = default_value + + +def get_valid_ids(data: SingleDataset, ref_components: Union[str, List[str]]) -> List[int]: + """ + This function returns the valid IDs specified by all ref_components + + Args: + data: The input/update data set for all components + ref_components: The component or components in which we want to look for ids + + Returns: + List[int]: the list of valid IDs + """ + # For convenience, ref_component may be a string and we'll convert it to a 'list' containing that string as it's + # single element. + if isinstance(ref_components, str): + ref_components = [ref_components] + + # Create a set of ids by chaining the ids of all ref_components + valid_ids = set() + for ref_component in ref_components: + if ref_component in data: + nan = nan_type(ref_component, "id") + if np.isnan(nan): + mask = ~np.isnan(data[ref_component]["id"]) + else: + mask = np.not_equal(data[ref_component]["id"], nan) + valid_ids.update(data[ref_component]["id"][mask]) + + return list(valid_ids) + + +def get_mask(data: SingleDataset, component: str, field: str, **filters: Any) -> np.ndarray: + """ + Get a mask based on the specified filters. E.g. measured_terminal_type=MeasuredTerminalType.source. + + Args: + data: The input/update data set for all components + component: The component of interest + field: The field of interest + ref_components: The component or components in which we want to look for ids + **filters: One or more filters on the dataset. E. + + Returns: + np.ndarray: the mask + """ + values = data[component][field] + mask = np.ones(shape=values.shape, dtype=bool) + for filter_field, filter_value in filters.items(): + mask = np.logical_and(mask, data[component][filter_field] == filter_value) + + return mask diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 0536af0d9..b9a5e1166 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -48,7 +48,9 @@ all_less_than, all_not_two_values_equal, all_not_two_values_zero, + all_supported_tap_control_side, all_unique, + all_valid_associated_enum_values, all_valid_clocks, all_valid_enum_values, all_valid_fault_phases, @@ -833,8 +835,26 @@ def validate_transformer_tap_regulator(data: SingleDataset) -> List[ValidationEr errors = validate_regulator(data, "transformer_tap_regulator") errors += all_boolean(data, "transformer_tap_regulator", "status") errors += all_valid_enum_values(data, "transformer_tap_regulator", "control_side", [BranchSide, Branch3Side]) + errors += all_valid_associated_enum_values( + data, "transformer_tap_regulator", "control_side", "regulated_object", ["transformer"], [BranchSide] + ) + errors += all_valid_associated_enum_values( + data, + "transformer_tap_regulator", + "control_side", + "regulated_object", + ["three_winding_transformer"], + [Branch3Side], + ) errors += all_greater_than_or_equal_to_zero(data, "transformer_tap_regulator", "u_set") errors += all_greater_than_zero(data, "transformer_tap_regulator", "u_band") errors += all_greater_than_or_equal_to_zero(data, "transformer_tap_regulator", "line_drop_compensation_r", 0.0) errors += all_greater_than_or_equal_to_zero(data, "transformer_tap_regulator", "line_drop_compensation_x", 0.0) + errors += all_supported_tap_control_side( + data, + "transformer_tap_regulator", + "control_side", + "regulated_object", + [("transformer", "tap_side"), ("three_winding_transformer", "tap_side")], + ) return errors diff --git a/tests/unit/validation/test_errors.py b/tests/unit/validation/test_errors.py index df03be8ef..5c0d87968 100644 --- a/tests/unit/validation/test_errors.py +++ b/tests/unit/validation/test_errors.py @@ -8,6 +8,7 @@ from power_grid_model.validation.errors import ( ComparisonError, + InvalidAssociatedEnumValueError, InvalidEnumValueError, InvalidIdError, MultiComponentValidationError, @@ -111,3 +112,17 @@ def test_error_context_tuple_ids(): error = MultiComponentValidationError(fields=[("a", "x"), ("b", "y")], ids=[("a", 1), ("b", 2), ("a", 3)]) context = error.get_context(id_lookup={1: "Victor", 3: "Whiskey"}) assert context["ids"] == {("a", 1): "Victor", ("b", 2): None, ("a", 3): "Whiskey"} + + +def test_invalid_associated_enum_value_error(): + class CustomType(IntEnum): + pass + + error = InvalidAssociatedEnumValueError(component="foo", fields=["bar", "baz"], ids=[1, 2], enum=[CustomType]) + assert error.component == "foo" + assert error.field == ["bar", "baz"] + assert error.ids == [1, 2] + assert len(error.enum) == 1 + for actual, expected in zip(error.enum, [CustomType]): + assert actual is expected + assert str(error) == "The combination of fields 'bar' and 'baz' results in invalid CustomType values for 2 foos." diff --git a/tests/unit/validation/test_input_validation.py b/tests/unit/validation/test_input_validation.py index c5b3e7bf2..f1e809e9f 100644 --- a/tests/unit/validation/test_input_validation.py +++ b/tests/unit/validation/test_input_validation.py @@ -12,6 +12,7 @@ from power_grid_model.validation import validate_input_data from power_grid_model.validation.errors import ( FaultPhaseError, + InvalidAssociatedEnumValueError, InvalidEnumValueError, InvalidIdError, MultiComponentNotUniqueError, @@ -24,6 +25,7 @@ NotLessThanError, NotUniqueError, TwoValuesZeroError, + UnsupportedTransformerRegulationError, ) from power_grid_model.validation.utils import nan_type @@ -156,15 +158,15 @@ def input_data() -> Dict[str, np.ndarray]: three_winding_transformer["pk_13_max"] = [-40, nan_type("three_winding_transformer", "pk_12_max"), 40, 50] three_winding_transformer["pk_23_max"] = [-120, nan_type("three_winding_transformer", "pk_12_max"), 40, 30] - transformer_tap_regulator = initialize_array("input", "transformer_tap_regulator", 4) - transformer_tap_regulator["id"] = [51, 52, 53, 1] - transformer_tap_regulator["status"] = [0, -1, 2, 5] - transformer_tap_regulator["regulated_object"] = [14, 15, 28, 2] - transformer_tap_regulator["control_side"] = [1, 2, 0, 60] - transformer_tap_regulator["u_set"] = [100, -100, 100, 100] - transformer_tap_regulator["u_band"] = [100, -4, 100, 0] - transformer_tap_regulator["line_drop_compensation_r"] = [0.0, -1.0, 1.0, 2.0] - transformer_tap_regulator["line_drop_compensation_x"] = [0.0, 4.0, 2.0, -4.0] + transformer_tap_regulator = initialize_array("input", "transformer_tap_regulator", 5) + transformer_tap_regulator["id"] = [51, 52, 53, 54, 1] + transformer_tap_regulator["status"] = [0, -1, 2, 1, 5] + transformer_tap_regulator["regulated_object"] = [14, 15, 28, 14, 2] + transformer_tap_regulator["control_side"] = [1, 2, 0, 0, 60] + transformer_tap_regulator["u_set"] = [100, -100, 100, 100, 100] + transformer_tap_regulator["u_band"] = [100, -4, 100, 100, 0] + transformer_tap_regulator["line_drop_compensation_r"] = [0.0, -1.0, 1.0, 0.0, 2.0] + transformer_tap_regulator["line_drop_compensation_x"] = [0.0, 4.0, 2.0, 0.0, -4.0] source = initialize_array("input", "source", 3) source["id"] = [16, 17, 1] @@ -576,8 +578,12 @@ def test_validate_input_data_transformer_tap_regulator(input_data): InvalidEnumValueError("transformer_tap_regulator", "control_side", [1], [BranchSide, Branch3Side]) in validation_errors ) - # TODO (nbharambe) Add control side error after it is included - # assert InvalidControlSideError("transformer_tap_regulator", "control_side", [52], BranchSide) in validation_errors + assert ( + InvalidAssociatedEnumValueError( + "transformer_tap_regulator", ["control_side", "regulated_object"], [52], [BranchSide] + ) + in validation_errors + ) assert NotGreaterOrEqualError("transformer_tap_regulator", "u_set", [52], 0.0) in validation_errors assert NotGreaterThanError("transformer_tap_regulator", "u_band", [52, 1], 0.0) in validation_errors assert ( @@ -586,6 +592,10 @@ def test_validate_input_data_transformer_tap_regulator(input_data): assert ( NotGreaterOrEqualError("transformer_tap_regulator", "line_drop_compensation_x", [1], 0.0) in validation_errors ) + assert ( + UnsupportedTransformerRegulationError("transformer_tap_regulator", ["control_side", "regulated_object"], [54]) + in validation_errors + ) def test_fault(input_data): diff --git a/tests/unit/validation/test_validation_functions.py b/tests/unit/validation/test_validation_functions.py index 3e07b13d7..7b2d4f86a 100644 --- a/tests/unit/validation/test_validation_functions.py +++ b/tests/unit/validation/test_validation_functions.py @@ -10,16 +10,19 @@ import pytest from power_grid_model import CalculationType, LoadGenType, MeasuredTerminalType, initialize_array, power_grid_meta_data -from power_grid_model.enum import CalculationType, FaultPhase, FaultType +from power_grid_model.enum import Branch3Side, BranchSide, CalculationType, FaultType, TapChangingStrategy from power_grid_model.validation import assert_valid_input_data from power_grid_model.validation.errors import ( IdNotInDatasetError, InfinityError, + InvalidAssociatedEnumValueError, + InvalidEnumValueError, InvalidIdError, MissingValueError, MultiComponentNotUniqueError, MultiFieldValidationError, NotUniqueError, + UnsupportedTransformerRegulationError, ) from power_grid_model.validation.validation import ( assert_valid_data_structure, @@ -717,3 +720,76 @@ def test_power_sigma_or_p_q_sigma(): } assert_valid_input_data(input_data=input_data, calculation_type=CalculationType.state_estimation) + + +@patch("power_grid_model.validation.validation.validate_transformer", new=MagicMock(return_value=[])) +@patch("power_grid_model.validation.validation.validate_three_winding_transformer", new=MagicMock(return_value=[])) +def test_validate_values__tap_regulator_control_side(): + # Create valid transformer + transformer = initialize_array("input", "transformer", 4) + transformer["id"] = [0, 1, 2, 3] + transformer["tap_side"] = [BranchSide.from_side, BranchSide.from_side, BranchSide.from_side, BranchSide.from_side] + + # Create valid three winding transformer + three_winding_transformer = initialize_array("input", "three_winding_transformer", 3) + three_winding_transformer["id"] = [4, 5, 6] + three_winding_transformer["tap_side"] = [Branch3Side.side_1, Branch3Side.side_1, Branch3Side.side_1] + + # Create invalid regulator + transformer_tap_regulator = initialize_array("input", "transformer_tap_regulator", 7) + transformer_tap_regulator["id"] = np.arange(7, 14) + transformer_tap_regulator["status"] = 1 + transformer_tap_regulator["regulated_object"] = np.arange(7) + transformer_tap_regulator["control_side"] = [ + BranchSide.to_side, # OK + BranchSide.from_side, # control side is same as tap side (unsupported) + Branch3Side.side_3, # branch3 provided but it is a 2-winding transformer (invalid) + 10, # control side entirely out of range (invalid) + Branch3Side.side_3, # OK + Branch3Side.side_1, # control side is same as tap side (unsupported) + 10, # control side entirely out of range (invalid) + ] + + input_data = { + "transformer": transformer, + "three_winding_transformer": three_winding_transformer, + "transformer_tap_regulator": transformer_tap_regulator, + } + all_errors = validate_values(input_data) + power_flow_errors = validate_values(input_data, calculation_type=CalculationType.power_flow) + state_estimation_errors = validate_values(input_data, calculation_type=CalculationType.state_estimation) + + assert power_flow_errors == all_errors + assert not state_estimation_errors + + assert len(all_errors) == 4 + assert ( + InvalidEnumValueError("transformer_tap_regulator", "control_side", [10, 13], [BranchSide, Branch3Side]) + in all_errors + ) + assert ( + InvalidAssociatedEnumValueError( + "transformer_tap_regulator", + ["control_side", "regulated_object"], + [9, 10], + [BranchSide], + ) + in all_errors + ) + assert ( + InvalidAssociatedEnumValueError( + "transformer_tap_regulator", + ["control_side", "regulated_object"], + [13], + [Branch3Side], + ) + in all_errors + ) + assert ( + UnsupportedTransformerRegulationError( + "transformer_tap_regulator", + ["control_side", "regulated_object"], + [8, 12], + ) + in all_errors + )