Merge pull request #86 from Deltares/feature/DEI-54-summarize-warnings

Feature/dei 54 summarize warnings

decoimpact/business/entities/rule_processor.py

@@ -245,8 +245,26 @@ def _process_by_cell(
         np_array = input_variable.to_numpy()
         result_variable = _np.zeros_like(np_array)
 
+        # define variables to count value exceedings (for some rules): min and max
+        warning_counter = [0, 0]
+        warning_counter_total = [0, 0]
+
+        # execute rule and gather warnings for exceeded values (for some rules)
         for indices, value in _np.ndenumerate(np_array):
-            result_variable[indices] = rule.execute(value, logger)
+            result_variable[indices], warning_counter = rule.execute(value, logger)
+            # update total counter for both min and max
+            warning_counter_total[0] += warning_counter[0]
+            warning_counter_total[1] += warning_counter[1]
+
+        # show warnings values outside range (for some rules):
+        if warning_counter_total[0] > 0:
+            logger.log_warning(
+                f"value less than min: {warning_counter_total[0]} occurence(s)"
+            )
+        if warning_counter_total[1] > 0:
+            logger.log_warning(
+                f"value greater than max: {warning_counter_total[1]} occurence(s)"
+            )
 
         # use copy to get the same dimensions as the
         # original input variable

decoimpact/business/entities/rules/response_curve_rule.py

@@ -25,7 +25,6 @@ def __init__(
         output_variable_name="output",
         description: str = "",
     ):
-
         super().__init__(name, [input_variable_name], output_variable_name, description)
 
         self._input_values = _np.array(input_values)
@@ -62,18 +61,22 @@ def execute(self, value: float, logger: ILogger):
 
         Returns:
             float: response corresponding to value to classify
+            int[]: number of warnings less than minimum and greater than maximum
         """
 
         values_input = self._input_values
         values_output = self._output_values
+        warning_counter = [0, 0]
 
         # values are constant
         if value < _np.min(values_input):
-            logger.log_warning("value less than min")
-            return values_output[0]
+            # count warning exceeding min:
+            warning_counter[0] = 1
+            return values_output[0], warning_counter
 
         if value > _np.max(values_input):
-            logger.log_warning("value greater than max")
-            return values_output[-1]
+            # count warning exceeding max:
+            warning_counter[1] = 1
+            return values_output[-1], warning_counter
 
-        return _np.interp(value, values_input, values_output)
+        return _np.interp(value, values_input, values_output), warning_counter

decoimpact/business/entities/rules/step_function_rule.py

@@ -73,7 +73,7 @@ def validate(self, logger: ILogger) -> bool:
             return False
         return True
 
-    def execute(self, value: float, logger: ILogger) -> float:
+    def execute(self, value: float, logger: ILogger):
         """Classify a variable, based on given bins.
         Values lower than lowest bin will produce a warning and will
         be assigned class 0.
@@ -86,21 +86,25 @@ def execute(self, value: float, logger: ILogger) -> float:
 
         Returns:
             float: response corresponding to value to classify
+            int[]: number of warnings less than minimum and greater than maximum
         """
 
         bins = self._limits
         responses = self._responses
 
         # bins are constant
         selected_bin = -1
+        warning_counter = [0, 0]
         if _np.isnan(value):
-            return value
+            return value, warning_counter
         if value < _np.min(bins):
-            logger.log_warning("value less than min")
+            # count warning exceeding min:
+            warning_counter[0] = 1
             selected_bin = 0
         else:
             selected_bin = _np.digitize(value, bins) - 1
             if value > _np.max(bins):
-                logger.log_warning("value greater than max")
+                # count warning exceeding max:
+                warning_counter[1] = 1
 
-        return responses[selected_bin]
+        return responses[selected_bin], warning_counter

decoimpact/data/entities/data_access_layer.py

@@ -61,8 +61,7 @@ def read_input_file(self, path: Path) -> IModelData:
             try:
                 yaml_data = model_data_builder.parse_yaml_data(contents)
             except AttributeError as exc:
-                raise AttributeError("Error reading input file") from exc
-
+                raise AttributeError(f"Error reading input file. {exc}") from exc
             return yaml_data
 
     def read_input_dataset(self, dataset_data: IDatasetData) -> _xr.Dataset:
@@ -123,8 +122,11 @@ def read_input_dataset(self, dataset_data: IDatasetData) -> _xr.Dataset:
         return dataset
 
     def write_output_file(
-        self, dataset: _xr.Dataset, path: Path, application_version: str,
-        application_name: str
+        self,
+        dataset: _xr.Dataset,
+        path: Path,
+        application_version: str,
+        application_name: str,
     ) -> None:
         """Write XArray dataset to specified path
 

tests/business/entities/rules/test_response_curve_rule.py

@@ -13,7 +13,10 @@
 
 import numpy as _np
 import pytest
+import xarray as _xr
 
+from decoimpact.business.entities.rule_processor import RuleProcessor
+from decoimpact.business.entities.rules.i_cell_based_rule import ICellBasedRule
 from decoimpact.business.entities.rules.response_curve_rule import ResponseCurveRule
 from decoimpact.crosscutting.i_logger import ILogger
 
@@ -47,7 +50,7 @@ def test_create_response_rule(example_rule):
 
 @pytest.mark.parametrize(
     "input_value, expected_output_value",
-    [(25, 0.5), (75, 1.1), (770, 2.1)],
+    [(25, (0.5, [0, 0])), (75, (1.1, [0, 0])), (770, (2.1, [0, 0]))],
 )
 def test_execute_response_rule_values_between_limits(
     example_rule, input_value: int, expected_output_value: float
@@ -63,30 +66,6 @@ def test_execute_response_rule_values_between_limits(
     logger.log_warning.assert_not_called()
 
 
-@pytest.mark.parametrize(
-    "input_value, expected_output_value, expected_log_message",
-    [
-        (-1, 0, "value less than min"),
-        (6000, 3, "value greater than max"),
-    ],
-)
-def test_execute_response_rule_values_outside_limits(
-    example_rule,
-    input_value: int,
-    expected_output_value: int,
-    expected_log_message: str,
-):
-    """
-    Test the function execution with input values outside the interval limits.
-    """
-    # Arrange
-    logger = Mock(ILogger)
-
-    # Assert
-    assert example_rule.execute(input_value, logger) == expected_output_value
-    logger.log_warning.assert_called_with(expected_log_message)
-
-
 def test_inputs_and_outputs_have_different_lengths(example_rule):
     """
     Test the function execution when input and outputs have different lengths
@@ -131,7 +110,14 @@ def fixture_example_rule_combined():
 
 @pytest.mark.parametrize(
     "input_value, expected_output_value",
-    [(-1, 22), (0.5, 18.5), (1.5, 12.5), (3.5, 11), (7.5, 16), (10.5, 20)],
+    [
+        (-1, (22, [1, 0])),
+        (0.5, (18.5, [0, 0])),
+        (1.5, (12.5, [0, 0])),
+        (3.5, (11, [0, 0])),
+        (7.5, (16, [0, 0])),
+        (10.5, (20, [0, 1])),
+    ],
 )
 def test_execute_values_combined_dec_inc(
     example_rule_combined,

tests/business/entities/rules/test_step_function_rule.py

@@ -9,10 +9,16 @@
 """
 
 
+from typing import Dict, List
+
 import numpy as _np
 import pytest
+import xarray as _xr
 from mock import Mock
 
+from decoimpact.business.entities.rule_processor import RuleProcessor
+from decoimpact.business.entities.rules.i_cell_based_rule import ICellBasedRule
+from decoimpact.business.entities.rules.i_rule import IRule
 from decoimpact.business.entities.rules.step_function_rule import StepFunctionRule
 from decoimpact.crosscutting.i_logger import ILogger
 
@@ -46,10 +52,10 @@ def test_create_step_function(example_rule):
 @pytest.mark.parametrize(
     "input_value, expected_output_value",
     [
-        (0.5, 10),
-        (1.5, 11),
-        (2.5, 12),
-        (5.5, 15),
+        (0.5, (10, [0, 0])),
+        (1.5, (11, [0, 0])),
+        (2.5, (12, [0, 0])),
+        (5.5, (15, [0, 0])),
     ],
 )
 def test_execute_values_between_limits(
@@ -68,7 +74,13 @@ def test_execute_values_between_limits(
 
 @pytest.mark.parametrize(
     "input_value, expected_output_value",
-    [(0, 10), (1, 11), (2, 12), (5, 15), (10, 20)],
+    [
+        (0, (10, [0, 0])),
+        (1, (11, [0, 0])),
+        (2, (12, [0, 0])),
+        (5, (15, [0, 0])),
+        (10, (20, [0, 0])),
+    ],
 )
 def test_execute_values_at_limits(
     example_rule, input_value: int, expected_output_value: int
@@ -84,27 +96,6 @@ def test_execute_values_at_limits(
     logger.log_warning.assert_not_called()
 
 
-@pytest.mark.parametrize(
-    "input_value, expected_output_value, expected_log_message",
-    [(-1, 10, "value less than min"), (11, 20, "value greater than max")],
-)
-def test_execute_values_outside_limits(
-    example_rule,
-    input_value: int,
-    expected_output_value: int,
-    expected_log_message: str,
-):
-    """
-    Test the function execution with input values outside the interval limits.
-    """
-    # Arrange
-    logger = Mock(ILogger)
-
-    # Assert
-    assert example_rule.execute(input_value, logger) == expected_output_value
-    logger.log_warning.assert_called_with(expected_log_message)
-
-
 def test_limits_and_responses_have_different_lengths(example_rule):
     """
     Test the function execution when limits and responses have different lengths
@@ -164,7 +155,14 @@ def fixture_example_rule_combined():
 
 @pytest.mark.parametrize(
     "input_value, expected_output_value",
-    [(-1, 22), (0.5, 22), (1.5, 15), (2.5, 10), (5.5, 12), (10.5, 20)],
+    [
+        (-1, (22, [1, 0])),
+        (0.5, (22, [0, 0])),
+        (1.5, (15, [0, 0])),
+        (2.5, (10, [0, 0])),
+        (5.5, (12, [0, 0])),
+        (10.5, (20, [0, 1])),
+    ],
 )
 def test_execute_values_combined_dec_inc(
     example_rule_combined,

tests/business/entities/test_rule_processor.py

@@ -27,6 +27,7 @@
     IMultiCellBasedRule,
 )
 from decoimpact.business.entities.rules.i_rule import IRule
+from decoimpact.business.entities.rules.step_function_rule import StepFunctionRule
 from decoimpact.business.entities.rules.time_aggregation_rule import TimeAggregationRule
 from decoimpact.crosscutting.i_logger import ILogger
 from decoimpact.data.api.i_time_aggregation_rule_data import ITimeAggregationRuleData
@@ -284,7 +285,8 @@ def test_process_rules_calls_cell_based_rule_execute_correctly():
     rule.input_variable_names = ["test"]
     rule.output_variable_name = "output"
 
-    rule.execute.return_value = 1
+    # expected return value = 1; number of warnings (min and max) = 0 and 0
+    rule.execute.return_value = [1, [0, 0]]
 
     processor = RuleProcessor([rule], dataset)
 
@@ -298,6 +300,7 @@ def test_process_rules_calls_cell_based_rule_execute_correctly():
 
     assert rule.execute.call_count == 6
 
+
 def test_process_rules_calls_multi_cell_based_rule_execute_correctly():
     """Tests if during processing the rule its execute method of
     an IMultiCellBasedRule is called with the right parameter."""
@@ -330,6 +333,7 @@ def test_process_rules_calls_multi_cell_based_rule_execute_correctly():
 
     assert rule.execute.call_count == 6
 
+
 def test_process_rules_calls_array_based_rule_execute_correctly():
     """Tests if during processing the rule its execute method of
     an IArrayBasedRule is called with the right parameter."""
@@ -505,3 +509,50 @@ def test_execute_rule_throws_error_for_unknown_input_variable():
         + "in input datasets or in calculated output dataset."
     )
     assert exception_raised.args[0] == expected_message
+
+
+@pytest.fixture(name="example_rule")
+def fixture_example_rule():
+    """Inititaion of StepFunctionRule to be reused in the following tests"""
+    return StepFunctionRule(
+        "rule_name",
+        "input_variable_name",
+        [0, 1, 2, 5, 10],
+        [10, 11, 12, 15, 20],
+    )
+
+
+@pytest.mark.parametrize(
+    "input_value, expected_output_value, expected_log_message",
+    [
+        (-1, (10, [1, 0]), "value less than min: 1 occurence(s)"),
+        (11, (20, [0, 1]), "value greater than max: 1 occurence(s)"),
+    ],
+)
+def test_process_values_outside_limits(
+    example_rule,
+    input_value: int,
+    expected_output_value: int,
+    expected_log_message: str,
+):
+    """
+    Test the function execution with input values outside the interval limits.
+    """
+    # Arrange
+    logger = Mock(ILogger)
+    dataset = _xr.Dataset()
+    dataset["test1"] = _xr.DataArray(input_value)
+    rule = Mock(ICellBasedRule)
+    rule.input_variable_names = ["test1"]
+    rule.output_variable_name = "output"
+    rule.execute.return_value = expected_output_value
+    processor = RuleProcessor([rule], dataset)
+
+    # Act
+    assert processor.initialize(logger)
+    processor.process_rules(dataset, logger)
+
+    # Assert
+    assert example_rule.execute(input_value, logger) == expected_output_value
+    processor.process_rules(dataset, logger)
+    logger.log_warning.assert_called_with(expected_log_message)