diff --git a/decoimpact/business/entities/rule_based_model.py b/decoimpact/business/entities/rule_based_model.py index 294361dd..5b2bdac1 100644 --- a/decoimpact/business/entities/rule_based_model.py +++ b/decoimpact/business/entities/rule_based_model.py @@ -21,6 +21,7 @@ from decoimpact.business.entities.i_model import IModel, ModelStatus from decoimpact.business.entities.rule_processor import RuleProcessor from decoimpact.business.entities.rules.i_rule import IRule +from decoimpact.crosscutting.delft3d_specific_data import delft3d_specific_names from decoimpact.crosscutting.i_logger import ILogger @@ -153,6 +154,7 @@ def _make_output_variables_list(self) -> list: return _lu.remove_duplicates_from_list(all_vars) + # pylint: disable=too-many-locals def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: """Checks if the provided mappings are valid. @@ -169,6 +171,7 @@ def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: for ds in self._input_datasets ] ) + self._check_keys_with_suffixes(mappings, delft3d_specific_names) valid = True @@ -198,10 +201,17 @@ def _validate_mappings(self, mappings: dict[str, str], logger: ILogger) -> bool: rule_inputs = self._get_direct_rule_inputs(rule_names) + for dataset in self._input_datasets: + dummy_var_name = _du.get_dummy_variable_in_ugrid(dataset) + # check for missing rule inputs for rule_name, rule_input in rule_inputs.items(): needed_rule_inputs = _lu.remove_duplicates_from_list(rule_input) rule_input_vars = input_vars + list(mappings.values()) + needed_rule_inputs = _du.extend_to_full_name( + needed_rule_inputs, + dummy_var_name + ) missing_rule_inputs = _lu.items_not_in(needed_rule_inputs, rule_input_vars) if len(missing_rule_inputs) > 0: logger.log_error( @@ -239,3 +249,19 @@ def _extend_names(self, dummy_variable_name: str): rule.input_variable_names, dummy_variable_name ) + + def _check_keys_with_suffixes(self, dictionary, suffixes): + """ + Checks if any key in the dictionary ends with provided suffixes. + Raise eror if that occurs. + + Args: + dictionary (dict) : The dictionary to check. + suffixes (List[str]): List of suffixes to check against. + + """ + for key in dictionary: + if any(key.endswith(suffix) for suffix in suffixes): + raise ValueError(f"Remapping variables ending with" + f" {delft3d_specific_names} is not" + f" allowed.") diff --git a/decoimpact/business/entities/rule_processor.py b/decoimpact/business/entities/rule_processor.py index 6c70735c..dcc950c6 100644 --- a/decoimpact/business/entities/rule_processor.py +++ b/decoimpact/business/entities/rule_processor.py @@ -68,6 +68,7 @@ def initialize(self, logger: ILogger) -> bool: inputs = _lu.flatten_list( [_du.list_vars(self._input_dataset), _du.list_coords(self._input_dataset)] ) + tree, success = self._create_rule_sets(inputs, self._rules, [], logger) if success: self._processing_list = tree diff --git a/decoimpact/business/entities/rules/depth_average_rule.py b/decoimpact/business/entities/rules/depth_average_rule.py index ac9e83b7..8119429f 100644 --- a/decoimpact/business/entities/rules/depth_average_rule.py +++ b/decoimpact/business/entities/rules/depth_average_rule.py @@ -20,6 +20,7 @@ from decoimpact.business.entities.rules.rule_base import RuleBase from decoimpact.crosscutting.delft3d_specific_data import ( BED_LEVEL_SUFFIX, + INTERFACES_SIGMA_SUFFIX, INTERFACES_Z_SUFFIX, WATER_LEVEL_SUFFIX, ) @@ -29,6 +30,11 @@ class DepthAverageRule(RuleBase, IMultiArrayBasedRule): """Implementation for the depth average rule""" + def __init__(self, name: str, input_variable_names: List[str], layer_type: str): + super().__init__(name, input_variable_names) + self._layer_type = layer_type + + # pylint: disable=too-many-locals def execute( self, value_arrays: Dict[str, _xr.DataArray], logger: ILogger ) -> _xr.DataArray: @@ -45,12 +51,13 @@ def execute( # but the name of the key is given by the user, and is unknown here, so # just use the first value. variables = next(iter(value_arrays.values())) + interface_suffix = _get_layer_suffix(self._layer_type, logger) - bed_level_values = self._extract_variable_based_on_suffix( + bed_level_values = _extract_variable_based_on_suffix( value_arrays, BED_LEVEL_SUFFIX) - depths_interfaces = self._extract_variable_based_on_suffix( - value_arrays, INTERFACES_Z_SUFFIX) - water_level_values = self._extract_variable_based_on_suffix( + depths_interfaces = _extract_variable_based_on_suffix( + value_arrays, interface_suffix) + water_level_values = _extract_variable_based_on_suffix( value_arrays, WATER_LEVEL_SUFFIX) # Get the dimension names for the interfaces and for the layers @@ -109,20 +116,42 @@ def execute( dim=dim_layer_name ) - def _extract_variable_based_on_suffix( - self, - value_arrays: Dict[str, _xr.DataArray], - suffix: str - ) -> List: - """Extract the values from the XArray dataset based on the name - suffixes by matching the name, irrespective of the dummy name prefix. - - Args: - value_array (DataArray): Values - suffix (str) : Suffix of the name - Returns: - values (List[str]): Values based on prefix + suffix name - """ - variable = [value_arrays[name] for name in value_arrays if suffix in name][0] - return variable +def _extract_variable_based_on_suffix( + value_arrays: Dict[str, _xr.DataArray], + suffix: str + ) -> List: + """Extract the values from the XArray dataset based on the name + suffixes by matching the name, irrespective of the dummy name prefix. + + Args: + value_array (DataArray): Values + suffix (str) : Suffix of the name + + Returns: + values (List[str]): Values based on prefix + suffix name + """ + return [value_arrays[name] for name in value_arrays if suffix in name][0] + + +def _get_layer_suffix( + layer_type: str, + logger: ILogger + ): + """Get the interface suffix depending on whether the model is a sigma or z + layer model. Give error if the interface suffix cannot be determined. + + Args: + value_array (DataArray): Values + + Returns: + layer_type (str): sigma or z + """ + if layer_type.lower() == "sigma": + return INTERFACES_SIGMA_SUFFIX + if layer_type.lower() == "z": + return INTERFACES_Z_SUFFIX + logger.log_error(f"Layer type {layer_type} unknown. Allowed layer " + "type: z or sigma. Interface " + "variable could not be determined.") + return "_unknown" diff --git a/decoimpact/business/workflow/model_builder.py b/decoimpact/business/workflow/model_builder.py index 0c039fdc..1eee980c 100644 --- a/decoimpact/business/workflow/model_builder.py +++ b/decoimpact/business/workflow/model_builder.py @@ -107,6 +107,7 @@ def _create_rule(rule_data: IRuleData) -> IRule: rule = DepthAverageRule( rule_data.name, rule_data.input_variables, + rule_data.layer_type, ) elif isinstance(rule_data, ILayerFilterRuleData): rule = LayerFilterRule( diff --git a/decoimpact/data/api/i_depth_average_rule_data.py b/decoimpact/data/api/i_depth_average_rule_data.py index 9272ba52..01eb9891 100644 --- a/decoimpact/data/api/i_depth_average_rule_data.py +++ b/decoimpact/data/api/i_depth_average_rule_data.py @@ -26,3 +26,8 @@ class IDepthAverageRuleData(IRuleData, ABC): @abstractmethod def input_variables(self) -> List[str]: """List with input variable name and standard depth name""" + + @property + @abstractmethod + def layer_type(self) -> str: + """Layer type of the model (z or sigma)""" diff --git a/decoimpact/data/entities/depth_average_rule_data.py b/decoimpact/data/entities/depth_average_rule_data.py index 99afa504..f3355022 100644 --- a/decoimpact/data/entities/depth_average_rule_data.py +++ b/decoimpact/data/entities/depth_average_rule_data.py @@ -13,6 +13,7 @@ """ from typing import List + from decoimpact.data.api.i_depth_average_rule_data import IDepthAverageRuleData from decoimpact.data.entities.rule_data import RuleData @@ -24,11 +25,18 @@ def __init__( self, name: str, input_variables: List[str], + layer_type: str, ): super().__init__(name) self._input_variables = input_variables + self._layer_type = layer_type @property def input_variables(self) -> List[str]: """List with input variables""" return self._input_variables + + @property + def layer_type(self) -> str: + """Layer type of the model (z or sigma)""" + return self._layer_type diff --git a/decoimpact/data/parsers/parser_depth_average_rule.py b/decoimpact/data/parsers/parser_depth_average_rule.py index 93df79a5..8257d7ed 100644 --- a/decoimpact/data/parsers/parser_depth_average_rule.py +++ b/decoimpact/data/parsers/parser_depth_average_rule.py @@ -14,6 +14,7 @@ from decoimpact.crosscutting.delft3d_specific_data import ( BED_LEVEL_SUFFIX, + INTERFACES_SIGMA_SUFFIX, INTERFACES_Z_SUFFIX, WATER_LEVEL_SUFFIX, ) @@ -41,9 +42,12 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: RuleBase: Rule based on the provided data """ name: str = get_dict_element("name", dictionary) + layer_type: str = get_dict_element("layer_type", dictionary) + interface_suffix = _obtain_interface_suffix(layer_type) + input_variable_names: List[str] = [ get_dict_element("input_variable", dictionary), - INTERFACES_Z_SUFFIX, + interface_suffix, WATER_LEVEL_SUFFIX, BED_LEVEL_SUFFIX, ] @@ -51,9 +55,28 @@ def parse_dict(self, dictionary: Dict[str, Any], logger: ILogger) -> IRuleData: output_variable_name: str = get_dict_element("output_variable", dictionary) description: str = get_dict_element("description", dictionary, False) or "" - rule_data = DepthAverageRuleData(name, input_variable_names) + rule_data = DepthAverageRuleData(name, input_variable_names, layer_type) rule_data.output_variable = output_variable_name rule_data.description = description return rule_data + + +def _obtain_interface_suffix(layer_type: str): + """Obtain the interface variable based on the layer_type specified. + Give an error if layer_type is not recognised + + Args: + layer_type (str): z or sigma layers + + Returns: + Suffix for z or sigma layers based on Delft3D + defined suffixes + """ + if layer_type.lower() == 'z': + return INTERFACES_Z_SUFFIX + if layer_type.lower() == 'sigma': + return INTERFACES_SIGMA_SUFFIX + raise NotImplementedError(f"Layer_type '{layer_type}' is not recognized. " + f"Supported options are 'z' and 'sigma'.") diff --git a/docs/manual/input.md b/docs/manual/input.md index 2458347a..ade420d0 100644 --- a/docs/manual/input.md +++ b/docs/manual/input.md @@ -19,6 +19,8 @@ output-data: In the input data the variables that are present in the input data provided through “filename” are selected for use. It is possible to filter the input data by providing a start date or end date (format: "dd-mm-yyyy"); this is optional. The variables that are used can be selected under “variable_mapping”. Here you are also able to rename variables as the name used for storage is often cryptic. +Note: remapping the variables 'mesh2d_interface_z', 'mesh2d_interface_sigma', 'mesh2d_flowelem_bl and 'mesh2d_s1' is currently not supported to enable depth-averaging. This will be remedied in a future version. For now a work-around is to use a multiply rule with a multiplication factor of 1. This will in effect create a new variable, to which the remapped name can be assigned as output. + At output data the location where the output file needs to be written can be provided through “filename”. In this output file only variables that have been used from the input data and variables that have been created in the model are stored. It is possible to reduce the file size with the optional parameter "save_only_variables", which can take the name of one or several variables. The model needs at least one rule under “rules” to execute. diff --git a/docs/manual/rules/depth_average_rule.md b/docs/manual/rules/depth_average_rule.md index eafec297..2781f763 100644 --- a/docs/manual/rules/depth_average_rule.md +++ b/docs/manual/rules/depth_average_rule.md @@ -1,4 +1,3 @@ - ### Depth average rule ``` @@ -7,14 +6,17 @@ FORMAT name: description: input_variable: + layer_type: output_variable: ``` -The depth average rule allows for an averaging over depth using the weighted values according to a mesh with z-layers. The input file must include a variable called 'mesh2d_interface_z' over which the the input variable will be averaged. The input_variable will be a 2D/3D variable, with or without time axis. The output_variable has the same dimensions, excluding the dimension for the depth, as it will be represented as one averaged value per cell. +The depth average rule allows for an averaging over depth using the weighted values according to a mesh with z- or sigma-layers. The current implementation only supports input netCDF files generated by D-Hydro. The input file must include a variable called 'mesh2d_interface_z' or 'mesh2d_interface_sigma' over which the input variable will be averaged. Also two variables specifying the bedlevel and water level, 'mesh2d_flowelem_bl' and 'mesh2d_s1' are needed. The input_variable will be a 2D/3D variable, with or without time axis. The output_variable has the same dimensions, excluding the dimension for the depth, as it will be represented as one averaged value per cell. + +Note: remapping the variables 'mesh2d_interface_z', 'mesh2d_interface_sigma', 'mesh2d_flowelem_bl and 'mesh2d_s1' in combination with the depth average rule is currently not supported. Combined z-sigma layers are also not supported. An explanation of how the depth rule works is shown in the example below. -![Example depth average rule](../../assets/images/3_depth_average.png "An example of a simplified grid with Z-layers. This model has 6 faces, 4 layers and 2 timesteps.") +![Example depth average rule](../assets/images/3_depth_average.png "An example of a simplified grid with Z-layers. This model has 6 faces, 4 layers and 2 timesteps.") The image shows a simplified model with the following dimensions: - mesh2d_nFaces = 6 (number of faces) @@ -95,6 +97,6 @@ Below is an example of an input_file for the depth average rule: name: test depth average description: Test depth average input_variable: salinity + layer_type : sigma output_variable: average_salinity -``` - +``` \ No newline at end of file diff --git a/tests/business/entities/rules/test_depth_average_rule.py b/tests/business/entities/rules/test_depth_average_rule.py index f009c6d4..ef54e4aa 100644 --- a/tests/business/entities/rules/test_depth_average_rule.py +++ b/tests/business/entities/rules/test_depth_average_rule.py @@ -14,7 +14,6 @@ import numpy as _np import pytest import xarray as _xr -from tomlkit import value from decoimpact.business.entities.rules.depth_average_rule import DepthAverageRule from decoimpact.crosscutting.i_logger import ILogger @@ -24,7 +23,7 @@ def test_create_depth_average_rule_with_defaults(): """Test creating a depth average rule with defaults""" # Arrange & Act - rule = DepthAverageRule("test_rule_name", ["foo", "hello"]) + rule = DepthAverageRule("test_rule_name", ["foo", "hello"],"z") # Assert assert isinstance(rule, DepthAverageRule) @@ -32,6 +31,7 @@ def test_create_depth_average_rule_with_defaults(): assert rule.description == "" assert rule.input_variable_names == ["foo", "hello"] assert rule.output_variable_name == "output" + assert rule._layer_type == "z" def test_no_validate_error_with_correct_rule(): @@ -41,6 +41,7 @@ def test_no_validate_error_with_correct_rule(): rule = DepthAverageRule( "test_rule_name", ["foo", "hello"], + "z", ) # Assert @@ -95,7 +96,8 @@ def test_depth_average_rule( logger = Mock(ILogger) rule = DepthAverageRule( name="test", - input_variable_names=["foo"], + input_variable_names=["foo", "mesh2d_interface_z"], + layer_type="z", ) # Create dataset @@ -126,11 +128,13 @@ def test_depth_average_rule( def test_dimension_error(): - """If the number of interfaces > number of layers + 1. Give an error, no calculation is possible""" + """If the number of interfaces > number of layers + 1. Give an error, no + calculation is possible""" logger = Mock(ILogger) rule = DepthAverageRule( name="test", - input_variable_names=["foo"], + input_variable_names=["foo", "mesh2d_interface_z"], + layer_type="z", ) # Create dataset diff --git a/tests/business/entities/test_rule_based_model.py b/tests/business/entities/test_rule_based_model.py index ab99e102..080c84f6 100644 --- a/tests/business/entities/test_rule_based_model.py +++ b/tests/business/entities/test_rule_based_model.py @@ -72,6 +72,7 @@ def test_validation_of_rule_based_model(): logger = Mock(ILogger) dataset["test"] = _xr.DataArray([32, 94, 9]) + dataset["test"].attrs = {"cf_role": "mesh_topology"} rule.input_variable_names = ["input"] rule.output_variable_name = "output" diff --git a/tests/data/entities/test_depth_average_rule_data.py b/tests/data/entities/test_depth_average_rule_data.py index 86c32ee9..08e7f7a7 100644 --- a/tests/data/entities/test_depth_average_rule_data.py +++ b/tests/data/entities/test_depth_average_rule_data.py @@ -18,9 +18,10 @@ def test_depth_average_rule_data_creation_logic(): to correctly initialize itself during creation""" # Act - data = DepthAverageRuleData("test_name", "input1") + data = DepthAverageRuleData("test_name", "input1", "z") # Assert assert isinstance(data, IRuleData) assert data.input_variables == "input1" + assert data.layer_type == "z" diff --git a/tests/data/parsers/test_parser_depth_average_rule.py b/tests/data/parsers/test_parser_depth_average_rule.py index a704bb47..44e50465 100644 --- a/tests/data/parsers/test_parser_depth_average_rule.py +++ b/tests/data/parsers/test_parser_depth_average_rule.py @@ -9,6 +9,7 @@ """ from typing import Any, List + import pytest from mock import Mock @@ -38,6 +39,7 @@ def test_parse_dict_to_rule_data_logic(): { "name": "testname", "input_variable": "input", + "layer_type": "z", "output_variable": "output", } ) @@ -56,6 +58,7 @@ def test_parse_wrong_dict_to_rule_data_logic(): { "name": "testname", "output_variable": "output", + "layer_type": "sigma", } ) logger = Mock(ILogger) @@ -71,3 +74,30 @@ def test_parse_wrong_dict_to_rule_data_logic(): # Assert expected_message = "Missing element input_variable" assert exception_raised.args[0] == expected_message + +def test_parse_incorrect_layer_type_to_rule_data_logic(): + """Test if an incorrect layer_type not parsed""" + # Arrange + incorrect_layer_type = "UNKNOWN" + contents = dict( + { + "name": "testname", + "input_variable": "input", + "layer_type": incorrect_layer_type, + "output_variable": "output", + } + ) + logger = Mock(ILogger) + + # Act + data = ParserDepthAverageRule() + + with pytest.raises(NotImplementedError) as exc_info: + data.parse_dict(contents, logger) + + exception_raised = exc_info.value + + # Assert + expected_message = f"Layer_type '{incorrect_layer_type}' is not recognized. \ +Supported options are 'z' and 'sigma'." + assert exception_raised.args[0] == expected_message diff --git a/tests_acceptance/input_nc_files/test_depth_average_sigmalayer_rule.nc b/tests_acceptance/input_nc_files/test_depth_average_sigmalayer_rule.nc new file mode 100644 index 00000000..3b9e1102 Binary files /dev/null and b/tests_acceptance/input_nc_files/test_depth_average_sigmalayer_rule.nc differ diff --git a/tests_acceptance/input_nc_files/test_depth_average_rule.nc b/tests_acceptance/input_nc_files/test_depth_average_zlayer_rule.nc similarity index 100% rename from tests_acceptance/input_nc_files/test_depth_average_rule.nc rename to tests_acceptance/input_nc_files/test_depth_average_zlayer_rule.nc diff --git a/tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml b/tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml similarity index 85% rename from tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml rename to tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml index 8819e01a..47c5c699 100644 --- a/tests_acceptance/input_yaml_files/test17_depth_average_rule.yaml +++ b/tests_acceptance/input_yaml_files/test17_depth_average_zlayer_rule.yaml @@ -2,26 +2,29 @@ version: 0.0.0 input-data: - dataset: - filename: ./tests_acceptance/input_nc_files/test_depth_average_rule.nc + filename: ./tests_acceptance/input_nc_files/test_depth_average_zlayer_rule.nc rules: - depth_average_rule: name: Test A description: Test A input_variable: var_3d_A + layer_type: Z output_variable: output_A - depth_average_rule: name: Test B description: Test B input_variable: var_3d_B + layer_type: z output_variable: output_B - depth_average_rule: name: Test C description: Test C input_variable: var_3d_C + layer_type: z output_variable: output_C - + output-data: - filename: ./tests_acceptance/output_nc_files/test17_depth_average_rule.nc + filename: ./tests_acceptance/output_nc_files/test17_depth_average_zlayer_rule.nc diff --git a/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml b/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml new file mode 100644 index 00000000..3d499f16 --- /dev/null +++ b/tests_acceptance/input_yaml_files/test18_depth_average_sigmalayer_rule.yaml @@ -0,0 +1,30 @@ +version: 0.0.0 + +input-data: + - dataset: + filename: ./tests_acceptance/input_nc_files/test_depth_average_sigmalayer_rule.nc + +rules: + - depth_average_rule: + name: Test A + description: Test A + input_variable: var_3d_A + layer_type: SIGMA + output_variable: output_A + + - depth_average_rule: + name: Test B + description: Test B + input_variable: var_3d_B + layer_type: sigma + output_variable: output_B + + - depth_average_rule: + name: Test C + description: Test C + input_variable: var_3d_C + layer_type: sigma + output_variable: output_C + +output-data: + filename: ./tests_acceptance/output_nc_files/test18_depth_average_sigmalayer_rule.nc diff --git a/tests_acceptance/reference_nc_files/test17_depth_average_rule.nc b/tests_acceptance/reference_nc_files/test17_depth_average_zlayer_rule.nc similarity index 100% rename from tests_acceptance/reference_nc_files/test17_depth_average_rule.nc rename to tests_acceptance/reference_nc_files/test17_depth_average_zlayer_rule.nc diff --git a/tests_acceptance/reference_nc_files/test18_depth_average_sigmalayer_rule.nc b/tests_acceptance/reference_nc_files/test18_depth_average_sigmalayer_rule.nc new file mode 100644 index 00000000..b5ca6f6a Binary files /dev/null and b/tests_acceptance/reference_nc_files/test18_depth_average_sigmalayer_rule.nc differ