add unit tests

Signed-off-by: Nitish Bharambe <nitish.bharambe@alliander.com>

src/power_grid_model_io/config/excel/vision_en_9_7.yaml

@@ -425,6 +425,11 @@ grid:
       extra:
         - ID
         - Name
+      filter:
+        - power_grid_model_io.functions.filters.filter_all_columns_empty_or_zero:
+            cols: 
+            - Load.P
+            - Load.Q
     sym_gen:
       - id:
           auto_id:
@@ -445,6 +450,11 @@ grid:
         extra:
           - ID
           - Name
+        filter:
+          - power_grid_model_io.functions.filters.filter_all_columns_empty_or_zero:
+              cols: 
+              - Generation.P
+              - Generation.Q
       - id:
           auto_id:
             name: pv_generation
@@ -467,6 +477,10 @@ grid:
         extra:
           - ID
           - Name
+        filter:
+          - power_grid_model_io.functions.filters.filter_all_columns_empty_or_zero:
+              cols: 
+              - PV.Pnom
   Sources:
     source:
       id:

src/power_grid_model_io/config/excel/vision_nl.yaml

@@ -415,6 +415,11 @@ grid:
       extra:
         - ID
         - Naam
+      filter:
+        - power_grid_model_io.functions.filters.filter_all_columns_empty_or_zero:
+            cols: 
+            - Belasting.P
+            - Belasting.Q
     sym_gen:
       - id:
           auto_id:
@@ -435,6 +440,11 @@ grid:
         extra:
           - ID
           - Naam
+        filter:
+          - power_grid_model_io.functions.filters.filter_all_columns_empty_or_zero:
+              cols: 
+              - Opwekking.P
+              - Opwekking.Q
       - id:
           auto_id:
             name: pv_generation
@@ -457,6 +467,10 @@ grid:
         extra:
           - ID
           - Naam
+        filter:
+          - power_grid_model_io.functions.filters.filter_all_columns_empty_or_zero:
+              cols: 
+              - PV.Pnom
   Netvoedingen:
     source:
       id:

src/power_grid_model_io/converters/tabular_converter.py

@@ -179,7 +179,7 @@ def _convert_table_to_component(
             return None
 
         if "filter" in attributes:
-            table_mask = self._parse_table_filters(data, table, attributes["filter"])
+            table_mask = self._parse_table_filters(data=data, table=table, filtering_functions=attributes["filter"])
         else:
             table_mask = np.ones(len(data[table]), dtype=bool)
 
@@ -210,10 +210,11 @@ def _convert_table_to_component(
         return pgm_data
 
     def _parse_table_filters(self, data: TabularData, table: str, filtering_functions: Any) -> np.ndarray:
-        assert isinstance(data[table], pd.DataFrame)
-
         table_mask = np.ones(len(data[table]), dtype=bool)
 
+        if not isinstance(data[table], pd.DataFrame):
+            return table_mask
+
         for filtering_fn in filtering_functions:
             for fn_name, kwargs in filtering_fn.items():
                 fn_ptr = get_function(fn_name)

src/power_grid_model_io/functions/filters.py

@@ -19,9 +19,9 @@ def filter_empty(row: pd.Series, col: str) -> bool:
     return has_value(row[col]).values[0]
 
 
-def filter_by_value(row: pd.Series, col: str, value: float) -> bool:
+def filter_by_value(row: pd.Series, col: str, value: float | str) -> bool:
     """
-    filter out  by match value
+    filter out by match value
     """
     return (row[col] != value).values[0]
 

tests/unit/converters/test_tabular_converter.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 from pathlib import Path
-from typing import Tuple
+from typing import Callable, Tuple
 from unittest.mock import MagicMock, call, patch
 
 import numpy as np
@@ -154,6 +154,27 @@ def test_convert_table_to_component(converter: TabularConverter, tabular_data_no
     assert (pgm_node_data["u_rated"] == [10.5e3, 400]).all()
 
 
+def test_convert_table_to_component__filters(
+    converter: TabularConverter, tabular_data_no_units_no_substitutions: TabularData
+):
+    converter._convert_col_def_to_attribute = MagicMock()
+    converter._parse_table_filters = MagicMock()
+    node_attributes_with_filter = {"id": "id_number", "u_rated": "u_nom", "filter": [{"test_fn": {}}]}
+    converter._convert_table_to_component(
+        data=tabular_data_no_units_no_substitutions,
+        data_type="input",
+        table="nodes",
+        component="node",
+        attributes=node_attributes_with_filter,
+        extra_info=None,
+    )
+    converter._parse_table_filters.assert_called_once_with(
+        data=tabular_data_no_units_no_substitutions,
+        table="nodes",
+        filtering_functions=node_attributes_with_filter["filter"],
+    )
+
+
 def test_convert_col_def_to_attribute(
     converter: TabularConverter,
     tabular_data_no_units_no_substitutions: TabularData,
@@ -1135,3 +1156,40 @@ def test_lookup_ids__duplicate_keys(converter: TabularConverter):
 
     # Assert
     pd.testing.assert_frame_equal(reference, pd.DataFrame([[123, 456]], columns=["table", "name"], index=[0]))
+
+
+@pytest.mark.parametrize(
+    ("bool_fn", "expected"),
+    [((True), np.array([True, True])), ((False), np.array([False, False]))],
+)
+@patch("power_grid_model_io.converters.tabular_converter.get_function")
+def test_parse_table_filters(
+    mock_get_function: MagicMock,
+    converter: TabularConverter,
+    tabular_data: TabularData,
+    bool_fn: Callable,
+    expected: np.ndarray,
+):
+    filtering_functions = [{"test_fn": {"kwarg_1": "a"}}]
+
+    def bool_fn_filter(row: pd.Series, **kwargs):
+        assert kwargs == {"kwarg_1": "a"}
+        return bool_fn
+
+    mock_get_function.return_value = bool_fn_filter
+
+    actual = converter._parse_table_filters(data=tabular_data, table="nodes", filtering_functions=filtering_functions)
+
+    mock_get_function.assert_called_once_with("test_fn")
+    # check if return value is a 1d bool np array
+    assert isinstance(actual, np.ndarray)
+    assert actual.ndim == 1
+    assert actual.dtype == bool
+    assert all(actual == expected)
+
+
+def test_parse_table_filters__ndarray_data(converter: TabularConverter):
+    numpy_tabular_data = TabularData(numpy_table=np.ones((4, 3)))
+    actual = converter._parse_table_filters(data=numpy_tabular_data, table="numpy_table", filtering_functions=[])
+    assert all(actual)
+    assert len(actual) == 4