refactor for new paramset seperator

.github/workflows/build_pipeline.yml

@@ -58,7 +58,7 @@ jobs:
         aws s3 cp app.zip s3://arup-arup-ukimea-tcs-dev-test-code/$repo_slug.zip
     - name: Send build success notification
       if: success()
-      uses: rtCamp/action-slack-notify@v2.0.0
+      uses: rtCamp/action-slack-notify@v2.2.0
       env:
         SLACK_MESSAGE: ${{ github.repository }} build ${{ github.run_number }} launched by ${{ github.actor }} has succeeded
         SLACK_TITLE: Build Success
@@ -68,7 +68,7 @@ jobs:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
     - name: Send build failure notification
       if: failure()
-      uses: rtCamp/action-slack-notify@v2.0.0
+      uses: rtCamp/action-slack-notify@v2.2.0
       env:
         SLACK_COLOR: '#FF0000'
         SLACK_MESSAGE: ${{ github.repository }} build ${{ github.run_number }} launched by ${{ github.actor }} has failed

README.md

@@ -305,6 +305,11 @@ param {'name': 'monetaryDistanceRate', 'value': '-0.0001'}  # eg subpop B car
 
 In the examples above, you can see that wildcarding with `*` can be used to return 'all' config elements. The `*` operator tells the find method to search all at a given level. As shown above, it is useful for returning all elements within a parameterset or explicitly describing levels to search.
 
+The CLI supports to find the specific modules from given matsim config:
+```py
+mc find tests/test_data/test_config.json network
+```
+
 ### Find and Set
 
 Note that the `find` method is returning a reference to the found config objects, which can then be set:

mc/__init__.py

@@ -8,6 +8,6 @@
 __version__ = "1.0.2"
 _ROOT = Path(os.path.abspath(os.path.dirname(__file__)))
 
-_DEFAULTS_DIR = _ROOT / 'default_data'
+_DEFAULTS_DIR = _ROOT / "default_data"
 if not _DEFAULTS_DIR.is_dir():
     raise NotADirectoryError(f"Default data dir not found at {_DEFAULTS_DIR}")

mc/autostep.py

@@ -271,10 +271,13 @@ def dump_log_to_disk(log: list, path):
 
 
 def load_overrides(overrides_path: Path) -> dict:
+    if not overrides_path.exists():
+        logging.warning(f"Overrides file not found: {overrides_path}")
+        return {}
     with open(overrides_path) as o:
         overrides = json.loads(o.read())
         result = {}
-        for i, val in enumerate(overrides):
+        for val in overrides:
             for k, v in val.items():
                 result[k] = v
         return result

mc/base.py

@@ -145,7 +145,6 @@ def build_from_xml(self, xml_object: Element):
 
             elif sub_object.tag == "parameterset":
                 _, key, uid = build_paramset_key(sub_object)
-
                 self.parametersets[key] = ParamSet(key, xml_object=sub_object, uid=uid)
 
             else:
@@ -571,9 +570,9 @@ def __getitem__(self, key):
             return self.params[key].value
         if key in self.parametersets:
             return self.parametersets[key]
-        if key + ":default" in self.parametersets:
+        if key + "::default" in self.parametersets:
             print("WARNING assuming 'default' required")
-            return self.parametersets[key + ":default"]
+            return self.parametersets[key + "::default"]
 
         # try to collect list of paramsets
         collected = []
@@ -654,9 +653,9 @@ def get(self, key, default=None):
             return self.params[key].value
         if key in self.parametersets:
             return self.parametersets[key]
-        if key + ":default" in self.parametersets:
+        if key + "::default" in self.parametersets:
             print("WARNING assuming 'default' required")
-            return self.parametersets[key + ":default"]
+            return self.parametersets[key + "::default"]
 
         return default
 
@@ -715,7 +714,10 @@ def __init__(self, name, xml_object=None, json_object=None, uid=None) -> None:
             self.build_from_json(json_object)
 
     def __str__(self) -> str:
-        return f"{self.class_type.upper()}: {self.type} ({self.name})"
+        if ":" in self.type:
+            return f"{self.class_type.upper()}: {self.type} ({self.type})"
+        else:
+            return f"{self.class_type.upper()}: {self.type} ({self.name})"
 
     def print(self, i: int = 0) -> None:
         """
@@ -733,9 +735,9 @@ def __getitem__(self, key):
             return self.params[key].value
         if key in self.parametersets:
             return self.parametersets[key]
-        if key + ":default" in self.parametersets:
-            print("WARNING assuming '<parameterset>:default' required")
-            return self.parametersets[key + ":default"]
+        if key + "::default" in self.parametersets:
+            print("WARNING assuming '<parameterset>::default' required")
+            return self.parametersets[key + "::default"]
 
         # try to collect list of paramsets
         collected = []
@@ -834,9 +836,9 @@ def get(self, key, default=None):
             return self.params[key].value
         if key in self.parametersets:
             return self.parametersets[key]
-        if key + ":default" in self.parametersets:
+        if key + "::default" in self.parametersets:
             print("WARNING assuming 'default' required")
-            return self.parametersets[key + ":default"]
+            return self.parametersets[key + "::default"]
 
         return default
 
@@ -886,7 +888,7 @@ def __eq__(self, other):
         return True
 
 
-def specials_snap(a, b, divider=":", ignore="*"):
+def specials_snap(a, b, divider="::", ignore="*"):
     """
     Special function to check for key matches with consideration of
     special character '*' that represents 'all'.
@@ -938,7 +940,7 @@ def json_path(path: Path) -> bool:
     return False
 
 
-def build_paramset_key(elem: et.Element) -> Tuple[str, str, str]:
+def build_paramset_key(elem: et.Element, seperator: str = "::") -> Tuple[str, str, str]:
     """
     Function to extract the appropriate suffix from a given parameterset xml element. Returns the
     element type (either for subpopulation, mode or activity) and new key. This key is used to
@@ -952,7 +954,7 @@ def build_paramset_key(elem: et.Element) -> Tuple[str, str, str]:
         (uid,) = [
             p.attrib["value"] for p in elem.xpath("./param[@name='activityType']")
         ]
-        key = paramset_type + ":" + uid
+        key = paramset_type + seperator + uid
         return paramset_type, key, uid
 
     if paramset_type in [
@@ -962,14 +964,14 @@ def build_paramset_key(elem: et.Element) -> Tuple[str, str, str]:
         "modeRangeRestrictionSet",
     ]:
         (uid,) = [p.attrib["value"] for p in elem.xpath("./param[@name='mode']")]
-        key = paramset_type + ":" + uid
+        key = paramset_type + seperator + uid
         return paramset_type, key, uid
 
     if paramset_type in ["scoringParameters"]:
         (uid,) = [
             p.attrib["value"] for p in elem.xpath("./param[@name='subpopulation']")
         ]
-        key = paramset_type + ":" + uid
+        key = paramset_type + seperator + uid
         return paramset_type, key, uid
 
     if paramset_type in ["strategysettings"]:
@@ -979,15 +981,24 @@ def build_paramset_key(elem: et.Element) -> Tuple[str, str, str]:
         (strategy,) = [
             p.attrib["value"] for p in elem.xpath("./param[@name='strategyName']")
         ]
-        uid = subpop + ":" + strategy
-        key = paramset_type + ":" + uid
+        uid = subpop + seperator + strategy
+        key = paramset_type + seperator + uid
         return paramset_type, key, uid
 
     if paramset_type in ["modeMapping"]:
         (uid,) = [
             p.attrib["value"] for p in elem.xpath("./param[@name='passengerMode']")
         ]
-        key = paramset_type + ":" + uid
+        key = paramset_type + seperator + uid
+        return paramset_type, key, uid
+
+    if ":" in paramset_type:
+        """
+        special cases when matsim config has semicolons in paramerset such as selector:MultinomialLogit
+        and modeAvailability:Car from DMC module
+        """
+        uid = paramset_type.split(":")[-1]
+        key = paramset_type
         return paramset_type, key, uid
 
     raise ValueError(
@@ -1015,13 +1026,13 @@ def sets_diff(self: list, other: list, name: str, loc: str) -> list:
     return diffs
 
 
-def get_paramset_type(key: str) -> str:
+def get_paramset_type(key: str, seperator: str = "::") -> str:
     """
     Return parameterset type from unique key.
     :param key: str
     :return: str
     """
-    return key.split(":")[0]
+    return key.split(seperator)[0]
 
 
 def get_params_search(dic: dict, target: str) -> dict: