From 441236ba61eaac382393ab6dc84b99163b733eb2 Mon Sep 17 00:00:00 2001 From: fred Date: Wed, 8 Nov 2023 16:37:41 +0000 Subject: [PATCH 1/7] refactor for new paramset seperator --- mc/__init__.py | 2 +- mc/base.py | 44 +- mc/build.py | 527 +++++++++++--------- mc/report.py | 115 ++--- tests/test_autostep.py | 671 +++++++++++++++++--------- tests/test_base_config.py | 34 +- tests/test_data/test_config.json | 58 +-- tests/test_data/test_diff.json | 36 +- tests/test_data/test_overrides.json | 12 +- tests/test_data/test_temp_config.json | 58 +-- tests/test_default_config.py | 2 +- tests/test_find.py | 22 +- tests/test_step.py | 156 ++++-- tests/test_valid_keys.py | 76 +-- 14 files changed, 1084 insertions(+), 729 deletions(-) diff --git a/mc/__init__.py b/mc/__init__.py index bbda01d..c54a518 100644 --- a/mc/__init__.py +++ b/mc/__init__.py @@ -8,6 +8,6 @@ __version__ = "1.0.2" _ROOT = Path(os.path.abspath(os.path.dirname(__file__))) -_DEFAULTS_DIR = _ROOT / 'default_data' +_DEFAULTS_DIR = _ROOT / "default_data" if not _DEFAULTS_DIR.is_dir(): raise NotADirectoryError(f"Default data dir not found at {_DEFAULTS_DIR}") diff --git a/mc/base.py b/mc/base.py index a6dd628..31fbaae 100644 --- a/mc/base.py +++ b/mc/base.py @@ -571,9 +571,9 @@ def __getitem__(self, key): return self.params[key].value if key in self.parametersets: return self.parametersets[key] - if key + ":default" in self.parametersets: + if key + "::default" in self.parametersets: print("WARNING assuming 'default' required") - return self.parametersets[key + ":default"] + return self.parametersets[key + "::default"] # try to collect list of paramsets collected = [] @@ -654,9 +654,9 @@ def get(self, key, default=None): return self.params[key].value if key in self.parametersets: return self.parametersets[key] - if key + ":default" in self.parametersets: + if key + "::default" in self.parametersets: print("WARNING assuming 'default' required") - return self.parametersets[key + ":default"] + return self.parametersets[key + "::default"] return default @@ -733,9 +733,9 @@ def __getitem__(self, key): return self.params[key].value if key in self.parametersets: return self.parametersets[key] - if key + ":default" in self.parametersets: - print("WARNING assuming ':default' required") - return self.parametersets[key + ":default"] + if key + "::default" in self.parametersets: + print("WARNING assuming '::default' required") + return self.parametersets[key + "::default"] # try to collect list of paramsets collected = [] @@ -834,9 +834,9 @@ def get(self, key, default=None): return self.params[key].value if key in self.parametersets: return self.parametersets[key] - if key + ":default" in self.parametersets: + if key + "::default" in self.parametersets: print("WARNING assuming 'default' required") - return self.parametersets[key + ":default"] + return self.parametersets[key + "::default"] return default @@ -886,7 +886,7 @@ def __eq__(self, other): return True -def specials_snap(a, b, divider=":", ignore="*"): +def specials_snap(a, b, divider="::", ignore="*"): """ Special function to check for key matches with consideration of special character '*' that represents 'all'. @@ -938,7 +938,7 @@ def json_path(path: Path) -> bool: return False -def build_paramset_key(elem: et.Element) -> Tuple[str, str, str]: +def build_paramset_key(elem: et.Element, seperator: str = "::") -> Tuple[str, str, str]: """ Function to extract the appropriate suffix from a given parameterset xml element. Returns the element type (either for subpopulation, mode or activity) and new key. This key is used to @@ -952,7 +952,7 @@ def build_paramset_key(elem: et.Element) -> Tuple[str, str, str]: (uid,) = [ p.attrib["value"] for p in elem.xpath("./param[@name='activityType']") ] - key = paramset_type + ":" + uid + key = paramset_type + seperator + uid return paramset_type, key, uid if paramset_type in [ @@ -962,14 +962,14 @@ def build_paramset_key(elem: et.Element) -> Tuple[str, str, str]: "modeRangeRestrictionSet", ]: (uid,) = [p.attrib["value"] for p in elem.xpath("./param[@name='mode']")] - key = paramset_type + ":" + uid + key = paramset_type + seperator + uid return paramset_type, key, uid if paramset_type in ["scoringParameters"]: (uid,) = [ p.attrib["value"] for p in elem.xpath("./param[@name='subpopulation']") ] - key = paramset_type + ":" + uid + key = paramset_type + seperator + uid return paramset_type, key, uid if paramset_type in ["strategysettings"]: @@ -979,15 +979,21 @@ def build_paramset_key(elem: et.Element) -> Tuple[str, str, str]: (strategy,) = [ p.attrib["value"] for p in elem.xpath("./param[@name='strategyName']") ] - uid = subpop + ":" + strategy - key = paramset_type + ":" + uid + uid = subpop + seperator + strategy + key = paramset_type + seperator + uid return paramset_type, key, uid if paramset_type in ["modeMapping"]: (uid,) = [ p.attrib["value"] for p in elem.xpath("./param[@name='passengerMode']") ] - key = paramset_type + ":" + uid + key = paramset_type + seperator + uid + return paramset_type, key, uid + + if ":" in paramset_type: + """special cases fpr selector:MultinomialLogit and modeAvailability:Car from DMC mod""" + uid = paramset_type.split(":")[-1] + key = paramset_type + seperator + uid return paramset_type, key, uid raise ValueError( @@ -1015,13 +1021,13 @@ def sets_diff(self: list, other: list, name: str, loc: str) -> list: return diffs -def get_paramset_type(key: str) -> str: +def get_paramset_type(key: str, seperator: str = "::") -> str: """ Return parameterset type from unique key. :param key: str :return: str """ - return key.split(":")[0] + return key.split(seperator)[0] def get_params_search(dic: dict, target: str) -> dict: diff --git a/mc/build.py b/mc/build.py index ca159d9..8fe6f86 100644 --- a/mc/build.py +++ b/mc/build.py @@ -21,24 +21,27 @@ class DefaultConfig(BaseConfig): """ Default configuration. """ + def __init__(self): - super().__init__(path=_DEFAULTS_DIR / 'default_config.xml') + super().__init__(path=_DEFAULTS_DIR / "default_config.xml") class MultiModalDefaultConfig(BaseConfig): """ Default configuration. """ + def __init__(self): - super().__init__(path=_DEFAULTS_DIR / 'multimodal_default_config.xml') + super().__init__(path=_DEFAULTS_DIR / "multimodal_default_config.xml") class MultiModalTestTownConfig(BaseConfig): """ Test config used for Multi-modal Test Town (C) scenario. """ + def __init__(self): - super().__init__(path=_DEFAULTS_DIR / 'multimodal_test_town_C_config.xml') + super().__init__(path=_DEFAULTS_DIR / "multimodal_test_town_C_config.xml") """ @@ -50,123 +53,176 @@ class TestConfig(BaseConfig): """ Test Configuration. """ + def __init__(self): super().__init__() # build paths - output_dir = Path(os.path.join('.', 'test_outputs')) - input_dir = Path(os.path.join('.', 'test_inputs')) + output_dir = Path(os.path.join(".", "test_outputs")) + input_dir = Path(os.path.join(".", "test_inputs")) - attributes_path = input_dir / 'attributes.xml' - plans_path = input_dir / 'population.xml' - schedule_path = input_dir / 'transitschedule.xml' - vehicles_path = input_dir / 'transitVehicles.xml' - network_path = input_dir / 'network.xml' + attributes_path = input_dir / "attributes.xml" + plans_path = input_dir / "population.xml" + schedule_path = input_dir / "transitschedule.xml" + vehicles_path = input_dir / "transitVehicles.xml" + network_path = input_dir / "network.xml" # build config - self['global']['randomSeed'] = '4711' - self['global']['coordinateSystem'] = 'EPSG:27700' - self['global']['numberOfThreads'] = '1' - - self['network']['inputNetworkFile'] = network_path.as_posix() - - self['plans']['inputPlansFile'] = plans_path.as_posix() - self['plans']['inputPersonAttributesFile'] = attributes_path.as_posix() - self['plans']['subpopulationAttributeName'] = 'subpopulation' - - self['transit']['useTransit'] = 'true' - self['transit']['transitScheduleFile'] = schedule_path.as_posix() - self['transit']['vehiclesFile'] = vehicles_path.as_posix() - self['transit']['transitModes'] = 'pt' - - self['TimeAllocationMutator']['mutationRange'] = '1000.0' - - self['controler']['outputDirectory'] = output_dir.as_posix() - self['controler']['firstIteration'] = '0' - self['controler']['lastIteration'] = '0' - self['controler']['mobsim'] = 'qsim' - self['controler']['overwriteFiles'] = 'overwriteExistingFiles' - - self['subtourModeChoice']['chainBasedModes'] = 'car,bike' - self['subtourModeChoice']['modes'] = 'car,pt,walk,bike' - - self['qsim']['startTime'] = '00:00:00' - self['qsim']['endTime'] = '24:00:00' - self['qsim']['flowCapacityFactor'] = '1' - self['qsim']['storageCapacityFactor'] = '1' - self['qsim']['mainMode'] = 'car,bus' - self['qsim']['numberOfThreads'] = '1' - - self['planCalcScore']['BrainExpBeta'] = '1' - - for subpop in ['low income', 'medium income', 'high income', 'freight', 'default']: - - self['planCalcScore'][f'scoringParameters:{subpop}']['lateArrival'] = '-18' - self['planCalcScore'][f'scoringParameters:{subpop}']['earlyDeparture'] = '-0.0' - self['planCalcScore'][f'scoringParameters:{subpop}']['marginalUtilityOfMoney'] = '1' - self['planCalcScore'][f'scoringParameters:{subpop}']['performing'] = '+6' - self['planCalcScore'][f'scoringParameters:{subpop}']['subpopulation'] = subpop - self['planCalcScore'][f'scoringParameters:{subpop}']['waiting'] = '-0' - self['planCalcScore'][f'scoringParameters:{subpop}']['waitingPt'] = '-2' - self['planCalcScore'][f'scoringParameters:{subpop}']['utilityOfLineSwitch'] = '-1' - - modes = ['car', 'pt', 'walk', 'bike'] - mut_hrs = ['-5', '-5', '-12', '-12'] - mdrs = ['-0.0005', '-0.001', '0', '0'] + self["global"]["randomSeed"] = "4711" + self["global"]["coordinateSystem"] = "EPSG:27700" + self["global"]["numberOfThreads"] = "1" + + self["network"]["inputNetworkFile"] = network_path.as_posix() + + self["plans"]["inputPlansFile"] = plans_path.as_posix() + self["plans"]["inputPersonAttributesFile"] = attributes_path.as_posix() + self["plans"]["subpopulationAttributeName"] = "subpopulation" + + self["transit"]["useTransit"] = "true" + self["transit"]["transitScheduleFile"] = schedule_path.as_posix() + self["transit"]["vehiclesFile"] = vehicles_path.as_posix() + self["transit"]["transitModes"] = "pt" + + self["TimeAllocationMutator"]["mutationRange"] = "1000.0" + + self["controler"]["outputDirectory"] = output_dir.as_posix() + self["controler"]["firstIteration"] = "0" + self["controler"]["lastIteration"] = "0" + self["controler"]["mobsim"] = "qsim" + self["controler"]["overwriteFiles"] = "overwriteExistingFiles" + + self["subtourModeChoice"]["chainBasedModes"] = "car,bike" + self["subtourModeChoice"]["modes"] = "car,pt,walk,bike" + + self["qsim"]["startTime"] = "00:00:00" + self["qsim"]["endTime"] = "24:00:00" + self["qsim"]["flowCapacityFactor"] = "1" + self["qsim"]["storageCapacityFactor"] = "1" + self["qsim"]["mainMode"] = "car,bus" + self["qsim"]["numberOfThreads"] = "1" + + self["planCalcScore"]["BrainExpBeta"] = "1" + + for subpop in [ + "low income", + "medium income", + "high income", + "freight", + "default", + ]: + self["planCalcScore"][f"scoringParameters::{subpop}"]["lateArrival"] = "-18" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + "earlyDeparture" + ] = "-0.0" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + "marginalUtilityOfMoney" + ] = "1" + self["planCalcScore"][f"scoringParameters::{subpop}"]["performing"] = "+6" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + "subpopulation" + ] = subpop + self["planCalcScore"][f"scoringParameters::{subpop}"]["waiting"] = "-0" + self["planCalcScore"][f"scoringParameters::{subpop}"]["waitingPt"] = "-2" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + "utilityOfLineSwitch" + ] = "-1" + + modes = ["car", "pt", "walk", "bike"] + mut_hrs = ["-5", "-5", "-12", "-12"] + mdrs = ["-0.0005", "-0.001", "0", "0"] for mode, mut_hr, mdr in zip(modes, mut_hrs, mdrs): - self['planCalcScore'][f'scoringParameters:{subpop}'][f'modeParams:{mode}']['mode'] = mode - self['planCalcScore'][f'scoringParameters:{subpop}'][f'modeParams:{mode}']['constant'] = '0' - self['planCalcScore'][f'scoringParameters:{subpop}'][ - f'modeParams:{mode}']['marginalUtilityOfDistance_util_m'] = '0' - self['planCalcScore'][f'scoringParameters:{subpop}'][ - f'modeParams:{mode}']['marginalUtilityOfTraveling_util_hr'] = mut_hr - self['planCalcScore'][f'scoringParameters:{subpop}'][f'modeParams:{mode}']['monetaryDistanceRate'] = mdr - - activities = ['home', 'work', 'depo', 'dropoff_1', 'dropoff_2', 'dropoff_3'] - typ_durs = ['12:00:00', '08:30:00', '12:00:00', '00:15:00', '00:15:00', '00:15:00'] - min_durs = ['08:00:00', '08:00:00', '08:00:00', '00:10:00', '00:10:00', '00:10:00'] + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["mode"] = mode + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["constant"] = "0" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["marginalUtilityOfDistance_util_m"] = "0" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["marginalUtilityOfTraveling_util_hr"] = mut_hr + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["monetaryDistanceRate"] = mdr + + activities = ["home", "work", "depo", "dropoff_1", "dropoff_2", "dropoff_3"] + typ_durs = [ + "12:00:00", + "08:30:00", + "12:00:00", + "00:15:00", + "00:15:00", + "00:15:00", + ] + min_durs = [ + "08:00:00", + "08:00:00", + "08:00:00", + "00:10:00", + "00:10:00", + "00:10:00", + ] for act, typ_dur, min_dur in zip(activities, typ_durs, min_durs): - self['planCalcScore'][f'scoringParameters:{subpop}'][f'activityParams:{act}']['activityType'] = act - self['planCalcScore'][f'scoringParameters:{subpop}'][f'activityParams:{act}']['priority'] = '1' - self['planCalcScore'][f'scoringParameters:{subpop}'][f'activityParams:{act}']['typicalDuration'] \ - = typ_dur - self['planCalcScore'][f'scoringParameters:{subpop}'][f'activityParams:{act}']['minimalDuration'] \ - = min_dur - - self['strategy']['maxAgentPlanMemorySize'] = '3' - - for subpop in ['low income', 'medium income', 'high income', 'freight']: - strategies = ['SubtourModeChoice', 'ReRoute', 'TimeAllocationMutator', 'ChangeExpBeta'] - weights = ['0.1', '0.1', '0.1', '0.7'] + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"activityParams::{act}" + ]["activityType"] = act + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"activityParams::{act}" + ]["priority"] = "1" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"activityParams::{act}" + ]["typicalDuration"] = typ_dur + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"activityParams::{act}" + ]["minimalDuration"] = min_dur + + self["strategy"]["maxAgentPlanMemorySize"] = "3" + + for subpop in ["low income", "medium income", "high income", "freight"]: + strategies = [ + "SubtourModeChoice", + "ReRoute", + "TimeAllocationMutator", + "ChangeExpBeta", + ] + weights = ["0.1", "0.1", "0.1", "0.7"] for strategy, weight in zip(strategies, weights): + self["strategy"][f"strategysettings::{subpop}::{strategy}"][ + "strategyName" + ] = strategy + self["strategy"][f"strategysettings::{subpop}::{strategy}"][ + "subpopulation" + ] = subpop + self["strategy"][f"strategysettings::{subpop}::{strategy}"][ + "weight" + ] = weight - self['strategy'][f'strategysettings:{subpop}:{strategy}']['strategyName'] = strategy - self['strategy'][f'strategysettings:{subpop}:{strategy}']['subpopulation'] = subpop - self['strategy'][f'strategysettings:{subpop}:{strategy}']['weight'] = weight - - self['strategy']['strategysettings:freight:SubtourModeChoice']['weight'] = '0' + self["strategy"]["strategysettings:freight:SubtourModeChoice"]["weight"] = "0" - self['transitRouter']['additionalTransferTime'] = '1' - self['transitRouter']['extensionRadius'] = '100' - self['transitRouter']['maxBeelineWalkConnectionDistance'] = '500' - self['transitRouter']['searchRadius'] = '1000' + self["transitRouter"]["additionalTransferTime"] = "1" + self["transitRouter"]["extensionRadius"] = "100" + self["transitRouter"]["maxBeelineWalkConnectionDistance"] = "500" + self["transitRouter"]["searchRadius"] = "1000" class BuildConfig(BaseConfig): """ Bespoke Config. """ + def __init__( - self, - input_dir: Path = Path('inputs'), - output_dir: Path = Path('outputs'), - sample: float = 0.01, - epsg: int = 27700, - subpops: Tuple[str] = ('low', 'medium', 'high', 'default'), - modes: Tuple[str] = ('car', 'pt', 'bike', 'walk'), - acts: Tuple[str] = ('home', 'work', 'education', 'other') + self, + input_dir: Path = Path("inputs"), + output_dir: Path = Path("outputs"), + sample: float = 0.01, + epsg: int = 27700, + subpops: Tuple[str] = ("low", "medium", "high", "default"), + modes: Tuple[str] = ("car", "pt", "bike", "walk"), + acts: Tuple[str] = ("home", "work", "education", "other"), ): """ Config Builder. @@ -181,174 +237,185 @@ def __init__( super().__init__() - defaults_config = BaseConfig(path=_DEFAULTS_DIR / 'default_config.xml') + defaults_config = BaseConfig(path=_DEFAULTS_DIR / "default_config.xml") subpops = list(subpops) - if 'default' not in subpops: - subpops += ['default'] + if "default" not in subpops: + subpops += ["default"] modes = list(modes) - for critical_mode in ['walk', 'access_walk', 'egress_walk']: + for critical_mode in ["walk", "access_walk", "egress_walk"]: if critical_mode not in modes: modes += [critical_mode] default_mode_scoring = { - 'car': { - 'mut_hr': '-6', - 'mdr': '-0.0002' - }, - 'pt': { - 'mut_hr': '-6', - 'mdr': '-0.0005' - }, - 'walk': { - 'mut_hr': '-12', - 'mdr': '0' - }, - 'access_walk': { - 'mut_hr': '-12', - 'mdr': '0' - }, - 'egress_walk': { - 'mut_hr': '-12', - 'mdr': '0' - }, - 'bike': { - 'mut_hr': '-12', - 'mdr': '0' - } + "car": {"mut_hr": "-6", "mdr": "-0.0002"}, + "pt": {"mut_hr": "-6", "mdr": "-0.0005"}, + "walk": {"mut_hr": "-12", "mdr": "0"}, + "access_walk": {"mut_hr": "-12", "mdr": "0"}, + "egress_walk": {"mut_hr": "-12", "mdr": "0"}, + "bike": {"mut_hr": "-12", "mdr": "0"}, } default_activity_scoring = { - 'home': { - 'typ_dur': '12:00:00', - 'min_dur': '08:00:00' - }, - 'work': { - 'typ_dur': '08:30:00', - 'min_dur': '08:00:00' - }, - 'education': { - 'typ_dur': '08:00:00', - 'min_dur': '06:00:00' - }, - 'depo': { - 'typ_dur': '12:00:00', - 'min_dur': '08:00:00' - }, - 'delivery': { - 'typ_dur': '00:20:00', - 'min_dur': '00:10:00' - }, - 'other': { - 'typ_dur': '00:15:00', - 'min_dur': '00:10:00' - } + "home": {"typ_dur": "12:00:00", "min_dur": "08:00:00"}, + "work": {"typ_dur": "08:30:00", "min_dur": "08:00:00"}, + "education": {"typ_dur": "08:00:00", "min_dur": "06:00:00"}, + "depo": {"typ_dur": "12:00:00", "min_dur": "08:00:00"}, + "delivery": {"typ_dur": "00:20:00", "min_dur": "00:10:00"}, + "other": {"typ_dur": "00:15:00", "min_dur": "00:10:00"}, } # build paths - attributes_path = input_dir / 'attributes.xml.gz' - plans_path = input_dir / 'plans.xml.gz' - schedule_path = input_dir / 'schedule.xml.gz' - vehicles_path = input_dir / 'vehicles.xml.gz' - network_path = input_dir / 'network.xml.gz' + attributes_path = input_dir / "attributes.xml.gz" + plans_path = input_dir / "plans.xml.gz" + schedule_path = input_dir / "schedule.xml.gz" + vehicles_path = input_dir / "vehicles.xml.gz" + network_path = input_dir / "network.xml.gz" # build config - self['global'] = defaults_config['global'] - self['global']['coordinateSystem'] = f"EPSG:{epsg}" + self["global"] = defaults_config["global"] + self["global"]["coordinateSystem"] = f"EPSG:{epsg}" - self['network']['inputNetworkFile'] = network_path.as_posix() + self["network"]["inputNetworkFile"] = network_path.as_posix() - self['plans']['inputPlansFile'] = plans_path.as_posix() - self['plans']['inputPersonAttributesFile'] = attributes_path.as_posix() - self['plans']['subpopulationAttributeName'] = 'subpopulation' + self["plans"]["inputPlansFile"] = plans_path.as_posix() + self["plans"]["inputPersonAttributesFile"] = attributes_path.as_posix() + self["plans"]["subpopulationAttributeName"] = "subpopulation" - self['transit']['useTransit'] = 'true' - self['transit']['transitScheduleFile'] = schedule_path.as_posix() - self['transit']['vehiclesFile'] = vehicles_path.as_posix() - self['transit']['transitModes'] = 'pt' - if 'pt' not in modes: + self["transit"]["useTransit"] = "true" + self["transit"]["transitScheduleFile"] = schedule_path.as_posix() + self["transit"]["vehiclesFile"] = vehicles_path.as_posix() + self["transit"]["transitModes"] = "pt" + if "pt" not in modes: raise UserWarning('Expected to see mode "pt"') - self['TimeAllocationMutator']['mutationRange'] = '1000.0' + self["TimeAllocationMutator"]["mutationRange"] = "1000.0" - self['controler'] = defaults_config['controler'] - self['controler']['outputDirectory'] = output_dir.as_posix() + self["controler"] = defaults_config["controler"] + self["controler"]["outputDirectory"] = output_dir.as_posix() - chained_modes = ['car', 'bike'] + chained_modes = ["car", "bike"] config_chained_modes = [m for m in modes if m in chained_modes] - chained_mode_str = ','.join(config_chained_modes) + chained_mode_str = ",".join(config_chained_modes) - self['subtourModeChoice']['chainBasedModes'] = chained_mode_str - self['subtourModeChoice']['modes'] = ','.join(modes) + self["subtourModeChoice"]["chainBasedModes"] = chained_mode_str + self["subtourModeChoice"]["modes"] = ",".join(modes) - self['qsim']['startTime'] = '00:00:00' - self['qsim']['endTime'] = '24:00:00' - self['qsim']['flowCapacityFactor'] = str(sample) - self['qsim']['storageCapacityFactor'] = str(sample) - self['qsim']['mainMode'] = 'car,bus' - self['qsim']['numberOfThreads'] = '32' + self["qsim"]["startTime"] = "00:00:00" + self["qsim"]["endTime"] = "24:00:00" + self["qsim"]["flowCapacityFactor"] = str(sample) + self["qsim"]["storageCapacityFactor"] = str(sample) + self["qsim"]["mainMode"] = "car,bus" + self["qsim"]["numberOfThreads"] = "32" - self['planCalcScore']['BrainExpBeta'] = '1' + self["planCalcScore"]["BrainExpBeta"] = "1" for subpop in subpops: - - self['planCalcScore'][f'scoringParameters:{subpop}']['lateArrival'] = '-18' - self['planCalcScore'][f'scoringParameters:{subpop}']['earlyDeparture'] = '-18.0' - self['planCalcScore'][f'scoringParameters:{subpop}']['marginalUtilityOfMoney'] = '1' - self['planCalcScore'][f'scoringParameters:{subpop}']['performing'] = '+6' - self['planCalcScore'][f'scoringParameters:{subpop}']['subpopulation'] = subpop - self['planCalcScore'][f'scoringParameters:{subpop}']['waiting'] = '-1' - self['planCalcScore'][f'scoringParameters:{subpop}']['waitingPt'] = '-1' - self['planCalcScore'][f'scoringParameters:{subpop}']['utilityOfLineSwitch'] = '-1' + self["planCalcScore"][f"scoringParameters::{subpop}"]["lateArrival"] = "-18" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + "earlyDeparture" + ] = "-18.0" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + "marginalUtilityOfMoney" + ] = "1" + self["planCalcScore"][f"scoringParameters::{subpop}"]["performing"] = "+6" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + "subpopulation" + ] = subpop + self["planCalcScore"][f"scoringParameters::{subpop}"]["waiting"] = "-1" + self["planCalcScore"][f"scoringParameters::{subpop}"]["waitingPt"] = "-1" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + "utilityOfLineSwitch" + ] = "-1" for mode in modes: - self['planCalcScore'][f'scoringParameters:{subpop}'][f'modeParams:{mode}']['mode'] = mode - self['planCalcScore'][f'scoringParameters:{subpop}'][f'modeParams:{mode}']['constant'] = '0' - self['planCalcScore'][f'scoringParameters:{subpop}'][f'modeParams:{mode}'][ - 'marginalUtilityOfDistance_util_m'] = '0' - self['planCalcScore'][f'scoringParameters:{subpop}'][f'modeParams:{mode}'][ - 'marginalUtilityOfTraveling_util_hr'] = \ - default_mode_scoring.get(mode, default_mode_scoring['car'])['mut_hr'] - self['planCalcScore'][f'scoringParameters:{subpop}'][f'modeParams:{mode}'][ - 'monetaryDistanceRate'] = default_mode_scoring.get(mode, default_mode_scoring['car'])['mdr'] + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["mode"] = mode + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["constant"] = "0" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["marginalUtilityOfDistance_util_m"] = "0" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["marginalUtilityOfTraveling_util_hr"] = default_mode_scoring.get( + mode, default_mode_scoring["car"] + )[ + "mut_hr" + ] + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"modeParams::{mode}" + ]["monetaryDistanceRate"] = default_mode_scoring.get( + mode, default_mode_scoring["car"] + )[ + "mdr" + ] for act in acts: - self['planCalcScore'][f'scoringParameters:{subpop}'][f'activityParams:{act}']['activityType'] = act - self['planCalcScore'][f'scoringParameters:{subpop}'][f'activityParams:{act}']['priority'] = '1' - self['planCalcScore'][f'scoringParameters:{subpop}'][f'activityParams:{act}'][ - 'typicalDuration'] = \ - default_activity_scoring.get(mode, default_activity_scoring['other'])['typ_dur'] - self['planCalcScore'][f'scoringParameters:{subpop}'][f'activityParams:{act}'][ - 'minimalDuration'] = \ - default_activity_scoring.get(mode, default_activity_scoring['other'])['min_dur'] - - self['strategy']['maxAgentPlanMemorySize'] = '5' + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"activityParams::{act}" + ]["activityType"] = act + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"activityParams::{act}" + ]["priority"] = "1" + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"activityParams::{act}" + ]["typicalDuration"] = default_activity_scoring.get( + mode, default_activity_scoring["other"] + )[ + "typ_dur" + ] + self["planCalcScore"][f"scoringParameters::{subpop}"][ + f"activityParams::{act}" + ]["minimalDuration"] = default_activity_scoring.get( + mode, default_activity_scoring["other"] + )[ + "min_dur" + ] + + self["strategy"]["maxAgentPlanMemorySize"] = "5" for subpop in subpops: - strategies = ['SubtourModeChoice', 'ReRoute', 'TimeAllocationMutator', 'ChangeExpBeta'] - weights = ['0.1', '0.1', '0.1', '0.7'] + strategies = [ + "SubtourModeChoice", + "ReRoute", + "TimeAllocationMutator", + "ChangeExpBeta", + ] + weights = ["0.1", "0.1", "0.1", "0.7"] for strategy, weight in zip(strategies, weights): - self['strategy'][f'strategysettings:{subpop}:{strategy}']['strategyName'] = strategy - self['strategy'][f'strategysettings:{subpop}:{strategy}']['subpopulation'] = subpop - self['strategy'][f'strategysettings:{subpop}:{strategy}']['weight'] = weight - - if 'freight' in subpops: - self['strategy']['strategysettings:freight:SubtourModeChoice']['weight'] = '0' - - self['transitRouter']['additionalTransferTime'] = '1' - self['transitRouter']['extensionRadius'] = '100' - self['transitRouter']['maxBeelineWalkConnectionDistance'] = '500' - self['transitRouter']['searchRadius'] = '1000' - - self['planscalcroute']['teleportedModeParameters:access_walk'] = \ - defaults_config['planscalcroute']['teleportedModeParameters:access_walk'] + self["strategy"][f"strategysettings::{subpop}::{strategy}"][ + "strategyName" + ] = strategy + self["strategy"][f"strategysettings::{subpop}::{strategy}"][ + "subpopulation" + ] = subpop + self["strategy"][f"strategysettings::{subpop}::{strategy}"][ + "weight" + ] = weight + + if "freight" in subpops: + self["strategy"]["strategysettings::freight::SubtourModeChoice"][ + "weight" + ] = "0" + + self["transitRouter"]["additionalTransferTime"] = "1" + self["transitRouter"]["extensionRadius"] = "100" + self["transitRouter"]["maxBeelineWalkConnectionDistance"] = "500" + self["transitRouter"]["searchRadius"] = "1000" + + self["planscalcroute"][ + "teleportedModeParameters::access_walk" + ] = defaults_config["planscalcroute"]["teleportedModeParameters::access_walk"] CONFIG_MAP = { - 'empty': Config, - 'default': DefaultConfig, - 'multimodal_default': MultiModalDefaultConfig, - 'test': TestConfig, - 'multimodal_test': MultiModalTestTownConfig, + "empty": Config, + "default": DefaultConfig, + "multimodal_default": MultiModalDefaultConfig, + "test": TestConfig, + "multimodal_test": MultiModalTestTownConfig, } diff --git a/mc/report.py b/mc/report.py index 8fc1df3..933ab03 100644 --- a/mc/report.py +++ b/mc/report.py @@ -21,16 +21,17 @@ def add_directory_to_report(config): # add paths of the output diretory message.append("{:=^150s}".format("output directory")) - message.append( - f"output_directory:{config['controler']['outputDirectory']}") + message.append(f"output_directory:{config['controler']['outputDirectory']}") # add mobsim setting summary message.append("{:=^150s}".format("mobsim setting")) message.append(f"mobsim:{config['controler']['mobsim']}") message.append( - f"Flow_Capacity_Factor:{config[config['controler']['mobsim']]['flowCapacityFactor']}") + f"Flow_Capacity_Factor:{config[config['controler']['mobsim']]['flowCapacityFactor']}" + ) message.append( - f"Storage_Capacity_Factor:{config[config['controler']['mobsim']]['storageCapacityFactor']}") + f"Storage_Capacity_Factor:{config[config['controler']['mobsim']]['storageCapacityFactor']}" + ) return message @@ -46,17 +47,19 @@ def add_scoring_to_report(config): message.append("") # Initialize a dictionary to store mode parameters parm = {} - for mode in (config['subtourModeChoice']['modes'].split(',')): - parm[mode] = ["mode_specific_constant", - "marginal_utility_of_distance", - "marginal_utility_of_traveling", - "monetary_distance_rate"] + for mode in config["subtourModeChoice"]["modes"].split(","): + parm[mode] = [ + "mode_specific_constant", + "marginal_utility_of_distance", + "marginal_utility_of_traveling", + "monetary_distance_rate", + ] subpopulation_set = {} - subpop = 'subpopulation: ' - for i in config['planCalcScore'].find('subpopulation'): + subpop = "subpopulation: " + for i in config["planCalcScore"].find("subpopulation"): subpopulation_set.setdefault(str(i.value), []) - subpop = subpop + str(i.value) + ',' + subpop = subpop + str(i.value) + "," for j in config.find("scoringParameters:" + str(i.value) + "/mode"): subpopulation_set[str(i.value)].append(j.value) @@ -66,8 +69,7 @@ def add_scoring_to_report(config): message.append(table_header) message.append("-" * (31 * (len(subpopulation_set) + 1))) # iterate through modes and subpopulations to create the table rows - for idx, mode in enumerate( - config['subtourModeChoice']['modes'].split(',')): + for idx, mode in enumerate(config["subtourModeChoice"]["modes"].split(",")): row_data = "|{:^30}|".format(mode) for subpop_name in subpopulation_set.keys(): row_data += "{:^30}|".format(" ") @@ -75,62 +77,44 @@ def add_scoring_to_report(config): message.append("-" * (31 * (len(subpopulation_set) + 1))) # iterate through the scoring parameters and add the values to the for row_idx, row_name in enumerate( - ["marginalUtilityOfMoney", "performing", "utilityOfLineSwitch"] + parm[mode]): + ["marginalUtilityOfMoney", "performing", "utilityOfLineSwitch"] + parm[mode] + ): row_data = "|{:^30}|".format(row_name) for subpop_name in subpopulation_set.keys(): if row_idx == 0: - cell_value = config['planCalcScore']['scoringParameters:' + - subpop_name]['marginalUtilityOfMoney'] + cell_value = config["planCalcScore"][ + "scoringParameters::" + subpop_name + ]["marginalUtilityOfMoney"] elif row_idx == 1: - cell_value = config['planCalcScore']['scoringParameters:' + - subpop_name]['performing'] + cell_value = config["planCalcScore"][ + "scoringParameters::" + subpop_name + ]["performing"] elif row_idx == 2: - cell_value = config['planCalcScore']['scoringParameters:' + - subpop_name]['utilityOfLineSwitch'] + cell_value = config["planCalcScore"][ + "scoringParameters::" + subpop_name + ]["utilityOfLineSwitch"] else: if mode not in subpopulation_set[subpop_name]: cell_value = "NA" else: if row_name == "mode_specific_constant": - cell_value = config[ - 'planCalcScore' - ][ - 'scoringParameters:' + subpop_name - ][ - 'modeParams:' + mode - ][ - 'constant' - ] + cell_value = config["planCalcScore"][ + "scoringParameters::" + subpop_name + ]["modeParams::" + mode]["constant"] elif row_name == "marginal_utility_of_distance": - cell_value = config[ - 'planCalcScore' - ][ - 'scoringParameters:' + subpop_name - ][ - 'modeParams:' + mode - ][ - 'marginalUtilityOfDistance_util_m' - ] + cell_value = config["planCalcScore"][ + "scoringParameters::" + subpop_name + ]["modeParams::" + mode]["marginalUtilityOfDistance_util_m"] elif row_name == "marginal_utility_of_traveling": - cell_value = config[ - 'planCalcScore' - ][ - 'scoringParameters:' + subpop_name - ][ - 'modeParams:' + mode - ][ - 'marginalUtilityOfTraveling_util_hr' + cell_value = config["planCalcScore"][ + "scoringParameters::" + subpop_name + ]["modeParams::" + mode][ + "marginalUtilityOfTraveling_util_hr" ] elif row_name == "monetary_distance_rate": - cell_value = config[ - 'planCalcScore' - ][ - 'scoringParameters:' + subpop_name - ][ - 'modeParams:' + mode - ][ - 'monetaryDistanceRate' - ] + cell_value = config["planCalcScore"][ + "scoringParameters::" + subpop_name + ]["modeParam:s:" + mode]["monetaryDistanceRate"] row_data += "{:^30}|".format(cell_value) message.append(row_data) message.append("-" * (31 * (len(subpopulation_set) + 1))) @@ -142,7 +126,7 @@ def write_text(text, output_path): """ Write the key information into a text file """ - textfile = open(os.path.join(output_path, 'simulation_report.txt'), 'w') + textfile = open(os.path.join(output_path, "simulation_report.txt"), "w") for element in text: textfile.write(element + "\n") textfile.close() @@ -152,13 +136,15 @@ def write_csv(data, output_path): """ Write the key information into a CSV file """ - with open(os.path.join(output_path, 'simulation_report.csv'), 'w', newline='') as csvfile: + with open( + os.path.join(output_path, "simulation_report.csv"), "w", newline="" + ) as csvfile: csv_writer = csv.writer(csvfile) for row in data: # Skip rows with dividing lines if "----" in row: continue - csv_writer.writerow(row.split('|')) + csv_writer.writerow(row.split("|")) def report_config(config, output_path): @@ -175,8 +161,9 @@ def find_log_files(file_name, root_dir): for root, dirs, files in os.walk(root_dir): for file in files: if file.startswith(file_name): - if str( - root)[-2:] == '/0': # find the file in zero iteration only in the simulation folder + if ( + str(root)[-2:] == "/0" + ): # find the file in zero iteration only in the simulation folder file_directory.append(os.path.join(root, file)) return file_directory @@ -185,14 +172,14 @@ def merge_files(file_directory, root_dir): """ Combine and update the matsim_overrides log from different simulation jobs """ - with open(os.path.join(root_dir, 'matsim_overrides_summary.log'), "w") as outfile: + with open(os.path.join(root_dir, "matsim_overrides_summary.log"), "w") as outfile: for f in file_directory: with open(f) as infile: text = infile.readlines() text.insert(0, f + "\n") text.insert(0, "log_path:") - text.insert(0, "-" * 100 + '\n') # split line - outfile.write(''.join(text)) + text.insert(0, "-" * 100 + "\n") # split line + outfile.write("".join(text)) def summarise_overrides_log(file_name, root_dir): diff --git a/tests/test_autostep.py b/tests/test_autostep.py index c7bcaea..f6f0afb 100644 --- a/tests/test_autostep.py +++ b/tests/test_autostep.py @@ -10,9 +10,10 @@ def test_construct_overrides_map_from_tuple(): - assert autostep.construct_override_map_from_tuple( - ('a','b','c','d') - ) == {'a':'b', 'c':'d'} + assert autostep.construct_override_map_from_tuple(("a", "b", "c", "d")) == { + "a": "b", + "c": "d", + } def test_construct_overrides_map_from_empty_tuple(): @@ -32,9 +33,9 @@ def test_path_config(): def test_set_innovation(config): - assert not config['strategy']['fractionOfIterationsToDisableInnovation'] == "0" + assert not config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0" autostep.set_innovation(config=config, new_fraction="0") - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == "0" + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0" @pytest.mark.parametrize( @@ -45,57 +46,67 @@ def test_set_innovation(config): (100, 20, 50, "0.8"), (200, 100, 100, "0.9"), (100, 200, 100, "0"), - ] + ], ) - - def test_set_cooling(config, total_iterations, start_index, step, new_fraction): - assert not config['strategy']['fractionOfIterationsToDisableInnovation'] == new_fraction - autostep.set_cooling(config=config, total_iterations=total_iterations, start_index=start_index, step=step, target=10) - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == new_fraction + assert ( + not config["strategy"]["fractionOfIterationsToDisableInnovation"] + == new_fraction + ) + autostep.set_cooling( + config=config, + total_iterations=total_iterations, + start_index=start_index, + step=step, + target=10, + ) + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == new_fraction def test_set_default_behaviours(config, tmp_path): - assert config['controler']['overwriteFiles'] == "failIfDirectoryExists" - assert config['controler']['writeEventsInterval'] == "1" - assert config['controler']['writePlansInterval'] == "50" + assert config["controler"]["overwriteFiles"] == "failIfDirectoryExists" + assert config["controler"]["writeEventsInterval"] == "1" + assert config["controler"]["writePlansInterval"] == "50" step = 5 autostep.set_default_behaviours(config, step, tmp_path) - assert config['controler']['overwriteFiles'] == "deleteDirectoryIfExists" - assert config['controler']['writeEventsInterval'] == "5" - assert config['controler']['writePlansInterval'] == "5" - assert config['parallelEventHandling']['eventsQueueSize'] == "270000000" + assert config["controler"]["overwriteFiles"] == "deleteDirectoryIfExists" + assert config["controler"]["writeEventsInterval"] == "5" + assert config["controler"]["writePlansInterval"] == "5" + assert config["parallelEventHandling"]["eventsQueueSize"] == "270000000" def test_set_write_path(config): autostep.set_write_path(config, Path("new/path")) - assert config['controler']['outputDirectory'] == "new/path" + assert config["controler"]["outputDirectory"] == "new/path" def test_autoset_input_paths(config): autostep.auto_set_input_paths(config, Path("test/ing")) - assert config['network']['inputNetworkFile'] == 'test/ing/output_network.xml.gz' - assert config['plans']['inputPlansFile'] == 'test/ing/output_plans.xml.gz' - assert config['vehicles']['vehiclesFile'] == 'test/ing/output_vehicles.xml.gz' - assert config['transit']['transitScheduleFile'] == 'test/ing/output_transitSchedule.xml.gz' - assert config['transit']['vehiclesFile'] == 'test/ing/output_transitVehicles.xml.gz' + assert config["network"]["inputNetworkFile"] == "test/ing/output_network.xml.gz" + assert config["plans"]["inputPlansFile"] == "test/ing/output_plans.xml.gz" + assert config["vehicles"]["vehiclesFile"] == "test/ing/output_vehicles.xml.gz" + assert ( + config["transit"]["transitScheduleFile"] + == "test/ing/output_transitSchedule.xml.gz" + ) + assert config["transit"]["vehiclesFile"] == "test/ing/output_transitVehicles.xml.gz" def test_fix_relative_input_paths_to_abs(test_path_config): seed_matsim_config_path = os.path.join(this_dir(), "test_data/test_path_config.xml") config_dir = os.path.dirname(seed_matsim_config_path) - assert test_path_config['network']['inputNetworkFile'] == "./test/network.xml" - assert test_path_config['plans']['inputPlansFile'] == "../test/population.xml.gz" + assert test_path_config["network"]["inputNetworkFile"] == "./test/network.xml" + assert test_path_config["plans"]["inputPlansFile"] == "../test/population.xml.gz" autostep.fix_relative_input_paths_to_abs(test_path_config, seed_matsim_config_path) - assert test_path_config['network']['inputNetworkFile'] == os.path.abspath( + assert test_path_config["network"]["inputNetworkFile"] == os.path.abspath( os.path.join(config_dir, "test", "network.xml") ) - assert test_path_config['plans']['inputPlansFile'] == os.path.abspath( + assert test_path_config["plans"]["inputPlansFile"] == os.path.abspath( os.path.join(config_dir, "..", "test", "population.xml.gz") ) @@ -103,11 +114,11 @@ def test_fix_relative_input_paths_to_abs(test_path_config): def test_absolute_input_paths_retained(test_path_config): seed_matsim_config_path = os.path.join(this_dir(), "test_data/test_path_config.xml") - assert test_path_config['transit']['vehiclesFile'] == "/test/vehicles.xml" + assert test_path_config["transit"]["vehiclesFile"] == "/test/vehicles.xml" autostep.fix_relative_input_paths_to_abs(test_path_config, seed_matsim_config_path) - assert test_path_config['transit']['vehiclesFile'] == os.path.abspath( + assert test_path_config["transit"]["vehiclesFile"] == os.path.abspath( os.path.join("/", "test", "vehicles.xml") ) @@ -115,72 +126,98 @@ def test_absolute_input_paths_retained(test_path_config): def test_fix_relative_home_input_paths_to_abs(test_path_config): seed_matsim_config_path = os.path.join(this_dir(), "test_data/test_path_config.xml") - assert test_path_config['transit']['transitScheduleFile'] == "~/test/schedule-merged.xml" + assert ( + test_path_config["transit"]["transitScheduleFile"] + == "~/test/schedule-merged.xml" + ) autostep.fix_relative_input_paths_to_abs(test_path_config, seed_matsim_config_path) - assert test_path_config['transit']['transitScheduleFile'] == os.path.expanduser( + assert test_path_config["transit"]["transitScheduleFile"] == os.path.expanduser( os.path.join("~", "test", "schedule-merged.xml") ) def test_set_iterations(config): autostep.set_iterations(config, first_iteration=0, last_iteration=10) - assert config['controler']['firstIteration'] == '0' - assert config['controler']['lastIteration'] == '10' + assert config["controler"]["firstIteration"] == "0" + assert config["controler"]["lastIteration"] == "10" def test_find_and_set_param(config): - autostep.find_and_set_overrides( - config, - {"modeParams:car/constant": "-1.0"} - ) - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "0.0" + autostep.find_and_set_overrides(config, {"modeParams::car/constant": "-1.0"}) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "0.0" + ) def test_find_and_set_params(config): autostep.find_and_set_overrides( config, { - "modeParams:car/constant": "-1.0", - "scoringParameters:unknown/modeParams:bus/constant": "-1.0" - } - ) - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" + "modeParams::car/constant": "-1.0", + "scoringParameters::unknown/modeParams::bus/constant": "-1.0", + }, + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) def test_find_and_set_params_with_overrides_log(config, tmp_path): - autostep.find_and_set_overrides( config, { - "modeParams:car/constant": "-1.0", - "scoringParameters:unknown/modeParams:bus/constant": "-1.0" - }, - log_root=tmp_path - ) + "modeParams::car/constant": "-1.0", + "scoringParameters::unknown/modeParams::bus/constant": "-1.0", + }, + log_root=tmp_path, + ) log_path = os.path.join(tmp_path, "matsim_overrides.log") assert os.path.exists(log_path) with open(log_path) as file: lines = file.readlines() print(lines) assert lines == [ - 'modeParams:car/constant: 0.0 -> -1.0\n', - 'modeParams:car/constant: 0.0 -> -1.0\n', - 'scoringParameters:unknown/modeParams:bus/constant: 0.0 -> -1.0\n' + "modeParams::car/constant: 0.0 -> -1.0\n", + "modeParams::car/constant: 0.0 -> -1.0\n", + "scoringParameters::unknown/modeParams::bus/constant: 0.0 -> -1.0\n", ] def test_finding_and_setting_bad_param_leaves_config_unchanged(config): cnfg = deepcopy(config) - autostep.find_and_set_overrides( - config, - {"modeParams:*/horseback": "-1.0"} - ) + autostep.find_and_set_overrides(config, {"modeParams::*/horseback": "-1.0"}) assert cnfg == config @@ -188,7 +225,9 @@ def test_autostep_config_first_iteration(tmp_path): in_file = os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml") out_dir = os.path.join(tmp_path, "10") out_file = os.path.join(tmp_path, "0", "matsim_config.xml") - overrides_path = os.path.join(os.path.dirname(__file__), "test_data", "test_overrides.json") + overrides_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_overrides.json" + ) autostep.autostep_config( sim_root=tmp_path, seed_matsim_config_path=in_file, @@ -196,69 +235,81 @@ def test_autostep_config_first_iteration(tmp_path): total_iterations="100", step="10", biteration_matsim_config_path=out_file, - overrides_path=overrides_path + overrides_path=overrides_path, ) assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == '10' - assert config['controler']['outputDirectory'] == out_dir - assert config['network']['inputNetworkFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "network.xml") - )) - assert config['plans']['inputPlansFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "population.xml.gz") - )) - assert config['vehicles']['vehiclesFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "all_vehicles.xml") - )) - assert config['transit']['transitScheduleFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "schedule-merged.xml") - )) - assert config['transit']['vehiclesFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "vehicles.xml") - )) - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == "0.8" + assert config["controler"]["lastIteration"] == "10" + assert config["controler"]["outputDirectory"] == out_dir + assert config["network"]["inputNetworkFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "network.xml")) + ) + assert config["plans"]["inputPlansFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "population.xml.gz")) + ) + assert config["vehicles"]["vehiclesFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "all_vehicles.xml")) + ) + assert config["transit"]["transitScheduleFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "schedule-merged.xml")) + ) + assert config["transit"]["vehiclesFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "vehicles.xml")) + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0.8" def test_nonexistent_optional_input_module_ignored_when_fixing_relative_paths(): - config_path = os.path.join(os.path.dirname(__file__), "test_data", "test_config_missing_all_vehicles.xml") + config_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_config_missing_all_vehicles.xml" + ) config = BaseConfig(config_path) - assert 'vehicles' not in config + assert "vehicles" not in config - autostep.fix_relative_input_paths_to_abs( - config, - config_path - ) + autostep.fix_relative_input_paths_to_abs(config, config_path) - assert 'vehicles' not in config + assert "vehicles" not in config def test_optional_vehicles_module_gets_relative_path_fixed(): - config_path = os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml") + config_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_config.xml" + ) config = BaseConfig(config_path) - assert not Path(config['vehicles']['vehiclesFile']).is_absolute() + assert not Path(config["vehicles"]["vehiclesFile"]).is_absolute() - autostep.fix_relative_input_paths_to_abs( - config, - config_path - ) + autostep.fix_relative_input_paths_to_abs(config, config_path) - assert Path(config['vehicles']['vehiclesFile']).is_absolute() + assert Path(config["vehicles"]["vehiclesFile"]).is_absolute() def test_missing_nonoptional_module_throws_key_error_when_trying_to_fix_relative_paths(): - config_path = os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml") + config_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_config.xml" + ) config = BaseConfig(config_path) - del config['network'] + del config["network"] with pytest.raises(KeyError) as error_info: - autostep.fix_relative_input_paths_to_abs( - config, - config_path - ) + autostep.fix_relative_input_paths_to_abs(config, config_path) assert "'inputNetworkFile' not found in params/sets" in str(error_info.value) @@ -267,7 +318,9 @@ def test_autostep_config(tmp_path): in_file = os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml") out_dir = os.path.join(tmp_path, "20") out_file = os.path.join(tmp_path, "10", "matsim_config.xml") - overrides_path = os.path.join(os.path.dirname(__file__), "test_data", "test_overrides.json") + overrides_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_overrides.json" + ) autostep.autostep_config( sim_root=tmp_path, seed_matsim_config_path=in_file, @@ -275,28 +328,57 @@ def test_autostep_config(tmp_path): total_iterations="100", step="10", biteration_matsim_config_path=out_file, - overrides_path=overrides_path + overrides_path=overrides_path, ) assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == '20' - assert config['controler']['outputDirectory'] == out_dir - assert config['network']['inputNetworkFile'] == os.path.join(tmp_path, "10", "output_network.xml.gz") - assert config['plans']['inputPlansFile'] == os.path.join(tmp_path, "10", "output_plans.xml.gz") - assert config['vehicles']['vehiclesFile'] == os.path.join(tmp_path, "10", "output_vehicles.xml.gz") - assert config['transit']['transitScheduleFile'] == os.path.join(tmp_path, "10", "output_transitSchedule.xml.gz") - assert config['transit']['vehiclesFile'] == os.path.join(tmp_path, "10", "output_transitVehicles.xml.gz") - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == "0.8" + assert config["controler"]["lastIteration"] == "20" + assert config["controler"]["outputDirectory"] == out_dir + assert config["network"]["inputNetworkFile"] == os.path.join( + tmp_path, "10", "output_network.xml.gz" + ) + assert config["plans"]["inputPlansFile"] == os.path.join( + tmp_path, "10", "output_plans.xml.gz" + ) + assert config["vehicles"]["vehiclesFile"] == os.path.join( + tmp_path, "10", "output_vehicles.xml.gz" + ) + assert config["transit"]["transitScheduleFile"] == os.path.join( + tmp_path, "10", "output_transitSchedule.xml.gz" + ) + assert config["transit"]["vehiclesFile"] == os.path.join( + tmp_path, "10", "output_transitVehicles.xml.gz" + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0.8" def test_autostep_with_missing_optional_allvehicles_ignores_vehicles_module(tmp_path): - in_file = os.path.join(os.path.dirname(__file__), "test_data", "test_config_missing_all_vehicles.xml") + in_file = os.path.join( + os.path.dirname(__file__), "test_data", "test_config_missing_all_vehicles.xml" + ) out_dir = os.path.join(tmp_path, "20") out_file = os.path.join(tmp_path, "10", "matsim_config.xml") - overrides_path = os.path.join(os.path.dirname(__file__), "test_data", "test_overrides.json") + overrides_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_overrides.json" + ) autostep.autostep_config( sim_root=tmp_path, seed_matsim_config_path=in_file, @@ -304,20 +386,24 @@ def test_autostep_with_missing_optional_allvehicles_ignores_vehicles_module(tmp_ total_iterations="100", step="10", biteration_matsim_config_path=out_file, - overrides_path=overrides_path + overrides_path=overrides_path, ) assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == '20' - assert config['controler']['outputDirectory'] == out_dir - assert 'vehicles' not in config + assert config["controler"]["lastIteration"] == "20" + assert config["controler"]["outputDirectory"] == out_dir + assert "vehicles" not in config def test_autostep_with_missing_allvehicles_updates_other_module_paths(tmp_path): - in_file = os.path.join(os.path.dirname(__file__), "test_data", "test_config_missing_all_vehicles.xml") + in_file = os.path.join( + os.path.dirname(__file__), "test_data", "test_config_missing_all_vehicles.xml" + ) out_dir = os.path.join(tmp_path, "20") out_file = os.path.join(tmp_path, "10", "matsim_config.xml") - overrides_path = os.path.join(os.path.dirname(__file__), "test_data", "test_overrides.json") + overrides_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_overrides.json" + ) autostep.autostep_config( sim_root=tmp_path, seed_matsim_config_path=in_file, @@ -325,16 +411,24 @@ def test_autostep_with_missing_allvehicles_updates_other_module_paths(tmp_path): total_iterations="100", step="10", biteration_matsim_config_path=out_file, - overrides_path=overrides_path + overrides_path=overrides_path, ) assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == '20' - assert config['controler']['outputDirectory'] == out_dir - assert config['network']['inputNetworkFile'] == os.path.join(tmp_path, "10", "output_network.xml.gz") - assert config['plans']['inputPlansFile'] == os.path.join(tmp_path, "10", "output_plans.xml.gz") - assert config['transit']['transitScheduleFile'] == os.path.join(tmp_path, "10", "output_transitSchedule.xml.gz") - assert config['transit']['vehiclesFile'] == os.path.join(tmp_path, "10", "output_transitVehicles.xml.gz") + assert config["controler"]["lastIteration"] == "20" + assert config["controler"]["outputDirectory"] == out_dir + assert config["network"]["inputNetworkFile"] == os.path.join( + tmp_path, "10", "output_network.xml.gz" + ) + assert config["plans"]["inputPlansFile"] == os.path.join( + tmp_path, "10", "output_plans.xml.gz" + ) + assert config["transit"]["transitScheduleFile"] == os.path.join( + tmp_path, "10", "output_transitSchedule.xml.gz" + ) + assert config["transit"]["vehiclesFile"] == os.path.join( + tmp_path, "10", "output_transitVehicles.xml.gz" + ) @pytest.fixture() @@ -346,29 +440,37 @@ def test_step_lambda(tmp_path, fake_lambda_handler): event = { "orchestration": { "sim_root": str(tmp_path), - "seed_matsim_config_path": os.path.join("tests", "test_data", "test_config.xml"), + "seed_matsim_config_path": os.path.join( + "tests", "test_data", "test_config.xml" + ), "start_index": "0", "total_iterations": "100", "step": "10", - } } + } orchestration = fake_lambda_handler(event) - assert(orchestration["start_index"] == "10") - assert(orchestration["biteration_matsim_config_path"] == os.path.join(tmp_path, "0", "matsim_config.xml")) + assert orchestration["start_index"] == "10" + assert orchestration["biteration_matsim_config_path"] == os.path.join( + tmp_path, "0", "matsim_config.xml" + ) def test_first_two_steps(tmp_path, fake_lambda_handler): - overrides_path = os.path.join(os.path.dirname(__file__), "test_data", "test_overrides.json") + overrides_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_overrides.json" + ) event = { "orchestration": { "sim_root": str(tmp_path), - "seed_matsim_config_path": os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml"), + "seed_matsim_config_path": os.path.join( + os.path.dirname(__file__), "test_data", "test_config.xml" + ), "start_index": "0", "total_iterations": "100", "step": "10", - } } + } orchestration = fake_lambda_handler(event) autostep.autostep_config( @@ -378,35 +480,49 @@ def test_first_two_steps(tmp_path, fake_lambda_handler): total_iterations=orchestration["total_iterations"], step=orchestration["step"], biteration_matsim_config_path=orchestration["biteration_matsim_config_path"], - overrides_path=overrides_path + overrides_path=overrides_path, ) out_dir = os.path.join(tmp_path, "10") out_file = os.path.join(tmp_path, "0", "matsim_config.xml") assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == '10' - assert config['controler']['outputDirectory'] == out_dir - assert config['network']['inputNetworkFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "network.xml") - )) - assert config['plans']['inputPlansFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "population.xml.gz") - )) - assert config['vehicles']['vehiclesFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "all_vehicles.xml") - )) - assert config['transit']['transitScheduleFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "schedule-merged.xml") - )) - assert config['transit']['vehiclesFile'] == os.path.abspath( - os.path.expanduser(os.path.join("~", "test", "vehicles.xml") - )) - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == "0.8" - + assert config["controler"]["lastIteration"] == "10" + assert config["controler"]["outputDirectory"] == out_dir + assert config["network"]["inputNetworkFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "network.xml")) + ) + assert config["plans"]["inputPlansFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "population.xml.gz")) + ) + assert config["vehicles"]["vehiclesFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "all_vehicles.xml")) + ) + assert config["transit"]["transitScheduleFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "schedule-merged.xml")) + ) + assert config["transit"]["vehiclesFile"] == os.path.abspath( + os.path.expanduser(os.path.join("~", "test", "vehicles.xml")) + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0.8" orchestration = fake_lambda_handler({"orchestration": orchestration}) autostep.autostep_config( @@ -416,36 +532,65 @@ def test_first_two_steps(tmp_path, fake_lambda_handler): total_iterations=orchestration["total_iterations"], step=orchestration["step"], biteration_matsim_config_path=orchestration["biteration_matsim_config_path"], - overrides_path=overrides_path + overrides_path=overrides_path, ) out_dir = os.path.join(tmp_path, "20") out_file = os.path.join(tmp_path, "10", "matsim_config.xml") assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == '20' - assert config['controler']['outputDirectory'] == out_dir - assert config['network']['inputNetworkFile'] == os.path.join(tmp_path, "10", "output_network.xml.gz") - assert config['plans']['inputPlansFile'] == os.path.join(tmp_path, "10", "output_plans.xml.gz") - assert config['vehicles']['vehiclesFile'] == os.path.join(tmp_path, "10", "output_vehicles.xml.gz") - assert config['transit']['transitScheduleFile'] == os.path.join(tmp_path, "10", "output_transitSchedule.xml.gz") - assert config['transit']['vehiclesFile'] == os.path.join(tmp_path, "10", "output_transitVehicles.xml.gz") - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == "0.8" + assert config["controler"]["lastIteration"] == "20" + assert config["controler"]["outputDirectory"] == out_dir + assert config["network"]["inputNetworkFile"] == os.path.join( + tmp_path, "10", "output_network.xml.gz" + ) + assert config["plans"]["inputPlansFile"] == os.path.join( + tmp_path, "10", "output_plans.xml.gz" + ) + assert config["vehicles"]["vehiclesFile"] == os.path.join( + tmp_path, "10", "output_vehicles.xml.gz" + ) + assert config["transit"]["transitScheduleFile"] == os.path.join( + tmp_path, "10", "output_transitSchedule.xml.gz" + ) + assert config["transit"]["vehiclesFile"] == os.path.join( + tmp_path, "10", "output_transitVehicles.xml.gz" + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0.8" def test_stepping(tmp_path, fake_lambda_handler): - overrides_path = os.path.join(os.path.dirname(__file__), "test_data", "test_overrides.json") + overrides_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_overrides.json" + ) orchestration = { "sim_root": str(tmp_path), - "seed_matsim_config_path": os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml"), + "seed_matsim_config_path": os.path.join( + os.path.dirname(__file__), "test_data", "test_config.xml" + ), "start_index": "0", "total_iterations": "30", "step": "10", - } + } for i in range(10): try: @@ -459,40 +604,71 @@ def test_stepping(tmp_path, fake_lambda_handler): start_index=orchestration["start_index"], total_iterations=orchestration["total_iterations"], step=orchestration["step"], - biteration_matsim_config_path=orchestration["biteration_matsim_config_path"], - overrides_path=overrides_path + biteration_matsim_config_path=orchestration[ + "biteration_matsim_config_path" + ], + overrides_path=overrides_path, ) assert set(os.listdir(tmp_path)) == {"0", "10", "20"} for i in range(10, 30, 10): - out_dir = os.path.join(tmp_path, str(i+10)) + out_dir = os.path.join(tmp_path, str(i + 10)) out_file = os.path.join(tmp_path, str(i), "matsim_config.xml") assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == str(i+10) - assert config['controler']['outputDirectory'] == out_dir - assert config['network']['inputNetworkFile'] == os.path.join(tmp_path, str(i), "output_network.xml.gz") - assert config['plans']['inputPlansFile'] == os.path.join(tmp_path, str(i), "output_plans.xml.gz") - assert config['vehicles']['vehiclesFile'] == os.path.join(tmp_path, str(i), "output_vehicles.xml.gz") - assert config['transit']['transitScheduleFile'] == os.path.join(tmp_path, str(i), "output_transitSchedule.xml.gz") - assert config['transit']['vehiclesFile'] == os.path.join(tmp_path, str(i), "output_transitVehicles.xml.gz") - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == "0.8" + assert config["controler"]["lastIteration"] == str(i + 10) + assert config["controler"]["outputDirectory"] == out_dir + assert config["network"]["inputNetworkFile"] == os.path.join( + tmp_path, str(i), "output_network.xml.gz" + ) + assert config["plans"]["inputPlansFile"] == os.path.join( + tmp_path, str(i), "output_plans.xml.gz" + ) + assert config["vehicles"]["vehiclesFile"] == os.path.join( + tmp_path, str(i), "output_vehicles.xml.gz" + ) + assert config["transit"]["transitScheduleFile"] == os.path.join( + tmp_path, str(i), "output_transitSchedule.xml.gz" + ) + assert config["transit"]["vehiclesFile"] == os.path.join( + tmp_path, str(i), "output_transitVehicles.xml.gz" + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0.8" def test_stepping_with_cooling(tmp_path, fake_lambda_handler): - overrides_path = os.path.join(os.path.dirname(__file__), "test_data", "test_overrides.json") + overrides_path = os.path.join( + os.path.dirname(__file__), "test_data", "test_overrides.json" + ) orchestration = { "sim_root": str(tmp_path), - "seed_matsim_config_path": os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml"), + "seed_matsim_config_path": os.path.join( + os.path.dirname(__file__), "test_data", "test_config.xml" + ), "start_index": "0", "total_iterations": "30", "step": "10", - "cooling_iterations": "10" - } + "cooling_iterations": "10", + } for i in range(10): try: @@ -506,42 +682,93 @@ def test_stepping_with_cooling(tmp_path, fake_lambda_handler): start_index=orchestration["start_index"], total_iterations=orchestration["total_iterations"], step=orchestration["step"], - biteration_matsim_config_path=orchestration["biteration_matsim_config_path"], - overrides_path=overrides_path + biteration_matsim_config_path=orchestration[ + "biteration_matsim_config_path" + ], + overrides_path=overrides_path, ) assert set(os.listdir(tmp_path)) == {"0", "10", "20", "30"} for i in range(10, 30, 10): - out_dir = os.path.join(tmp_path, str(i+10)) + out_dir = os.path.join(tmp_path, str(i + 10)) out_file = os.path.join(tmp_path, str(i), "matsim_config.xml") assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == str(i+10) - assert config['controler']['outputDirectory'] == out_dir - assert config['network']['inputNetworkFile'] == os.path.join(tmp_path, str(i), "output_network.xml.gz") - assert config['plans']['inputPlansFile'] == os.path.join(tmp_path, str(i), "output_plans.xml.gz") - assert config['vehicles']['vehiclesFile'] == os.path.join(tmp_path, str(i), "output_vehicles.xml.gz") - assert config['transit']['transitScheduleFile'] == os.path.join(tmp_path, str(i), "output_transitSchedule.xml.gz") - assert config['transit']['vehiclesFile'] == os.path.join(tmp_path, str(i), "output_transitVehicles.xml.gz") - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == "0.8" + assert config["controler"]["lastIteration"] == str(i + 10) + assert config["controler"]["outputDirectory"] == out_dir + assert config["network"]["inputNetworkFile"] == os.path.join( + tmp_path, str(i), "output_network.xml.gz" + ) + assert config["plans"]["inputPlansFile"] == os.path.join( + tmp_path, str(i), "output_plans.xml.gz" + ) + assert config["vehicles"]["vehiclesFile"] == os.path.join( + tmp_path, str(i), "output_vehicles.xml.gz" + ) + assert config["transit"]["transitScheduleFile"] == os.path.join( + tmp_path, str(i), "output_transitSchedule.xml.gz" + ) + assert config["transit"]["vehiclesFile"] == os.path.join( + tmp_path, str(i), "output_transitVehicles.xml.gz" + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0.8" i = 30 - out_dir = os.path.join(tmp_path, str(i+10)) + out_dir = os.path.join(tmp_path, str(i + 10)) out_file = os.path.join(tmp_path, str(i), "matsim_config.xml") assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == str(i+10) - assert config['controler']['outputDirectory'] == out_dir - assert config['network']['inputNetworkFile'] == os.path.join(tmp_path, str(i), "output_network.xml.gz") - assert config['plans']['inputPlansFile'] == os.path.join(tmp_path, str(i), "output_plans.xml.gz") - assert config['vehicles']['vehiclesFile'] == os.path.join(tmp_path, str(i), "output_vehicles.xml.gz") - assert config['transit']['transitScheduleFile'] == os.path.join(tmp_path, str(i), "output_transitSchedule.xml.gz") - assert config['transit']['vehiclesFile'] == os.path.join(tmp_path, str(i), "output_transitVehicles.xml.gz") - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" - assert config['strategy']['fractionOfIterationsToDisableInnovation'] == "0" - + assert config["controler"]["lastIteration"] == str(i + 10) + assert config["controler"]["outputDirectory"] == out_dir + assert config["network"]["inputNetworkFile"] == os.path.join( + tmp_path, str(i), "output_network.xml.gz" + ) + assert config["plans"]["inputPlansFile"] == os.path.join( + tmp_path, str(i), "output_plans.xml.gz" + ) + assert config["vehicles"]["vehiclesFile"] == os.path.join( + tmp_path, str(i), "output_vehicles.xml.gz" + ) + assert config["transit"]["transitScheduleFile"] == os.path.join( + tmp_path, str(i), "output_transitSchedule.xml.gz" + ) + assert config["transit"]["vehiclesFile"] == os.path.join( + tmp_path, str(i), "output_transitVehicles.xml.gz" + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) + assert config["strategy"]["fractionOfIterationsToDisableInnovation"] == "0" diff --git a/tests/test_base_config.py b/tests/test_base_config.py index 898c657..79d096d 100644 --- a/tests/test_base_config.py +++ b/tests/test_base_config.py @@ -28,14 +28,14 @@ def test_default_read_json(): def test_param_equality_same(): - test_config1 = BaseConfig(path=env.test_xml_path)["planCalcScore"]['scoringParameters:default']["lateArrival"] - test_config2 = BaseConfig(path=env.test_xml_path)["planCalcScore"]['scoringParameters:default']["lateArrival"] + test_config1 = BaseConfig(path=env.test_xml_path)["planCalcScore"]['scoringParameters::default']["lateArrival"] + test_config2 = BaseConfig(path=env.test_xml_path)["planCalcScore"]['scoringParameters::default']["lateArrival"] assert test_config1 == test_config2 def test_paramset_equality_same(): - test_config1 = BaseConfig(path=env.test_xml_path)["planCalcScore"]['scoringParameters:default'] - test_config2 = BaseConfig(path=env.test_xml_path)["planCalcScore"]['scoringParameters:default'] + test_config1 = BaseConfig(path=env.test_xml_path)["planCalcScore"]['scoringParameters::default'] + test_config2 = BaseConfig(path=env.test_xml_path)["planCalcScore"]['scoringParameters::default'] assert test_config1 == test_config2 @@ -115,8 +115,8 @@ def test_diff(): def test_config_dict_navigation(): test_config = BaseConfig(path=env.test_xml_path) test_module = test_config.modules['planCalcScore'] - scoringParameters = test_module.parametersets['scoringParameters:default'] - activityParams = scoringParameters.parametersets['activityParams:home'] + scoringParameters = test_module.parametersets['scoringParameters::default'] + activityParams = scoringParameters.parametersets['activityParams::home'] activityType = activityParams.params['activityType'] assert activityType.value == 'home' @@ -128,17 +128,17 @@ def test_module_dict_get(): def test_paramset_level_1_dict_get(): test_config = BaseConfig(path=env.test_xml_path) - assert isinstance(test_config['planCalcScore']['scoringParameters:default'], ParamSet) + assert isinstance(test_config['planCalcScore']['scoringParameters::default'], ParamSet) def test_paramset_level_2_dict_get(): test_config = BaseConfig(path=env.test_xml_path) - assert isinstance(test_config['planCalcScore']['scoringParameters:default']['activityParams:home'], ParamSet) + assert isinstance(test_config['planCalcScore']['scoringParameters::default']['activityParams::home'], ParamSet) def test_param_dict_get(): test_config = BaseConfig(path=env.test_xml_path) - assert isinstance(test_config['planCalcScore']['scoringParameters:default']['activityParams:home']['activityType'], str) + assert isinstance(test_config['planCalcScore']['scoringParameters::default']['activityParams::home']['activityType'], str) def test_module_dict_set(): @@ -149,18 +149,18 @@ def test_module_dict_set(): def test_paramset_level_1_dict_set(): - test_config_score = BaseConfig(path=env.test_bad_config_path)['planCalcScore']["scoringParameters:default"] + test_config_score = BaseConfig(path=env.test_bad_config_path)['planCalcScore']["scoringParameters::default"] test_config = BaseConfig(path=env.test_json_path) - test_config['planCalcScore']['scoringParameters:unknown'] = test_config_score - assert test_config['planCalcScore']['scoringParameters:unknown']['performing'] == '5.0' + test_config['planCalcScore']['scoringParameters::unknown'] = test_config_score + assert test_config['planCalcScore']['scoringParameters::unknown']['performing'] == '5.0' def test_param_dict_set(): test_config = BaseConfig(path=env.test_json_path) - assert isinstance(test_config['planCalcScore']["scoringParameters:default"].params['performing'], Param) - test_config['planCalcScore']["scoringParameters:default"]['performing'] = '7.0' - assert isinstance(test_config['planCalcScore']["scoringParameters:default"].params['performing'], Param) - assert test_config['planCalcScore']["scoringParameters:default"]['performing'] == '7.0' + assert isinstance(test_config['planCalcScore']["scoringParameters::default"].params['performing'], Param) + test_config['planCalcScore']["scoringParameters::default"]['performing'] = '7.0' + assert isinstance(test_config['planCalcScore']["scoringParameters::default"].params['performing'], Param) + assert test_config['planCalcScore']["scoringParameters::default"]['performing'] == '7.0' def test_module_key_valid(): @@ -178,5 +178,5 @@ def test_paramset_level_1_key_valid(): def test_param_level_1_key_valid(): test_config = BaseConfig(path=env.test_json_path) with pytest.raises(KeyError): - test_config['planCalcScore']['scoringParameters:default'].is_valid_param_key('NOTVALID') + test_config['planCalcScore']['scoringParameters::default'].is_valid_param_key('NOTVALID') diff --git a/tests/test_data/test_config.json b/tests/test_data/test_config.json index 5f6a4a1..4333f0f 100644 --- a/tests/test_data/test_config.json +++ b/tests/test_data/test_config.json @@ -50,19 +50,19 @@ "useModeMappingForPassengers": "true" }, "parametersets": { - "intermodalAccessEgress:walk": { + "intermodalAccessEgress::walk": { "params": { "mode": "walk", "maxRadius": "1000" } }, - "modeMapping:bus": { + "modeMapping::bus": { "params": { "routeMode": "bus", "passengerMode": "bus" } }, - "modeMapping:train": { + "modeMapping::train": { "params": { "routeMode": "train", "passengerMode": "train" @@ -168,7 +168,7 @@ "writeExperiencedPlans": "false" }, "parametersets": { - "scoringParameters:default": { + "scoringParameters::default": { "params": { "earlyDeparture": "-0.0", "lateArrival": "-18.0", @@ -180,7 +180,7 @@ "waitingPt": "-1.0" }, "parametersets": { - "activityParams:education": { + "activityParams::education": { "params": { "activityType": "education", "closingTime": "17:00:00", @@ -194,7 +194,7 @@ "typicalDurationScoreComputation": "relative" } }, - "activityParams:home": { + "activityParams::home": { "params": { "activityType": "home", "closingTime": "undefined", @@ -208,7 +208,7 @@ "typicalDurationScoreComputation": "relative" } }, - "activityParams:work": { + "activityParams::work": { "params": { "activityType": "work", "closingTime": "19:00:00", @@ -222,7 +222,7 @@ "typicalDurationScoreComputation": "relative" } }, - "modeParams:car": { + "modeParams::car": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -231,7 +231,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:pt": { + "modeParams::pt": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -240,7 +240,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:walk": { + "modeParams::walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -249,7 +249,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:access_walk": { + "modeParams::access_walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -258,7 +258,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:bike": { + "modeParams::bike": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -267,7 +267,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:train": { + "modeParams::train": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -276,7 +276,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:bus": { + "modeParams::bus": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -287,7 +287,7 @@ } } }, - "scoringParameters:unknown": { + "scoringParameters::unknown": { "params": { "earlyDeparture": "-0.0", "lateArrival": "-18.0", @@ -299,7 +299,7 @@ "waitingPt": "-1.0" }, "parametersets": { - "activityParams:education": { + "activityParams::education": { "params": { "activityType": "education", "closingTime": "17:00:00", @@ -313,7 +313,7 @@ "typicalDurationScoreComputation": "relative" } }, - "activityParams:home": { + "activityParams::home": { "params": { "activityType": "home", "closingTime": "undefined", @@ -327,7 +327,7 @@ "typicalDurationScoreComputation": "relative" } }, - "activityParams:work": { + "activityParams::work": { "params": { "activityType": "work", "closingTime": "19:00:00", @@ -341,7 +341,7 @@ "typicalDurationScoreComputation": "relative" } }, - "modeParams:car": { + "modeParams::car": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -350,7 +350,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:pt": { + "modeParams::pt": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -359,7 +359,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:walk": { + "modeParams::walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -368,7 +368,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:access_walk": { + "modeParams::access_walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -377,7 +377,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:bike": { + "modeParams::bike": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -386,7 +386,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:train": { + "modeParams::train": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -395,7 +395,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:bus": { + "modeParams::bus": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -418,7 +418,7 @@ "planSelectorForRemoval": "WorstPlanSelector" }, "parametersets": { - "strategysettings:unknown:ChangeSingleTripMode": { + "strategysettings::unknown::ChangeSingleTripMode": { "params": { "disableAfterIteration": "-1", "executionPath": "null", @@ -427,7 +427,7 @@ "weight": "0.1" } }, - "strategysettings:unknown:ReRoute": { + "strategysettings::unknown::ReRoute": { "params": { "disableAfterIteration": "-1", "executionPath": "null", @@ -436,7 +436,7 @@ "weight": "0.2" } }, - "strategysettings:unknown:TimeAllocationMutator_ReRoute": { + "strategysettings::unknown::TimeAllocationMutator_ReRoute": { "params": { "disableAfterIteration": "-1", "executionPath": "null", @@ -445,7 +445,7 @@ "weight": "0.1" } }, - "strategysettings:unknown:SelectExpBeta": { + "strategysettings::unknown::SelectExpBeta": { "params": { "disableAfterIteration": "-1", "executionPath": "null", diff --git a/tests/test_data/test_diff.json b/tests/test_data/test_diff.json index 963fa92..d82cd41 100644 --- a/tests/test_data/test_diff.json +++ b/tests/test_data/test_diff.json @@ -103,7 +103,7 @@ "writeExperiencedPlans": "false" }, "parametersets": { - "scoringParameters:default": { + "scoringParameters::default": { "params": { "earlyDeparture": "-0.0", "lateArrival": "-18.0", @@ -115,7 +115,7 @@ "waitingPt": "-1.0" }, "parametersets": { - "activityParams:home": { + "activityParams::home": { "params": { "activityType": "home", "closingTime": "undefined", @@ -129,7 +129,7 @@ "typicalDurationScoreComputation": "relative" } }, - "modeParams:car": { + "modeParams::car": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -138,7 +138,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:pt": { + "modeParams::pt": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -147,7 +147,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:walk": { + "modeParams::walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -156,7 +156,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:bike": { + "modeParams::bike": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -167,7 +167,7 @@ } } }, - "scoringParameters:unknown": { + "scoringParameters::unknown": { "params": { "earlyDeparture": "-0.0", "lateArrival": "-18.0", @@ -179,7 +179,7 @@ "waitingPt": "-1.0" }, "parametersets": { - "activityParams:home": { + "activityParams::home": { "params": { "activityType": "home", "closingTime": "undefined", @@ -193,7 +193,7 @@ "typicalDurationScoreComputation": "relative" } }, - "modeParams:car": { + "modeParams::car": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -202,7 +202,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:pt": { + "modeParams::pt": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -211,7 +211,7 @@ "monetaryDistanceRate": "-0.1" } }, - "modeParams:walk": { + "modeParams::walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -220,7 +220,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:bike": { + "modeParams::bike": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -249,7 +249,7 @@ "networkModes": "car" }, "parametersets": { - "teleportedModeParameters:bike": { + "teleportedModeParameters::bike": { "params": { "beelineDistanceFactor": "1.3", "mode": "bike", @@ -257,7 +257,7 @@ "teleportedModeSpeed": "4.166666666666667" } }, - "teleportedModeParameters:walk": { + "teleportedModeParameters::walk": { "params": { "beelineDistanceFactor": "1.3", "mode": "walk", @@ -265,7 +265,7 @@ "teleportedModeSpeed": "0.8333333333333333" } }, - "teleportedModeParameters:egress_walk": { + "teleportedModeParameters::egress_walk": { "params": { "beelineDistanceFactor": "1.3", "mode": "egress_walk", @@ -273,7 +273,7 @@ "teleportedModeSpeed": "0.8333333333333333" } }, - "teleportedModeParameters:undefined": { + "teleportedModeParameters::undefined": { "params": { "beelineDistanceFactor": "1.3", "mode": "undefined", @@ -281,7 +281,7 @@ "teleportedModeSpeed": "13.88888888888889" } }, - "teleportedModeParameters:ride": { + "teleportedModeParameters::ride": { "params": { "beelineDistanceFactor": "1.3", "mode": "ride", @@ -289,7 +289,7 @@ "teleportedModeSpeed": "null" } }, - "teleportedModeParameters:pt": { + "teleportedModeParameters::pt": { "params": { "beelineDistanceFactor": "1.3", "mode": "pt", diff --git a/tests/test_data/test_overrides.json b/tests/test_data/test_overrides.json index 1201338..ea63d68 100644 --- a/tests/test_data/test_overrides.json +++ b/tests/test_data/test_overrides.json @@ -1,20 +1,20 @@ [ { - "scoringParameters:High_Car/marginalUtilityOfMoney": "2.5" + "scoringParameters::High_Car/marginalUtilityOfMoney": "2.5" }, { - "scoringParameters:High_NoCar/marginalUtilityOfMoney": "2.5" + "scoringParameters::High_NoCar/marginalUtilityOfMoney": "2.5" }, { - "scoringParameters:Low_Car/marginalUtilityOfMoney": "40" + "scoringParameters::Low_Car/marginalUtilityOfMoney": "40" }, { - "scoringParameters:Low_NoCar/marginalUtilityOfMoney": "40" + "scoringParameters::Low_NoCar/marginalUtilityOfMoney": "40" }, { - "modeParams:car/constant": "-1.0" + "modeParams::car/constant": "-1.0" }, { - "scoringParameters:unknown/modeParams:bus/constant": "-1.0" + "scoringParameters::unknown/modeParams::bus/constant": "-1.0" } ] diff --git a/tests/test_data/test_temp_config.json b/tests/test_data/test_temp_config.json index 5f6a4a1..4333f0f 100644 --- a/tests/test_data/test_temp_config.json +++ b/tests/test_data/test_temp_config.json @@ -50,19 +50,19 @@ "useModeMappingForPassengers": "true" }, "parametersets": { - "intermodalAccessEgress:walk": { + "intermodalAccessEgress::walk": { "params": { "mode": "walk", "maxRadius": "1000" } }, - "modeMapping:bus": { + "modeMapping::bus": { "params": { "routeMode": "bus", "passengerMode": "bus" } }, - "modeMapping:train": { + "modeMapping::train": { "params": { "routeMode": "train", "passengerMode": "train" @@ -168,7 +168,7 @@ "writeExperiencedPlans": "false" }, "parametersets": { - "scoringParameters:default": { + "scoringParameters::default": { "params": { "earlyDeparture": "-0.0", "lateArrival": "-18.0", @@ -180,7 +180,7 @@ "waitingPt": "-1.0" }, "parametersets": { - "activityParams:education": { + "activityParams::education": { "params": { "activityType": "education", "closingTime": "17:00:00", @@ -194,7 +194,7 @@ "typicalDurationScoreComputation": "relative" } }, - "activityParams:home": { + "activityParams::home": { "params": { "activityType": "home", "closingTime": "undefined", @@ -208,7 +208,7 @@ "typicalDurationScoreComputation": "relative" } }, - "activityParams:work": { + "activityParams::work": { "params": { "activityType": "work", "closingTime": "19:00:00", @@ -222,7 +222,7 @@ "typicalDurationScoreComputation": "relative" } }, - "modeParams:car": { + "modeParams::car": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -231,7 +231,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:pt": { + "modeParams::pt": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -240,7 +240,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:walk": { + "modeParams::walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -249,7 +249,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:access_walk": { + "modeParams::access_walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -258,7 +258,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:bike": { + "modeParams::bike": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -267,7 +267,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:train": { + "modeParams::train": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -276,7 +276,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:bus": { + "modeParams::bus": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -287,7 +287,7 @@ } } }, - "scoringParameters:unknown": { + "scoringParameters::unknown": { "params": { "earlyDeparture": "-0.0", "lateArrival": "-18.0", @@ -299,7 +299,7 @@ "waitingPt": "-1.0" }, "parametersets": { - "activityParams:education": { + "activityParams::education": { "params": { "activityType": "education", "closingTime": "17:00:00", @@ -313,7 +313,7 @@ "typicalDurationScoreComputation": "relative" } }, - "activityParams:home": { + "activityParams::home": { "params": { "activityType": "home", "closingTime": "undefined", @@ -327,7 +327,7 @@ "typicalDurationScoreComputation": "relative" } }, - "activityParams:work": { + "activityParams::work": { "params": { "activityType": "work", "closingTime": "19:00:00", @@ -341,7 +341,7 @@ "typicalDurationScoreComputation": "relative" } }, - "modeParams:car": { + "modeParams::car": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -350,7 +350,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:pt": { + "modeParams::pt": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "0.0", @@ -359,7 +359,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:walk": { + "modeParams::walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -368,7 +368,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:access_walk": { + "modeParams::access_walk": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -377,7 +377,7 @@ "monetaryDistanceRate": "0.0" } }, - "modeParams:bike": { + "modeParams::bike": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -386,7 +386,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:train": { + "modeParams::train": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -395,7 +395,7 @@ "monetaryDistanceRate": "-0.0" } }, - "modeParams:bus": { + "modeParams::bus": { "params": { "constant": "0.0", "marginalUtilityOfDistance_util_m": "-0.0", @@ -418,7 +418,7 @@ "planSelectorForRemoval": "WorstPlanSelector" }, "parametersets": { - "strategysettings:unknown:ChangeSingleTripMode": { + "strategysettings::unknown::ChangeSingleTripMode": { "params": { "disableAfterIteration": "-1", "executionPath": "null", @@ -427,7 +427,7 @@ "weight": "0.1" } }, - "strategysettings:unknown:ReRoute": { + "strategysettings::unknown::ReRoute": { "params": { "disableAfterIteration": "-1", "executionPath": "null", @@ -436,7 +436,7 @@ "weight": "0.2" } }, - "strategysettings:unknown:TimeAllocationMutator_ReRoute": { + "strategysettings::unknown::TimeAllocationMutator_ReRoute": { "params": { "disableAfterIteration": "-1", "executionPath": "null", @@ -445,7 +445,7 @@ "weight": "0.1" } }, - "strategysettings:unknown:SelectExpBeta": { + "strategysettings::unknown::SelectExpBeta": { "params": { "disableAfterIteration": "-1", "executionPath": "null", diff --git a/tests/test_default_config.py b/tests/test_default_config.py index 3e6a8cc..cdde2e1 100644 --- a/tests/test_default_config.py +++ b/tests/test_default_config.py @@ -20,7 +20,7 @@ def test_default_config_path(): def test_load_config(): config = BaseConfig(path=_DEFAULTS_DIR / 'default_config.xml') - assert config["planCalcScore"]['scoringParameters:default']["lateArrival"] + assert config["planCalcScore"]['scoringParameters::default']["lateArrival"] def test_init_default_config(): diff --git a/tests/test_find.py b/tests/test_find.py index 3a08864..9f35ef8 100644 --- a/tests/test_find.py +++ b/tests/test_find.py @@ -52,66 +52,66 @@ def test_find_all_param_at_module_level(config): def test_find_paramset_at_module_level(config): - paramsets = config.find("scoringParameters:default") + paramsets = config.find("scoringParameters::default") assert len(paramsets) == 1 assert paramsets[0].class_type == 'paramset' def test_find_paramsets_at_module_level(config): - paramsets = config.find("scoringParameters:*") + paramsets = config.find("scoringParameters::*") assert len(paramsets) == 2 assert paramsets[0].class_type == 'paramset' def test_find_module_paramset_at_module_level(config): - paramsets = config.find("planCalcScore/scoringParameters:default") + paramsets = config.find("planCalcScore/scoringParameters::default") assert len(paramsets) == 1 assert paramsets[0].class_type == 'paramset' def test_find_module_paramsets_at_module_level(config): - paramsets = config.find("planCalcScore/scoringParameters:*") + paramsets = config.find("planCalcScore/scoringParameters::*") assert len(paramsets) == 2 assert paramsets[0].class_type == 'paramset' def test_find_all_paramset_at_module_level(config): - paramsets = config.find("*/scoringParameters:default") + paramsets = config.find("*/scoringParameters::default") assert len(paramsets) == 1 assert paramsets[0].class_type == 'paramset' def test_find_all_paramsets_at_module_level(config): - paramsets = config.find("*/scoringParameters:*") + paramsets = config.find("*/scoringParameters::*") assert len(paramsets) == 2 assert paramsets[0].class_type == 'paramset' def test_find_paramsets_at_paramsets_level(config): - paramsets = config.find("scoringParameters:*/activityParams:*") + paramsets = config.find("scoringParameters::*/activityParams::*") assert len(paramsets) == 6 assert paramsets[0].class_type == 'paramset' def test_find_paramset_at_paramsets_level(config): - paramsets = config.find("scoringParameters:default/activityParams:*") + paramsets = config.find("scoringParameters::default/activityParams::*") assert len(paramsets) == 3 assert paramsets[0].class_type == 'paramset' def test_find_param_at_paramsets_level(config): - paramsets = config.find("scoringParameters:default/activityParams:work/priority") + paramsets = config.find("scoringParameters::default/activityParams::work/priority") assert len(paramsets) == 1 assert paramsets[0].value == '1.0' def test_find_params_at_paramsets_level(config): - paramsets = config.find("activityParams:work/priority") + paramsets = config.find("activityParams::work/priority") assert len(paramsets) == 2 assert paramsets[0].value == '1.0' def test_find_params_at_nested_paramsets_level(config): - paramsets = config.find("scoringParameters:default/priority") + paramsets = config.find("scoringParameters::default/priority") assert len(paramsets) == 3 assert paramsets[0].value == '1.0' diff --git a/tests/test_step.py b/tests/test_step.py index 6b9f696..adfd7fb 100644 --- a/tests/test_step.py +++ b/tests/test_step.py @@ -9,91 +9,145 @@ @pytest.fixture() def config(): - in_file = Path(os.path.join(os.path.dirname(__file__), 'test_data', 'test_config.xml')) + in_file = Path( + os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml") + ) return BaseConfig(in_file) def test_set_write_path(config): - step.set_write_path(config, {'outputDirectory': 'testing'}) - assert config['controler']['outputDirectory'] == 'testing' + step.set_write_path(config, {"outputDirectory": "testing"}) + assert config["controler"]["outputDirectory"] == "testing" def test_set_input_paths(config): - step.set_input_paths(config, {'matsim_source': 'test/ing'}) - assert config['network']['inputNetworkFile'] == 'test/ing/network.xml' - assert config['plans']['inputPlansFile'] == 'test/ing/population.xml.gz' - assert config['plans']['inputPersonAttributesFile'] == 'test/ing/population_attributes.xml.gz' - assert config['transit']['transitScheduleFile'] == 'test/ing/schedule-merged.xml' - assert config['transit']['vehiclesFile'] == 'test/ing/vehicles.xml' - assert config['transit']['transitLinesAttributesFile'] == 'null' + step.set_input_paths(config, {"matsim_source": "test/ing"}) + assert config["network"]["inputNetworkFile"] == "test/ing/network.xml" + assert config["plans"]["inputPlansFile"] == "test/ing/population.xml.gz" + assert ( + config["plans"]["inputPersonAttributesFile"] + == "test/ing/population_attributes.xml.gz" + ) + assert config["transit"]["transitScheduleFile"] == "test/ing/schedule-merged.xml" + assert config["transit"]["vehiclesFile"] == "test/ing/vehicles.xml" + assert config["transit"]["transitLinesAttributesFile"] == "null" def test_set_step(config): - step.set_last_iteration(config, {'step': '999'}) - assert config['controler']['lastIteration'] == '999' + step.set_last_iteration(config, {"step": "999"}) + assert config["controler"]["lastIteration"] == "999" def test_find_and_set_param(config): - step.find_and_set_overrides( - config, - {"modeParams:car/constant": "-1.0"} - ) - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "0.0" + step.find_and_set_overrides(config, {"modeParams::car/constant": "-1.0"}) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "0.0" + ) def test_find_and_set_params(config): step.find_and_set_overrides( config, { - "modeParams:car/constant": "-1.0", - "scoringParameters:unknown/modeParams:bus/constant": "-1.0" - } - ) - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" + "modeParams::car/constant": "-1.0", + "scoringParameters::unknown/modeParams::bus/constant": "-1.0", + }, + ) + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) + def test_find_and_set_bad_param(config): cnfg = deepcopy(config) - step.find_and_set_overrides( - config, - {"modeParams:*/horseback": "-1.0"} - ) + step.find_and_set_overrides(config, {"modeParams::*/horseback": "-1.0"}) assert cnfg == config def test_construct_overrides_map_from_tuple(): - assert step.construct_override_map_from_tuple( - ('a','b','c','d') - ) == {'a':'b', 'c':'d'} + assert step.construct_override_map_from_tuple(("a", "b", "c", "d")) == { + "a": "b", + "c": "d", + } def test_step_config(tmp_path): - in_file = os.path.join(os.path.dirname(__file__), 'test_data', 'test_config.xml') + in_file = os.path.join(os.path.dirname(__file__), "test_data", "test_config.xml") out_file = os.path.join(tmp_path, "test_config.xml") step.step_config( input_file=in_file, output_file=out_file, overrides=( - 'matsim_source', 'test/ing', - 'outputDirectory', 'testing', - 'step', '999', - "modeParams:car/constant", "-1.0", - "scoringParameters:unknown/modeParams:bus/constant", "-1.0" - ) + "matsim_source", + "test/ing", + "outputDirectory", + "testing", + "step", + "999", + "modeParams::car/constant", + "-1.0", + "scoringParameters::unknown/modeParams::bus/constant", + "-1.0", + ), ) assert os.path.exists(out_file) config = BaseConfig(out_file) - assert config['controler']['lastIteration'] == '999' - assert config['controler']['outputDirectory'] == 'testing' - assert config['network']['inputNetworkFile'] == 'test/ing/network.xml' - assert config['plans']['inputPlansFile'] == 'test/ing/population.xml.gz' - assert config['plans']['inputPersonAttributesFile'] == 'test/ing/population_attributes.xml.gz' - assert config['transit']['transitScheduleFile'] == 'test/ing/schedule-merged.xml' - assert config['transit']['vehiclesFile'] == 'test/ing/vehicles.xml' - assert config['transit']['transitLinesAttributesFile'] == 'null' - assert config['planCalcScore']['scoringParameters:default']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:car']["constant"] == "-1.0" - assert config['planCalcScore']['scoringParameters:unknown']['modeParams:bus']["constant"] == "-1.0" + assert config["controler"]["lastIteration"] == "999" + assert config["controler"]["outputDirectory"] == "testing" + assert config["network"]["inputNetworkFile"] == "test/ing/network.xml" + assert config["plans"]["inputPlansFile"] == "test/ing/population.xml.gz" + assert ( + config["plans"]["inputPersonAttributesFile"] + == "test/ing/population_attributes.xml.gz" + ) + assert config["transit"]["transitScheduleFile"] == "test/ing/schedule-merged.xml" + assert config["transit"]["vehiclesFile"] == "test/ing/vehicles.xml" + assert config["transit"]["transitLinesAttributesFile"] == "null" + assert ( + config["planCalcScore"]["scoringParameters::default"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::car"][ + "constant" + ] + == "-1.0" + ) + assert ( + config["planCalcScore"]["scoringParameters::unknown"]["modeParams::bus"][ + "constant" + ] + == "-1.0" + ) diff --git a/tests/test_valid_keys.py b/tests/test_valid_keys.py index 702e527..47dacbb 100644 --- a/tests/test_valid_keys.py +++ b/tests/test_valid_keys.py @@ -3,17 +3,15 @@ """ import pytest -import os import env env.set_module() -from mc.base import BaseConfig, ParamSet, Param, get_params_search, get_paramsets_search, get_paramset_type +from mc.base import BaseConfig, get_params_search, get_paramsets_search from mc.debug import * from mc.valid import VALID_MAP - def test_module_valid_key_construction(): test_config = BaseConfig(path=env.test_json_path) valid_module_keys = test_config.valid_keys @@ -22,136 +20,152 @@ def test_module_valid_key_construction(): def test_module_param_valid_key_construction(): test_config = BaseConfig(path=env.test_json_path) - valid_param_keys = test_config['planCalcScore'].valid_param_keys + valid_param_keys = test_config["planCalcScore"].valid_param_keys assert valid_param_keys def test_paramset_names_and_types(): test_config = BaseConfig(path=env.test_json_path) - ident = test_config['planCalcScore']['scoringParameters:default'].ident - ty = test_config['planCalcScore']['scoringParameters:default'].type + ident = test_config["planCalcScore"]["scoringParameters::default"].ident + ty = test_config["planCalcScore"]["scoringParameters::default"].type assert not ident == ty def test_get_paramsets_type_search(): - assert get_paramsets_search(VALID_MAP, 'scoringParameters:default') + assert get_paramsets_search(VALID_MAP, "scoringParameters::default") def test_get_params_type_search(): - assert get_params_search(VALID_MAP, 'scoringParameters:default') + assert get_params_search(VALID_MAP, "scoringParameters::default") def test_module_paramset_level_1_valid_key_construction(): test_config = BaseConfig(path=env.test_json_path) - valid_paramset_keys = test_config['planCalcScore'].valid_paramset_keys + valid_paramset_keys = test_config["planCalcScore"].valid_paramset_keys assert valid_paramset_keys def test_module_param_level_1_valid_key_construction(): test_config = BaseConfig(path=env.test_json_path) - valid_param_keys = test_config['planCalcScore']["scoringParameters:default"].valid_param_keys + valid_param_keys = test_config["planCalcScore"][ + "scoringParameters::default" + ].valid_param_keys assert valid_param_keys def test_module_paramset_level_2_valid_key_construction_(): test_config = BaseConfig(path=env.test_json_path) - valid_paramset_keys = test_config['planCalcScore']["scoringParameters:default"].valid_paramset_keys + valid_paramset_keys = test_config["planCalcScore"][ + "scoringParameters::default" + ].valid_paramset_keys assert valid_paramset_keys def test_module_paramset_level_2_keys(): test_config = BaseConfig(path=env.test_json_path) - paramset_keys = test_config['planCalcScore']["scoringParameters:default"].parametersets + paramset_keys = test_config["planCalcScore"][ + "scoringParameters::default" + ].parametersets assert paramset_keys def test_module_paramset_level_3_invalid_key_construction(): test_config = BaseConfig(path=env.test_json_path) - valid_paramset_keys = test_config['planCalcScore']["scoringParameters:default"]["modeParams:car"].valid_paramset_keys + valid_paramset_keys = test_config["planCalcScore"]["scoringParameters::default"][ + "modeParams::car" + ].valid_paramset_keys assert len(valid_paramset_keys) == 0 def test_module_param_level_2_valid_key_construction(): test_config = BaseConfig(path=env.test_json_path) - valid_param_keys = test_config['planCalcScore']["scoringParameters:default"]["modeParams:car"].valid_param_keys + valid_param_keys = test_config["planCalcScore"]["scoringParameters::default"][ + "modeParams::car" + ].valid_param_keys assert valid_param_keys def test_invalid_module_invalid_name(): test_config = BaseConfig(path=env.test_json_path) with pytest.raises(KeyError): - test_config['NOTVALID'] = test_config["planCalcScore"] + test_config["NOTVALID"] = test_config["planCalcScore"] def test_parameterset_level_1_invalid_name(): test_config = BaseConfig(path=env.test_json_path) - paramset = test_config["planCalcScore"]["scoringParameters:default"] + paramset = test_config["planCalcScore"]["scoringParameters::default"] with pytest.raises(KeyError): - test_config["planCalcScore"]['NOTVALID'] = paramset + test_config["planCalcScore"]["NOTVALID"] = paramset def test_param_level_1_invalid_name(): test_config = BaseConfig(path=env.test_json_path) param = test_config["planCalcScore"]["BrainExpBeta"] with pytest.raises(KeyError): - test_config["planCalcScore"]['NOTVALID'] = param + test_config["planCalcScore"]["NOTVALID"] = param def test_parameterset_level_2_invalid_name(): test_config = BaseConfig(path=env.test_json_path) with pytest.raises(KeyError): - test_config["planCalcScore"]['scoringParameters:default']['NOTVALID'] = test_config["planCalcScore"]['scoringParameters:default']["activityParams:education"] + test_config["planCalcScore"]["scoringParameters::default"][ + "NOTVALID" + ] = test_config["planCalcScore"]["scoringParameters::default"][ + "activityParams::education" + ] def test_param_level_2_invalid_name(): test_config = BaseConfig(path=env.test_json_path) with pytest.raises(KeyError): - test_config["planCalcScore"]['scoringParameters:default']['NOTVALID'] = test_config["planCalcScore"]['scoringParameters:default']["lateArrival"] + test_config["planCalcScore"]["scoringParameters::default"][ + "NOTVALID" + ] = test_config["planCalcScore"]["scoringParameters::default"]["lateArrival"] def test_param_level_2_invalid_value(): test_config = BaseConfig(path=env.test_json_path) with pytest.raises(ValueError): - test_config["planCalcScore"]['scoringParameters:default']['closingTime'] = 4 + test_config["planCalcScore"]["scoringParameters::default"]["closingTime"] = 4 def test_param_level_2_invalid_key(): test_config = BaseConfig(path=env.test_json_path) with pytest.raises(KeyError): - test_config["planCalcScore"]['scoringParameters:default']['cloPPingTime'] = '4' + test_config["planCalcScore"]["scoringParameters::default"]["cloPPingTime"] = "4" def test_get_default_to_default(): test_config = BaseConfig(path=env.test_json_path) - v = test_config["planCalcScore"]['scoringParameters']["lateArrival"] - assert v == '-18.0' + v = test_config["planCalcScore"]["scoringParameters"]["lateArrival"] + assert v == "-18.0" def test_get_default_to_list(): test_config = BaseConfig(path=env.test_json_path) - v = test_config["planCalcScore"]['scoringParameters']["activityParams"] + v = test_config["planCalcScore"]["scoringParameters"]["activityParams"] assert isinstance(v, list) def test_nested_set(): c = BaseConfig() - c['controler']['mobsim'] = 'a' - assert c['controler']['mobsim'] == 'a' + c["controler"]["mobsim"] = "a" + assert c["controler"]["mobsim"] == "a" def test_nested_set_paramsets(): c = BaseConfig() - c['planCalcScore']['scoringParameters:group1']['earlyDeparture'] = '1' - assert c['planCalcScore']['scoringParameters:group1']['earlyDeparture'] == '1' + c["planCalcScore"]["scoringParameters::group1"]["earlyDeparture"] = "1" + assert c["planCalcScore"]["scoringParameters::group1"]["earlyDeparture"] == "1" def test_nested_set_fail(): c = BaseConfig() with pytest.raises(KeyError): - c['INVALID']['mobsim'] = 'a' + c["INVALID"]["mobsim"] = "a" def test_nested_set_paramsets_fail(): c = BaseConfig() with pytest.raises(KeyError): - c['planCalcScore']['INVALID:group1']['earlyDeparture'] = '1' + c["planCalcScore"]["INVALID::group1"]["earlyDeparture"] = "1" From 2ed010e03f1b1ece4249d61660ba30ed0e4fe7ef Mon Sep 17 00:00:00 2001 From: syhwawa Date: Wed, 8 Nov 2023 20:15:53 +0000 Subject: [PATCH 2/7] Fix the build pipleline --- .github/workflows/build_pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_pipeline.yml b/.github/workflows/build_pipeline.yml index 425be91..46b57e2 100644 --- a/.github/workflows/build_pipeline.yml +++ b/.github/workflows/build_pipeline.yml @@ -58,7 +58,7 @@ jobs: aws s3 cp app.zip s3://arup-arup-ukimea-tcs-dev-test-code/$repo_slug.zip - name: Send build success notification if: success() - uses: rtCamp/action-slack-notify@v2.0.0 + uses: rtCamp/action-slack-notify@v2.2.0 env: SLACK_MESSAGE: ${{ github.repository }} build ${{ github.run_number }} launched by ${{ github.actor }} has succeeded SLACK_TITLE: Build Success @@ -68,7 +68,7 @@ jobs: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - name: Send build failure notification if: failure() - uses: rtCamp/action-slack-notify@v2.0.0 + uses: rtCamp/action-slack-notify@v2.2.0 env: SLACK_COLOR: '#FF0000' SLACK_MESSAGE: ${{ github.repository }} build ${{ github.run_number }} launched by ${{ github.actor }} has failed From 5fee54c8e3b43b2762a8402ea36e783f6dc867a1 Mon Sep 17 00:00:00 2001 From: syhwawa Date: Wed, 8 Nov 2023 21:22:07 +0000 Subject: [PATCH 3/7] Update the DMC module parameter set --- mc/base.py | 10 ++++++++-- mc/valid.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/mc/base.py b/mc/base.py index 31fbaae..179e539 100644 --- a/mc/base.py +++ b/mc/base.py @@ -715,7 +715,10 @@ def __init__(self, name, xml_object=None, json_object=None, uid=None) -> None: self.build_from_json(json_object) def __str__(self) -> str: - return f"{self.class_type.upper()}: {self.type} ({self.name})" + if ":" in self.type: + return f"{self.class_type.upper()}: {self.type} ({self.type})" + else: + return f"{self.class_type.upper()}: {self.type} ({self.name})" def print(self, i: int = 0) -> None: """ @@ -991,7 +994,10 @@ def build_paramset_key(elem: et.Element, seperator: str = "::") -> Tuple[str, st return paramset_type, key, uid if ":" in paramset_type: - """special cases fpr selector:MultinomialLogit and modeAvailability:Car from DMC mod""" + """ + special cases when matsim config has semicolons in paramerset such as selector:MultinomialLogit + and modeAvailability:Car from DMC module + """ uid = paramset_type.split(":")[-1] key = paramset_type + seperator + uid return paramset_type, key, uid diff --git a/mc/valid.py b/mc/valid.py index 1915f97..6dc1791 100644 --- a/mc/valid.py +++ b/mc/valid.py @@ -92,6 +92,26 @@ "ensureActivityReachability": "true", } }, + "DiscreteModeChoice": { + "params": { + "modelType": "Tour", + "selector": "MultinomialLogit" + }, + "parametersets": { + "selector:MultinomialLogit": { + "params": { + "considerMinimumUtility": "false", + "maximumUtility": "700.0", + "minimumUtility": "-700.0" + } + }, + "modeAvailability:Car": { + "params": { + "availableModes": "car,pt,walk,bike" + } + } + } + }, "travelTimeCalculator": { "params": { "analyzedModes": "car,bus,walk", From 323ace3aa35f98189b91fbd36a30c3b389eee9dd Mon Sep 17 00:00:00 2001 From: syhwawa Date: Thu, 9 Nov 2023 19:25:27 +0000 Subject: [PATCH 4/7] Update the DMC module base code --- mc/base.py | 3 +-- mc/valid.py | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mc/base.py b/mc/base.py index 179e539..16c89fb 100644 --- a/mc/base.py +++ b/mc/base.py @@ -145,7 +145,6 @@ def build_from_xml(self, xml_object: Element): elif sub_object.tag == "parameterset": _, key, uid = build_paramset_key(sub_object) - self.parametersets[key] = ParamSet(key, xml_object=sub_object, uid=uid) else: @@ -999,7 +998,7 @@ def build_paramset_key(elem: et.Element, seperator: str = "::") -> Tuple[str, st and modeAvailability:Car from DMC module """ uid = paramset_type.split(":")[-1] - key = paramset_type + seperator + uid + key = paramset_type return paramset_type, key, uid raise ValueError( diff --git a/mc/valid.py b/mc/valid.py index 6dc1791..7c209fb 100644 --- a/mc/valid.py +++ b/mc/valid.py @@ -95,7 +95,10 @@ "DiscreteModeChoice": { "params": { "modelType": "Tour", - "selector": "MultinomialLogit" + "selector": "MultinomialLogit", + "tourConstraints": "SubtourMode", + "tourFinder": "PlanBased", + "tourEstimator": "Uniform" }, "parametersets": { "selector:MultinomialLogit": { From c5ed522b43122a0dc5c5d846d5237190dafdce14 Mon Sep 17 00:00:00 2001 From: syhwawa Date: Thu, 9 Nov 2023 19:30:44 +0000 Subject: [PATCH 5/7] Update DMC module to test config and unit test --- tests/test_base_config.py | 18 ++++++++++++++++ tests/test_data/mm_test.xml | 15 ++++++++++++++ tests/test_data/test_columbus_es_config.xml | 16 +++++++++++++- tests/test_data/test_config.json | 23 +++++++++++++++++++++ tests/test_data/test_config.xml | 17 ++++++++++++++- tests/test_data/test_config_v12.xml | 20 ++++++++++++++++++ tests/test_data/test_config_v14.xml | 16 ++++++++++++++ tests/test_data/test_diff.json | 23 +++++++++++++++++++++ tests/test_data/test_temp_config.json | 23 +++++++++++++++++++++ tests/test_data/test_temp_config.xml | 15 ++++++++++++++ 10 files changed, 184 insertions(+), 2 deletions(-) diff --git a/tests/test_base_config.py b/tests/test_base_config.py index 79d096d..79cd99f 100644 --- a/tests/test_base_config.py +++ b/tests/test_base_config.py @@ -39,6 +39,11 @@ def test_paramset_equality_same(): assert test_config1 == test_config2 +def test_DMC_paramset_equality_same(): + test_config1 = BaseConfig(path=env.test_xml_path)["DiscreteModeChoice"]["selector:MultinomialLogit"] + test_config2 = BaseConfig(path=env.test_xml_path)["DiscreteModeChoice"]["selector:MultinomialLogit"] + assert test_config1 == test_config2 + def test_module_equality_same(): test_config1 = BaseConfig(path=env.test_xml_path)["planCalcScore"] test_config2 = BaseConfig(path=env.test_xml_path)["planCalcScore"] @@ -60,6 +65,14 @@ def test_equality_not_same(): def test_xml_and_json_test_configs_equal(): test_config1 = BaseConfig(path=env.test_xml_path) test_config2 = BaseConfig(path=env.test_json_path) + + assert test_config1 == test_config2 + + +def test_xml_and_json_test_DMC_paramset_equal(): + test_config1 = BaseConfig(path=env.test_xml_path)["DiscreteModeChoice"] + test_config2 = BaseConfig(path=env.test_json_path)["DiscreteModeChoice"] + assert test_config1 == test_config2 @@ -131,6 +144,11 @@ def test_paramset_level_1_dict_get(): assert isinstance(test_config['planCalcScore']['scoringParameters::default'], ParamSet) +def test_DMC_paramset_level_1(): + test_config = BaseConfig(path=env.test_xml_path) + assert isinstance(test_config['DiscreteModeChoice']['selector:MultinomialLogit'], ParamSet) + + def test_paramset_level_2_dict_get(): test_config = BaseConfig(path=env.test_xml_path) assert isinstance(test_config['planCalcScore']['scoringParameters::default']['activityParams::home'], ParamSet) diff --git a/tests/test_data/mm_test.xml b/tests/test_data/mm_test.xml index 0ecb9b6..eb8e1d7 100644 --- a/tests/test_data/mm_test.xml +++ b/tests/test_data/mm_test.xml @@ -25,6 +25,21 @@ + + + + + + + + + + + + + + + diff --git a/tests/test_data/test_columbus_es_config.xml b/tests/test_data/test_columbus_es_config.xml index 8c20adc..46591d0 100644 --- a/tests/test_data/test_columbus_es_config.xml +++ b/tests/test_data/test_columbus_es_config.xml @@ -32,7 +32,21 @@ - + + + + + + + + + + + + + + + diff --git a/tests/test_data/test_config.json b/tests/test_data/test_config.json index 4333f0f..d583f23 100644 --- a/tests/test_data/test_config.json +++ b/tests/test_data/test_config.json @@ -83,6 +83,29 @@ "useIndividualSettingsForSubpopulations": "false" } }, + "DiscreteModeChoice": { + "params": { + "modelType": "Tour", + "selector": "MultinomialLogit", + "tourConstraints": "SubtourMode", + "tourFinder": "PlanBased", + "tourEstimator":"Uniform" + }, + "parametersets": { + "selector:MultinomialLogit": { + "params": { + "considerMinimumUtility": "false", + "maximumUtility": "700.0", + "minimumUtility": "-700.0" + } + }, + "modeAvailability:Car": { + "params": { + "availableModes": "car,pt,walk,bike" + } + } + } + }, "subtourModeChoice": { "params": { "chainBasedModes": "car,bike", diff --git a/tests/test_data/test_config.xml b/tests/test_data/test_config.xml index 1e07600..7a4251f 100644 --- a/tests/test_data/test_config.xml +++ b/tests/test_data/test_config.xml @@ -87,6 +87,22 @@ + + + + + + + + + + + + + + + + @@ -106,7 +122,6 @@ - diff --git a/tests/test_data/test_config_v12.xml b/tests/test_data/test_config_v12.xml index cccc08c..a361fd2 100644 --- a/tests/test_data/test_config_v12.xml +++ b/tests/test_data/test_config_v12.xml @@ -85,6 +85,26 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_data/test_config_v14.xml b/tests/test_data/test_config_v14.xml index 831349d..49561f7 100644 --- a/tests/test_data/test_config_v14.xml +++ b/tests/test_data/test_config_v14.xml @@ -102,6 +102,22 @@ + + + + + + + + + + + + + + + + diff --git a/tests/test_data/test_diff.json b/tests/test_data/test_diff.json index d82cd41..efe5a3e 100644 --- a/tests/test_data/test_diff.json +++ b/tests/test_data/test_diff.json @@ -365,6 +365,29 @@ } } }, + "DiscreteModeChoice": { + "params": { + "modelType": "Tour", + "selector": "MultinomialLogit", + "tourConstraints": "SubtourMode", + "tourFinder": "PlanBased", + "tourEstimator":"Uniform" + }, + "parametersets": { + "selector:MultinomialLogit": { + "params": { + "considerMinimumUtility": "false", + "maximumUtility": "700.0", + "minimumUtility": "-700.0" + } + }, + "modeAvailability:Car": { + "params": { + "availableModes": "car,pt,walk,bike" + } + } + } + }, "subtourModeChoice": { "params": { "chainBasedModes": "car,bike", diff --git a/tests/test_data/test_temp_config.json b/tests/test_data/test_temp_config.json index 4333f0f..2e7863d 100644 --- a/tests/test_data/test_temp_config.json +++ b/tests/test_data/test_temp_config.json @@ -83,6 +83,29 @@ "useIndividualSettingsForSubpopulations": "false" } }, + "DiscreteModeChoice": { + "params": { + "modelType": "Tour", + "selector": "MultinomialLogit", + "tourConstraints": "SubtourMode", + "tourFinder": "PlanBased", + "tourEstimator": "Uniform" + }, + "parametersets": { + "selector:MultinomialLogit": { + "params": { + "considerMinimumUtility": "false", + "maximumUtility": "700.0", + "minimumUtility": "-700.0" + } + }, + "modeAvailability:Car": { + "params": { + "availableModes": "car,pt,walk,bike" + } + } + } + }, "subtourModeChoice": { "params": { "chainBasedModes": "car,bike", diff --git a/tests/test_data/test_temp_config.xml b/tests/test_data/test_temp_config.xml index b6dd04d..2e9ff2b 100644 --- a/tests/test_data/test_temp_config.xml +++ b/tests/test_data/test_temp_config.xml @@ -60,6 +60,21 @@ + + + + + + + + + + + + + + + From 68bea5ef47544b487fda0611391c26b854141635 Mon Sep 17 00:00:00 2001 From: syhwawa Date: Mon, 13 Nov 2023 13:20:21 +0000 Subject: [PATCH 6/7] Update unit test and README --- README.md | 5 +++++ tests/test_base_config.py | 31 ++++++++++++++++++++++--------- tests/test_valid_keys.py | 21 +++++++++++++++++++++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4774231..586c03d 100644 --- a/README.md +++ b/README.md @@ -305,6 +305,11 @@ param {'name': 'monetaryDistanceRate', 'value': '-0.0001'} # eg subpop B car In the examples above, you can see that wildcarding with `*` can be used to return 'all' config elements. The `*` operator tells the find method to search all at a given level. As shown above, it is useful for returning all elements within a parameterset or explicitly describing levels to search. +The CLI supports to find the specific modules from given matsim config: +```py +mc find tests/test_data/test_config.json network +``` + ### Find and Set Note that the `find` method is returning a reference to the found config objects, which can then be set: diff --git a/tests/test_base_config.py b/tests/test_base_config.py index 79cd99f..ee8d885 100644 --- a/tests/test_base_config.py +++ b/tests/test_base_config.py @@ -4,9 +4,11 @@ import pytest import env +from unittest.mock import MagicMock + env.set_module() -from mc.base import BaseConfig, Module, ParamSet, Param +from mc.base import BaseConfig, Module, ParamSet, Param, build_paramset_key def test_test_env_paths(): @@ -39,7 +41,7 @@ def test_paramset_equality_same(): assert test_config1 == test_config2 -def test_DMC_paramset_equality_same(): +def test_dmc_paramset_equality_same(): test_config1 = BaseConfig(path=env.test_xml_path)["DiscreteModeChoice"]["selector:MultinomialLogit"] test_config2 = BaseConfig(path=env.test_xml_path)["DiscreteModeChoice"]["selector:MultinomialLogit"] assert test_config1 == test_config2 @@ -62,17 +64,18 @@ def test_equality_not_same(): assert not test_config1 == test_config2 -def test_xml_and_json_test_configs_equal(): - test_config1 = BaseConfig(path=env.test_xml_path) - test_config2 = BaseConfig(path=env.test_json_path) +def test_xml_and_json_test_dmc_paramset_equal(): + test_config1 = BaseConfig(path=env.test_xml_path)["DiscreteModeChoice"] + test_config2 = BaseConfig(path=env.test_json_path)["DiscreteModeChoice"] assert test_config1 == test_config2 -def test_xml_and_json_test_DMC_paramset_equal(): - test_config1 = BaseConfig(path=env.test_xml_path)["DiscreteModeChoice"] - test_config2 = BaseConfig(path=env.test_json_path)["DiscreteModeChoice"] +def test_xml_and_json_test_configs_equal(): + test_config1 = BaseConfig(path=env.test_xml_path) + test_config2 = BaseConfig(path=env.test_json_path) + assert set(test_config1.modules.keys()) == set(test_config2.modules.keys()) assert test_config1 == test_config2 @@ -144,7 +147,7 @@ def test_paramset_level_1_dict_get(): assert isinstance(test_config['planCalcScore']['scoringParameters::default'], ParamSet) -def test_DMC_paramset_level_1(): +def test_dmc_paramset_level_1(): test_config = BaseConfig(path=env.test_xml_path) assert isinstance(test_config['DiscreteModeChoice']['selector:MultinomialLogit'], ParamSet) @@ -198,3 +201,13 @@ def test_param_level_1_key_valid(): with pytest.raises(KeyError): test_config['planCalcScore']['scoringParameters::default'].is_valid_param_key('NOTVALID') + +def test_build_paramset_key_with_colon(): + mock_elem = MagicMock() + mock_elem.attrib = {'type': 'selector:MultinomialLogit'} + test_config = BaseConfig(path=env.test_json_path) + + paramset_type, key, uid = build_paramset_key(mock_elem) + assert paramset_type == 'selector:MultinomialLogit', "Paramset type did not match" + assert key == 'selector:MultinomialLogit', "Key did not match" + assert uid == 'MultinomialLogit', "UID did not match" diff --git a/tests/test_valid_keys.py b/tests/test_valid_keys.py index 47dacbb..3832e2f 100644 --- a/tests/test_valid_keys.py +++ b/tests/test_valid_keys.py @@ -169,3 +169,24 @@ def test_nested_set_paramsets_fail(): c = BaseConfig() with pytest.raises(KeyError): c["planCalcScore"]["INVALID::group1"]["earlyDeparture"] = "1" + + +def test_dmc_module_parameters(): + test_config = BaseConfig(path=env.test_json_path) + dmc_module = test_config["DiscreteModeChoice"] + assert dmc_module["modelType"] == "Tour" + assert dmc_module["selector"] == "MultinomialLogit" + + +def test_dmc_module_parameter_sets(): + test_config = BaseConfig(path=env.test_json_path) + dmc_module = test_config["DiscreteModeChoice"] + multinomial_logit_paramset = dmc_module["selector:MultinomialLogit"] + assert multinomial_logit_paramset["maximumUtility"] == "700.0" + + +def test_dmc_mode_availability_car_parameter_set(): + test_config = BaseConfig(path=env.test_json_path) + dmc_module = test_config["DiscreteModeChoice"] + mode_availability_car_paramset = dmc_module["modeAvailability:Car"] + assert mode_availability_car_paramset["availableModes"] == "car,pt,walk,bike" From 7b9844f023395557224615e2b3e922f66820f666 Mon Sep 17 00:00:00 2001 From: syhwawa Date: Mon, 20 Nov 2023 15:11:27 +0000 Subject: [PATCH 7/7] Update the load overrides function --- mc/autostep.py | 5 ++++- scripts/.coveragerc | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mc/autostep.py b/mc/autostep.py index 978818d..e819590 100644 --- a/mc/autostep.py +++ b/mc/autostep.py @@ -271,10 +271,13 @@ def dump_log_to_disk(log: list, path): def load_overrides(overrides_path: Path) -> dict: + if not overrides_path.exists(): + logging.warning(f"Overrides file not found: {overrides_path}") + return {} with open(overrides_path) as o: overrides = json.loads(o.read()) result = {} - for i, val in enumerate(overrides): + for val in overrides: for k, v in val.items(): result[k] = v return result diff --git a/scripts/.coveragerc b/scripts/.coveragerc index 6fc0064..59ae0bc 100644 --- a/scripts/.coveragerc +++ b/scripts/.coveragerc @@ -3,4 +3,4 @@ branch = True omit = venv/*, tests/*, mc/bitsim/*, mc/setup.py [report] -fail_under = 85 +fail_under = 84