From 850b4d0fde66b3f36d4749b5d17493e01e376c02 Mon Sep 17 00:00:00 2001 From: paulbourelly999 <77466294+paulbourelly999@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:49:25 -0500 Subject: [PATCH] Add CollisionTest and RedLighRunningTest to scenario (#317) # PR Details ## Description Adding CollisionTest and RedLightRunning test to scenario definition. CollisionTest adds collision detector to vehicle ([atomic_criteria.py](https://github.com/carla-simulator/scenario_runner/blob/0ff0ebe7fb0ee2de1f157b11ed450e7b981398bf/srunner/scenariomanager/scenarioatomics/atomic_criteria.py#L313)). Both test criteria are evaluated for the run and generate a JSON file indicating success or failure as well as information about any collisions or events in which we run a red light ```json { "CollisionTest": { "children": [], "feedback_message": "", "blackbox_level": 4, "_terminate_on_failure": false, "test_status": "FAILURE", "expected_value_success": 0, "expected_value_acceptable": null, "actual_value": 1, "optional": false, "_collision_sensor": null, "other_actor": null, "other_actor_type": null, "registered_collisions": [], "last_id": null, "collision_time": 27.05000040307641, "terminate_on_failure": false }, "RunningRedLightTest": { "children": [], "feedback_message": "", "blackbox_level": 4, "_terminate_on_failure": false, "test_status": "SUCCESS", "expected_value_success": 0, "expected_value_acceptable": null, "actual_value": 0, "optional": false, "list_traffic_events": [], "_last_red_light_id": null, "debug": false, "terminate_on_failure": false } } ``` **NOTE** : While the collision test seems to be correctly recognizing collisions and the time they occur at it is not currently providing data about the collisions likes actors involved, type of collision, kinematic information. This is likely a bug in CarlaScenarioRunner. Investigation for this information will be considered outside of this PR ## Related Issue CDAR-746 ## Motivation and Context Collect data on collisions and traffic violations ## How Has This Been Tested? CDASim deployment ## Types of changes - [ ] Defect fix (non-breaking change that fixes an issue) - [x] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that cause existing functionality to change) ## Checklist: - [ ] I have added any new packages to the sonar-scanner.properties file - [x] My change requires a change to the documentation. - [x] I have updated the documentation accordingly. - [x] I have read the **CONTRIBUTING** document. [CARMA Contributing Guide](https://github.com/usdot-fhwa-stol/carma-platform/blob/develop/Contributing.md) - [ ] I have added tests to cover my changes. - [ ] All new and existing tests passed. --- ail_vru_uc1_scenario/README.md | 40 ++++++++++++++++++- .../scenario-runner/vru_collision.py | 18 +++++---- ail_vru_uc1_scenario/docker-compose.yml | 1 + 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/ail_vru_uc1_scenario/README.md b/ail_vru_uc1_scenario/README.md index 5f77cc8e..87f1b66f 100644 --- a/ail_vru_uc1_scenario/README.md +++ b/ail_vru_uc1_scenario/README.md @@ -26,5 +26,43 @@ The Virtual Signal Controller is built locally and currently only available to l ## Data Collection ### Carla Recorder -This **CARMA Config** includes volumes and images that will use **Carla's** [recorder](https://carla.readthedocs.io/en/0.9.10/adv_recorder/) functionality to record **CARLA** simulation data into a `carla-recorder/` directory. Included will be a `Trb2024_1.json` file and a `Trb2024_1.log` file. The name of the file comes from the scenario name defined in the **scenario-runner** image. The `.json` file is a criteria file created by the **scenario runner** image and the `.log` file the carla simulation recording created by the carla server running in the **CDASim** image(see https://carla-scenariorunner.readthedocs.io/en/latest/metrics_module/). Using the metrics module in carla-scenario-runner we can define metrics in carla and evaluate the metrics from the carla recordings. This works for collision monitoring and will be prototyped for a custom defined near miss metric. +This **CARMA Config** includes volumes and images that will use **Carla's** [recorder](https://carla.readthedocs.io/en/0.9.10/adv_recorder/) functionality to record **CARLA** simulation data into a `carla-recorder/` directory. Included will be a `Trb2024_1.json` file and a `Trb2024_1.log` file. The name of the file comes from the scenario name defined in the **scenario-runner** image. The `.json` file is a criteria file created by the **scenario runner** image and the `.log` file the carla simulation recording created by the carla server running in the **CDASim** image(see https://carla-scenariorunner.readthedocs.io/en/latest/metrics_module/).The criteria json for the VulnerableRoadUser scenarion should look something like this : +```json +{ + "CollisionTest": { + "children": [], + "feedback_message": "", + "blackbox_level": 4, + "_terminate_on_failure": false, + "test_status": "FAILURE", + "expected_value_success": 0, + "expected_value_acceptable": null, + "actual_value": 1, + "optional": false, + "_collision_sensor": null, + "other_actor": null, + "other_actor_type": null, + "registered_collisions": [], + "last_id": null, + "collision_time": 27.05000040307641, + "terminate_on_failure": false + }, + "RunningRedLightTest": { + "children": [], + "feedback_message": "", + "blackbox_level": 4, + "_terminate_on_failure": false, + "test_status": "SUCCESS", + "expected_value_success": 0, + "expected_value_acceptable": null, + "actual_value": 0, + "optional": false, + "list_traffic_events": [], + "_last_red_light_id": null, + "debug": false, + "terminate_on_failure": false + } +} +``` +This JSON file will provide information on whether any Collisions and RedLightRunning events took place during the run. Both are considered conditions for a test failure.The JSON should also provide information about the event that caused test failures like time, actor involved, and other event information althoguth this functionality seems to be currently broken, likely due to Carla Scenario Runner bugs. diff --git a/ail_vru_uc1_scenario/cdasim_config/scenario-runner/vru_collision.py b/ail_vru_uc1_scenario/cdasim_config/scenario-runner/vru_collision.py index a8b4f50b..5f1c18d0 100644 --- a/ail_vru_uc1_scenario/cdasim_config/scenario-runner/vru_collision.py +++ b/ail_vru_uc1_scenario/cdasim_config/scenario-runner/vru_collision.py @@ -25,7 +25,10 @@ KeepVelocity, WaypointFollower, ) -from srunner.scenariomanager.scenarioatomics.atomic_criteria import CollisionTest +from srunner.scenariomanager.scenarioatomics.atomic_criteria import ( + CollisionTest, + RunningRedLightTest +) from srunner.scenariomanager.scenarioatomics.atomic_trigger_conditions import ( DriveDistance, InTriggerDistanceToVehicle, @@ -132,7 +135,7 @@ def _create_behavior(self): :return: Behavior tree root """ - carma_vehicle = None + self.carma_vehicle = None for actor in CarlaDataProvider.get_world().get_actors(): if "role_name" in actor.attributes: print(f"ACTOR ROLE: {actor.attributes['role_name']}") @@ -142,7 +145,7 @@ def _create_behavior(self): "role_name" in actor.attributes and actor.attributes["role_name"] == "carma_1" ): - carma_vehicle = actor + self.carma_vehicle = actor break CarlaDataProvider.register_actor(actor) @@ -150,7 +153,7 @@ def _create_behavior(self): start_condition = Idle(5, name="start_condition") start_condition = InTriggerDistanceToVehicle( - crossing_person, carma_vehicle, WALKING_PERSON_TRIGGER_WALKING_DISTANCE_IN_METERS + crossing_person, self.carma_vehicle, WALKING_PERSON_TRIGGER_WALKING_DISTANCE_IN_METERS ) walk_across_street = KeepVelocity( @@ -164,7 +167,7 @@ def _create_behavior(self): actor_behaviors = py_trees.composites.Parallel(name="actor_behaviors") actor_behaviors.add_child(walk_across_street) - end_condition = DriveDistance(carma_vehicle, 100) + end_condition = DriveDistance(self.carma_vehicle, 100) root = py_trees.composites.Sequence(name="root_sequence") root.add_child(start_condition) @@ -182,9 +185,8 @@ def _create_test_criteria(self) -> list: :return: List of test criteria """ return [ - # This is an example usage for including test criteria in - # ScenarioRunner. - # CollisionTest(self.carma_vehicle) + CollisionTest(self.carma_vehicle), + RunningRedLightTest(self.carma_vehicle) ] def __del__(self): diff --git a/ail_vru_uc1_scenario/docker-compose.yml b/ail_vru_uc1_scenario/docker-compose.yml index 23717873..6d79f9e4 100644 --- a/ail_vru_uc1_scenario/docker-compose.yml +++ b/ail_vru_uc1_scenario/docker-compose.yml @@ -409,6 +409,7 @@ services: && python3 scenario_runner.py --scenario VulnerableRoadUserCollision_LeftTurn --record metrics/data/ + --debug --host 172.2.0.2" networks: