clean up and update docker files

telematic_system/docker-compose.local.yml

@@ -101,6 +101,20 @@ services:
         - DB_USER=telematic
         - DB_PASSWORD=telematic
         - GRAFANA_DB=wfd_grafana
+        # File upload settings
+        - UPLOAD_DESTINATION=host
+        - UPLOAD_DESTINATION_PATH=/opt/telematics/upload
+        - AWS_ACCESS_KEY_ID=
+        - AWS_SECRET_KEY=
+        - S3_BUCKET=
+        - S3_REGION=us-east-2
+        - UPLOAD_HTTP_PORT = 9011
+        - UPLOAD_TIME_OUT=3600000 # Milliseconds
+        - UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB
+        - CONCURRENT_QUEUE_SIZE=5 
+        - PART_SIZE = 10485760 
+        - NATS_SERVERS = localhost:4222
+        - FILE_PROCESSING_SUBJECT=ui.file.procressing
       volumes:
         - /opt/apache2/grafana_htpasswd:/opt/apache2/grafana_htpasswd
 

telematic_system/docker-compose.webapp.yml

@@ -23,8 +23,23 @@ services:
         - DB_USER=<DB_USER>
         - DB_PASSWORD=<DB_PASSWORD>
         - GRAFANA_DB=<GRAFANA_DB>
+        # File upload settings
+        - UPLOAD_DESTINATION=s3
+        - UPLOAD_DESTINATION_PATH=/opt/telematics/upload
+        - AWS_ACCESS_KEY_ID=
+        - AWS_SECRET_KEY=
+        - S3_BUCKET=
+        - S3_REGION=us-east-2
+        - UPLOAD_HTTP_PORT = 9011
+        - UPLOAD_TIME_OUT=3600000 # Milliseconds
+        - UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB
+        - CONCURRENT_QUEUE_SIZE=5 
+        - PART_SIZE = 10485760 
+        - NATS_SERVERS = localhost:4222
+        - FILE_PROCESSING_SUBJECT=ui.file.procressing
       volumes:
         - /opt/apache2/grafana_htpasswd:/opt/apache2/grafana_htpasswd
+        - /opt/telematics/upload:/opt/telematics/upload
 
   telematic_web_client:
     build:

telematic_system/telematic.env

@@ -42,3 +42,46 @@ KAFKA_CONSUMER_RESET=earliest
 STREETS_BRIDGE_EXCLUSION_LIST=
 VEHICLE_BRIDGE_EXCLUSION_LIST= /environment/map_filtered_points,/environment/points_clustered,/environment/points_in_map,/environment/detected_objects,/environment/roadway_objects,/environment/motion_computation_visualization,/hardware_interface/lidar/points_raw,/environment/points_ground,/environment/points_no_ground,/environment/points_in_base_link,/environment/lidar_detected_objects,/environment/map_update,/environment/semantic_map,/environment/base_map,/diagnostics,/environment/base_map,/environment/bounding_box_converter/transition_event,/environment/carma_wm_broadcaster/transition_event,/environment/detected_objects,/environment/environment_perception_controller/transition_event,/environment/external_object/transition_event,/environment/external_objects,/environment/external_objects_viz,/environment/leftover_clusters,/environment/lidar_bounding_boxes,/environment/lidar_bounding_boxes_viz,/environment/lidar_detected_objects,/environment/lidar_frame_transformer/transition_event,/environment/lidar_to_map_frame_transformer/transition_event,/environment/map_filtered_points,/environment/map_update,/environment/motion_computation_node/transition_event,/environment/motion_computation_visualize,/environment/motion_prediction_visualizer/transition_event,/environment/object_visualizer_node/transition_event,/environment/perception_points_filter_container/transition_event,/environment/points_clustered,/environment/points_ground,/environment/points_in_base_link,/environment/points_in_map,/environment/points_map_filter/transition_event,/environment/points_no_ground,/environment/roadway_objects,/environment/roadway_objects_node/transition_event,/environment/roadway_obstacles,/environment/roadway_obstacles_viz,/environment/semantic_map,/environment/tcm_visualizer,/environment/tracked_objects,/environment/traffic_incident_parser_node/transition_event,/guidance/arbitrator/transition_event,/guidance/cav_marker,/guidance/guidance_controller/transition_event,/guidance/guidance_node/transition_event,/guidance/host_marker,/guidance/label_marker,/guidance/mobilitypath_visualizer_node/transition_event,/guidance/plan_delegator/transition_event,/guidance/plugin_discovery,/guidance/plugins/inlanecruising_plugin/transition_event,/guidance/plugins/light_controlled_intersection_tactical_plugin/transition_event,/guidance/plugins/platooning_tactical_plugin_node/transition_event,/guidance/plugins/pure_pursuit_wrapper/transition_event,/guidance/plugins/route_following_plugin/transition_event,/guidance/plugins/sci_strategic_plugin/transition_event,/guidance/plugins/stop_and_wait_plugin/transition_event,/guidance/plugins/stop_controlled_intersection_tactical_plugin/transition_event,/guidance/plugins/yield_plugin/transition_event,/guidance/port_drayage_plugin_node/transition_event,/guidance/route_marker,/guidance/route_node/transition_event,/guidance/trajectory_executor_node/transition_event,/guidance/trajectory_visualizer,/guidance/trajectory_visualizer_node/transition_event,/hardware_interface/accel_aux_rpt,/hardware_interface/all_system_statuses,/hardware_interface/as/speed_model/transition_event,/hardware_interface/as/speed_model_config,/hardware_interface/as/steering_model/transition_event,/hardware_interface/as/steering_model_config,/hardware_interface/as/veh_controller/transition_event,/hardware_interface/as/veh_controller_config,/hardware_interface/as/veh_interface/transition_event,/hardware_interface/as/veh_interface_config,/hardware_interface/carma_novatel_driver_wrapper_node/transition_event,/hardware_interface/comms/inbound_binary_msg,/hardware_interface/comms/outbound_binary_msg,/hardware_interface/component_rpt,/hardware_interface/date_time_rpt,/hardware_interface/driver_shutdown_5cff53178b6c4b64891942011c611f9a/transition_event,/hardware_interface/driver_shutdown_8767cd437e204a1491116fbacbe0e0bc/transition_event,/hardware_interface/driver_shutdown_ab3249fe2c1842489af73c22ad5fd775/transition_event,/hardware_interface/driver_shutdown_fd9b1bee52464111a097362c7cfc70f4/transition_event,/hardware_interface/drivers_controller/transition_event,/hardware_interface/dsrc_driver_node/transition_event,/hardware_interface/imu_raw,/hardware_interface/interior_lights_rpt,/hardware_interface/lat_lon_heading_rpt,/hardware_interface/libkvaser_reader_node/transition_event,/hardware_interface/libkvaser_writer_node/transition_event,/hardware_interface/lidar/points_raw,/hardware_interface/novatel/oem7/bestpos,/hardware_interface/novatel/oem7/bestutm,/hardware_interface/novatel/oem7/bestvel,/hardware_interface/novatel/oem7/corrimu,/hardware_interface/novatel/oem7/fix,/hardware_interface/novatel/oem7/gps,/hardware_interface/novatel/oem7/heading2,/hardware_interface/novatel/oem7/imu/data,/hardware_interface/novatel/oem7/insconfig,/hardware_interface/novatel/oem7/inspva,/hardware_interface/novatel/oem7/inspvax,/hardware_interface/novatel/oem7/insstdev,/hardware_interface/novatel/oem7/odom,/hardware_interface/novatel/oem7/oem7raw,/hardware_interface/novatel/oem7/rxstatus,/hardware_interface/novatel/oem7/time,/hardware_interface/pacmod3_node/transition_event,/hardware_interface/ssc_converter_node/transition_event,/hardware_interface/ssc_interface_wrapper_node/transition_event,/hardware_interface/velodyne_lidar_driver_wrapper_node/transition_event,/hardware_interface/velodyne_packets,/joint_states,/localization/gnss_to_map_convertor/transition_event,/localization/localization_controller/transition_event,/localization/localization_manager/transition_event,/localization/map_param_loader/georeference,/message/bsm_generator_node/transition_event,/message/carma_cloud_client_node/transition_event,/message/cpp_message_node/transition_event,/message/incoming_j2735_bsm,/message/incoming_j2735_geofence_control,/message/incoming_j2735_geofence_request,/message/incoming_j2735_map,/message/incoming_j2735_psm,/message/incoming_j2735_spat,/message/incoming_map,/message/incoming_mobility_operation,/message/incoming_mobility_path,/message/incoming_mobility_request,/message/incoming_mobility_response,/message/incoming_psm,/message/incoming_spat,/message/j2735_convertor_node/transition_event,/message/mobilitypath_publisher_node/transition_event,/message/outgoing_j2735_bsm,/message/outgoing_j2735_geofence_control,/message/outgoing_j2735_geofence_request,/message/outgoing_j2735_psm,/message/v2x_controller/transition_event,/parameter_events,/robot_description,/rosout,/system_alert,/tf,/tf_static,/ui/client_count,/ui/connected_clients,/ui/ui_instructions,/guidance/twist_filter_node/transition_event,/guidance/approaching_emergency_vehicle_plugin/transition_event,/guidance/plugins/cooperative_lanechange/transition_event,/guidance/twist_gate_node/transition_event,/hardware_interface/driver_shutdown_494b616fe26443fca421ea1f2e04d1d4/transition_event,/hardware_interface/driver_shutdown_77e0426440b045b580837068ea149ea1/transition_event,/hardware_interface/driver_shutdown_9219ae46d4084143a883557f6b60bfc3/transition_event,/hardware_interface/driver_shutdown_494b616fe26443fca421ea1f2e04d1d4/transition_event,/hardware_interface/driver_shutdown_77e0426440b045b580837068ea149ea1/transition_event,/hardware_interface/driver_shutdown_9219ae46d4084143a883557f6b60bfc3/transition_event,/hardware_interface/lightbar_manager/transition_event,/hardware_interface/lightbar_manager_container/transition_event,/guidance/plugins/approaching_emergency_vehicle_plugin/transition_event,/environment/detected_objects,/environment/external_object_predictions,/environment/external_objects,/environment/intersection_signal_group_ids,/environment/leftover_clusters,/environment/map_filtered_points,/environment/roadway_objects,/environment/roadway_obstacles,/environment/tcm_visualizer,/environment/tcr_bounding_points,/guidance/final_maneuver_plan,/guidance/plan_trajectory,/guidance/plugins/cooperative_lanechange/transition_event,/guidance/plugins/debug/trajectory_planning,/guidance/plugins/platoon_control/plan_trajectory,/guidance/plugins/pure_pursuit/plan_trajectory,/guidance/route,/guidance/upcoming_lane_change_status,/hardware_interface/accel_aux_rpt,/hardware_interface/all_system_statuses,/hardware_interface/as/brake_command_echo,/hardware_interface/as/brake_feedback,/hardware_interface/as/curvature_feedback,/hardware_interface/as/gear_command_echo,/hardware_interface/as/gear_feedback,/hardware_interface/as/gear_select,/hardware_interface/as/pacmod/as_rx/shift_cmd,/hardware_interface/as/pacmod/parsed_tx/global_rpt,/hardware_interface/as/pacmod/parsed_tx/shift_rpt,/hardware_interface/as/pacmod/parsed_tx/wheel_speed_rpt,/hardware_interface/as/speed_pedals,/hardware_interface/as/steering_command_echo,/hardware_interface/as/steering_feedback,/hardware_interface/as/steering_wheel,/hardware_interface/as/throttle_command_echo,/hardware_interface/as/throttle_feedback,/hardware_interface/as/turn_signal_command,/hardware_interface/as/vehicle_platform,/hardware_interface/as/velocity_accel_cov,/hardware_interface/brake_aux_rpt,/hardware_interface/can/brake_position,/hardware_interface/can/steering_wheel_angle,/hardware_interface/can/transmission_state,/hardware_interface/can_rx,/hardware_interface/can_tx,/hardware_interface/hazard_lights_cmd,/hardware_interface/hazard_lights_rpt,/hardware_interface/headlight_aux_rpt,/hardware_interface/headlight_cmd,/hardware_interface/headlight_rpt,/hardware_interface/horn_cmd,/hardware_interface/horn_rpt,/hardware_interface/lat_lon_heading_rpt,/hardware_interface/pacmod/parsed_tx/brake_rpt,/hardware_interface/pacmod/parsed_tx/shift_rpt,/hardware_interface/pacmod/parsed_tx/steer_rpt,/hardware_interface/shift_aux_rpt,/hardware_interface/state,/hardware_interface/steering_aux_rpt,/hardware_interface/turn_aux_rpt,/hardware_interface/turn_rpt,/hardware_interface/vehicle/engage,/hardware_interface/velocity_accel_cov,/hardware_interface/vin_rpt,/localization/current_pose_with_covariance,/localization/gnss_pose,/localization/initialpose,/localization/localization_status,/localization/managed_initialpose,/localization/map_param_loader/georeference,/localization/ndt_pose,/localization/ndt_stat,/localization/selected_pose,/robot_description,/message/j2735_convertor_node/transition_event,/hardware_interface/driver_shutdown_81dc0955537246e38b742b0cc74a1a8d/transition_event,/hardware_interface/driver_shutdown_8d5d9db8edc243aca0016a9d9e60e918/transition_event,/hardware_interface/driver_shutdown_98a97ce77cda4cfbb7e8912c56276e62/transition_event,/hardware_interface/driver_shutdown_c19bda6e09c2471bbb762c8d2f0af03d/transition_event,/guidance/twist_filter_node/limitation_debug/ctrl/lateral_accel,/guidance/twist_filter_node/limitation_debug/ctrl/lateral_jerk,/guidance/twist_filter_node/limitation_debug/twist/lateral_accel,/guidance/twist_filter_node/limitation_debug/twist/lateral_jerk,/hardware_interface/as/pacmod/as_rx/enable,/hardware_interface/as/dbw_enabled_feedback,/hardware_interface/as/module_states
 CLOUD_BRIDGE_EXCLUSION_LIST=
+
+## Web server
+# Server port 
+PORT=9010
+#Allowed client url
+ALLOW_CLIENT_URL=http://local.cav-telematics.com:8888
+
+# DB credentials
+DB_HOST=localhost
+DB_USER=root
+DB_PASSWORD=root
+GRAFANA_DB=wfd_grafana
+DB_PORT=3307
+
+# Security
+GRAFANA_HTPASSWORD=/opt/apache2/grafana_htpasswd
+SESSION_TIMEOUT=3600
+SECRET=my_secret_telematic_ui
+
+# File upload settings
+### Destination: HOST or S3
+### If S3, files are uploaded to the s3 bucket. Otherwise, files are uploaded to HOST machine with destination path
+UPLOAD_DESTINATION=host
+UPLOAD_DESTINATION_PATH=/opt/telematics/upload
+
+# S3 bucket credentials
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_KEY=
+S3_BUCKET=
+S3_REGION=us-east-2
+
+# File upload service
+UPLOAD_HTTP_PORT = 9011
+UPLOAD_TIME_OUT=3600000 # Milliseconds
+UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB
+CONCURRENT_QUEUE_SIZE=5 # How many parts can be parallel processed
+PART_SIZE = 10485760 # The size of each part during a multipart upload, in bytes, at least 10MB
+
+# NATS config
+NATS_SERVERS = localhost:4222
+### This topic is used by processing service. Make sure this is consistent with processing service topic setup
+FILE_PROCESSING_SUBJECT=ui.file.procressing
+

telematic_system/telematic_apps/web_app/server/Dockerfile

@@ -12,5 +12,6 @@ RUN npm install
 
 # Bundle app source
 COPY . .
+RUN chmod +x service.sh
 
-ENTRYPOINT ["npm", "start"]
+CMD ./service.sh

telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js

@@ -14,7 +14,6 @@
  * the License.
  */
 const { file_info } = require("../models");
-const ADMIN_ID = 1;
 
 /**
  * List file info
@@ -42,7 +41,7 @@ exports.list = (filterFields) => {
       return data;
     })
     .catch((err) => {
-      console.log("Error findAll file info from MYSQL DB!");
+      console.error("Error findAll file info from MYSQL DB!");
       throw err;
     });
 };

telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js

@@ -22,7 +22,7 @@ const filterFiles = async (req_fields) => {
   try {
     return await fileInfoController.list(req_fields);
   } catch (err) {
-    console.log("Cannot filter a list of DB files!");
+    console.error("Cannot filter a list of DB files!");
     console.trace();
     throw err;
   }
@@ -36,7 +36,7 @@ const listAllFiles = async () => {
       return await listAllDBFiles();
     }
   } catch (err) {
-    console.log(err);
+    console.error(err);
     throw err;
   }
 };
@@ -45,7 +45,7 @@ const listAllDBFiles = async () => {
   try {
     return await fileInfoController.list({});
   } catch (err) {
-    console.log("Cannot get a list of All DB files!");
+    console.error("Cannot get a list of All DB files!");
     console.trace();
     throw err;
   }
@@ -78,13 +78,12 @@ const listAllDBFilesAndS3Objects = async () => {
             .upsertFileInfo(newFileFromS3)
             .catch((error) => console.log(error));
           contents.push(newFile);
-          console.log(newFile);
         }
       }
     }
     return contents;
   } catch (err) {
-    console.log("Cannot get a list of all DB files or S3 objects!");
+    console.error("Cannot get a list of all DB files or S3 objects!");
     console.trace();
     throw err;
   }

telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js

@@ -68,7 +68,7 @@ exports.uploadFile = async (req) => {
       }
     });
   } catch (error) {
-    console.log(error);
+    console.error(error);
     throw error;
   }
 }; //End upload file

telematic_system/telematic_apps/web_app/server/service.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Start the main app
+npm start & 
+
+# Start the file upload service
+npm run file-upload-server &
+
+# Wait for any process to exit
+wait -n
+
+# Exit with status of process that exited first
+exit $?

telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js

@@ -22,7 +22,7 @@ describe("POST file upload service", () => {
 
   it("/api/upload ", async () => {
     jest
-    .spyOn(NATSConnModule, "CreateNatsConn")
+    .spyOn(NATSConnModule, "createNatsConn")
     .mockResolvedValueOnce(NATSMock.connect());
     let filename = "mock.txt";
     let formData = {

telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js

@@ -1,4 +1,4 @@
-const { CreateNatsConn } = require("../../nats_client/nats_connection");
+const { createNatsConn } = require("../../nats_client/nats_connection");
 const {
   pubFileProcessingReq,
 } = require("../../nats_client/file_processing_nats_publisher");
@@ -7,7 +7,7 @@ require("dotenv").config();
 describe("Test NATS publisher", () => {
   it("Test NATS publish to topic ui.file.procressing", async () => {
     try {
-      const natsConn = await CreateNatsConn();
+      const natsConn = await createNatsConn();
       if (natsConn) {
         let processingReq = {
           filepath: "/opt/telematic/test.txt",