diff --git a/telematic_system/docker-compose.local.yml b/telematic_system/docker-compose.local.yml index f9768743..aa1b09e8 100644 --- a/telematic_system/docker-compose.local.yml +++ b/telematic_system/docker-compose.local.yml @@ -101,6 +101,20 @@ services: - DB_USER=telematic - DB_PASSWORD=telematic - GRAFANA_DB=wfd_grafana + # File upload settings + - UPLOAD_DESTINATION=host + - UPLOAD_DESTINATION_PATH=/opt/telematics/upload + - AWS_ACCESS_KEY_ID= + - AWS_SECRET_KEY= + - S3_BUCKET= + - S3_REGION=us-east-2 + - UPLOAD_HTTP_PORT = 9011 + - UPLOAD_TIME_OUT=3600000 # Milliseconds + - UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB + - CONCURRENT_QUEUE_SIZE=5 + - PART_SIZE = 10485760 + - NATS_SERVERS = localhost:4222 + - FILE_PROCESSING_SUBJECT=ui.file.procressing volumes: - /opt/apache2/grafana_htpasswd:/opt/apache2/grafana_htpasswd diff --git a/telematic_system/docker-compose.webapp.yml b/telematic_system/docker-compose.webapp.yml index 5ed19090..73dec846 100644 --- a/telematic_system/docker-compose.webapp.yml +++ b/telematic_system/docker-compose.webapp.yml @@ -23,8 +23,23 @@ services: - DB_USER= - DB_PASSWORD= - GRAFANA_DB= + # File upload settings + - UPLOAD_DESTINATION=s3 + - UPLOAD_DESTINATION_PATH=/opt/telematics/upload + - AWS_ACCESS_KEY_ID= + - AWS_SECRET_KEY= + - S3_BUCKET= + - S3_REGION=us-east-2 + - UPLOAD_HTTP_PORT = 9011 + - UPLOAD_TIME_OUT=3600000 # Milliseconds + - UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB + - CONCURRENT_QUEUE_SIZE=5 + - PART_SIZE = 10485760 + - NATS_SERVERS = localhost:4222 + - FILE_PROCESSING_SUBJECT=ui.file.procressing volumes: - /opt/apache2/grafana_htpasswd:/opt/apache2/grafana_htpasswd + - /opt/telematics/upload:/opt/telematics/upload telematic_web_client: build: diff --git a/telematic_system/telematic.env b/telematic_system/telematic.env index 4790d707..3fb8b1d3 100644 --- a/telematic_system/telematic.env +++ b/telematic_system/telematic.env @@ -42,3 +42,46 @@ KAFKA_CONSUMER_RESET=earliest STREETS_BRIDGE_EXCLUSION_LIST= VEHICLE_BRIDGE_EXCLUSION_LIST= /environment/map_filtered_points,/environment/points_clustered,/environment/points_in_map,/environment/detected_objects,/environment/roadway_objects,/environment/motion_computation_visualization,/hardware_interface/lidar/points_raw,/environment/points_ground,/environment/points_no_ground,/environment/points_in_base_link,/environment/lidar_detected_objects,/environment/map_update,/environment/semantic_map,/environment/base_map,/diagnostics,/environment/base_map,/environment/bounding_box_converter/transition_event,/environment/carma_wm_broadcaster/transition_event,/environment/detected_objects,/environment/environment_perception_controller/transition_event,/environment/external_object/transition_event,/environment/external_objects,/environment/external_objects_viz,/environment/leftover_clusters,/environment/lidar_bounding_boxes,/environment/lidar_bounding_boxes_viz,/environment/lidar_detected_objects,/environment/lidar_frame_transformer/transition_event,/environment/lidar_to_map_frame_transformer/transition_event,/environment/map_filtered_points,/environment/map_update,/environment/motion_computation_node/transition_event,/environment/motion_computation_visualize,/environment/motion_prediction_visualizer/transition_event,/environment/object_visualizer_node/transition_event,/environment/perception_points_filter_container/transition_event,/environment/points_clustered,/environment/points_ground,/environment/points_in_base_link,/environment/points_in_map,/environment/points_map_filter/transition_event,/environment/points_no_ground,/environment/roadway_objects,/environment/roadway_objects_node/transition_event,/environment/roadway_obstacles,/environment/roadway_obstacles_viz,/environment/semantic_map,/environment/tcm_visualizer,/environment/tracked_objects,/environment/traffic_incident_parser_node/transition_event,/guidance/arbitrator/transition_event,/guidance/cav_marker,/guidance/guidance_controller/transition_event,/guidance/guidance_node/transition_event,/guidance/host_marker,/guidance/label_marker,/guidance/mobilitypath_visualizer_node/transition_event,/guidance/plan_delegator/transition_event,/guidance/plugin_discovery,/guidance/plugins/inlanecruising_plugin/transition_event,/guidance/plugins/light_controlled_intersection_tactical_plugin/transition_event,/guidance/plugins/platooning_tactical_plugin_node/transition_event,/guidance/plugins/pure_pursuit_wrapper/transition_event,/guidance/plugins/route_following_plugin/transition_event,/guidance/plugins/sci_strategic_plugin/transition_event,/guidance/plugins/stop_and_wait_plugin/transition_event,/guidance/plugins/stop_controlled_intersection_tactical_plugin/transition_event,/guidance/plugins/yield_plugin/transition_event,/guidance/port_drayage_plugin_node/transition_event,/guidance/route_marker,/guidance/route_node/transition_event,/guidance/trajectory_executor_node/transition_event,/guidance/trajectory_visualizer,/guidance/trajectory_visualizer_node/transition_event,/hardware_interface/accel_aux_rpt,/hardware_interface/all_system_statuses,/hardware_interface/as/speed_model/transition_event,/hardware_interface/as/speed_model_config,/hardware_interface/as/steering_model/transition_event,/hardware_interface/as/steering_model_config,/hardware_interface/as/veh_controller/transition_event,/hardware_interface/as/veh_controller_config,/hardware_interface/as/veh_interface/transition_event,/hardware_interface/as/veh_interface_config,/hardware_interface/carma_novatel_driver_wrapper_node/transition_event,/hardware_interface/comms/inbound_binary_msg,/hardware_interface/comms/outbound_binary_msg,/hardware_interface/component_rpt,/hardware_interface/date_time_rpt,/hardware_interface/driver_shutdown_5cff53178b6c4b64891942011c611f9a/transition_event,/hardware_interface/driver_shutdown_8767cd437e204a1491116fbacbe0e0bc/transition_event,/hardware_interface/driver_shutdown_ab3249fe2c1842489af73c22ad5fd775/transition_event,/hardware_interface/driver_shutdown_fd9b1bee52464111a097362c7cfc70f4/transition_event,/hardware_interface/drivers_controller/transition_event,/hardware_interface/dsrc_driver_node/transition_event,/hardware_interface/imu_raw,/hardware_interface/interior_lights_rpt,/hardware_interface/lat_lon_heading_rpt,/hardware_interface/libkvaser_reader_node/transition_event,/hardware_interface/libkvaser_writer_node/transition_event,/hardware_interface/lidar/points_raw,/hardware_interface/novatel/oem7/bestpos,/hardware_interface/novatel/oem7/bestutm,/hardware_interface/novatel/oem7/bestvel,/hardware_interface/novatel/oem7/corrimu,/hardware_interface/novatel/oem7/fix,/hardware_interface/novatel/oem7/gps,/hardware_interface/novatel/oem7/heading2,/hardware_interface/novatel/oem7/imu/data,/hardware_interface/novatel/oem7/insconfig,/hardware_interface/novatel/oem7/inspva,/hardware_interface/novatel/oem7/inspvax,/hardware_interface/novatel/oem7/insstdev,/hardware_interface/novatel/oem7/odom,/hardware_interface/novatel/oem7/oem7raw,/hardware_interface/novatel/oem7/rxstatus,/hardware_interface/novatel/oem7/time,/hardware_interface/pacmod3_node/transition_event,/hardware_interface/ssc_converter_node/transition_event,/hardware_interface/ssc_interface_wrapper_node/transition_event,/hardware_interface/velodyne_lidar_driver_wrapper_node/transition_event,/hardware_interface/velodyne_packets,/joint_states,/localization/gnss_to_map_convertor/transition_event,/localization/localization_controller/transition_event,/localization/localization_manager/transition_event,/localization/map_param_loader/georeference,/message/bsm_generator_node/transition_event,/message/carma_cloud_client_node/transition_event,/message/cpp_message_node/transition_event,/message/incoming_j2735_bsm,/message/incoming_j2735_geofence_control,/message/incoming_j2735_geofence_request,/message/incoming_j2735_map,/message/incoming_j2735_psm,/message/incoming_j2735_spat,/message/incoming_map,/message/incoming_mobility_operation,/message/incoming_mobility_path,/message/incoming_mobility_request,/message/incoming_mobility_response,/message/incoming_psm,/message/incoming_spat,/message/j2735_convertor_node/transition_event,/message/mobilitypath_publisher_node/transition_event,/message/outgoing_j2735_bsm,/message/outgoing_j2735_geofence_control,/message/outgoing_j2735_geofence_request,/message/outgoing_j2735_psm,/message/v2x_controller/transition_event,/parameter_events,/robot_description,/rosout,/system_alert,/tf,/tf_static,/ui/client_count,/ui/connected_clients,/ui/ui_instructions,/guidance/twist_filter_node/transition_event,/guidance/approaching_emergency_vehicle_plugin/transition_event,/guidance/plugins/cooperative_lanechange/transition_event,/guidance/twist_gate_node/transition_event,/hardware_interface/driver_shutdown_494b616fe26443fca421ea1f2e04d1d4/transition_event,/hardware_interface/driver_shutdown_77e0426440b045b580837068ea149ea1/transition_event,/hardware_interface/driver_shutdown_9219ae46d4084143a883557f6b60bfc3/transition_event,/hardware_interface/driver_shutdown_494b616fe26443fca421ea1f2e04d1d4/transition_event,/hardware_interface/driver_shutdown_77e0426440b045b580837068ea149ea1/transition_event,/hardware_interface/driver_shutdown_9219ae46d4084143a883557f6b60bfc3/transition_event,/hardware_interface/lightbar_manager/transition_event,/hardware_interface/lightbar_manager_container/transition_event,/guidance/plugins/approaching_emergency_vehicle_plugin/transition_event,/environment/detected_objects,/environment/external_object_predictions,/environment/external_objects,/environment/intersection_signal_group_ids,/environment/leftover_clusters,/environment/map_filtered_points,/environment/roadway_objects,/environment/roadway_obstacles,/environment/tcm_visualizer,/environment/tcr_bounding_points,/guidance/final_maneuver_plan,/guidance/plan_trajectory,/guidance/plugins/cooperative_lanechange/transition_event,/guidance/plugins/debug/trajectory_planning,/guidance/plugins/platoon_control/plan_trajectory,/guidance/plugins/pure_pursuit/plan_trajectory,/guidance/route,/guidance/upcoming_lane_change_status,/hardware_interface/accel_aux_rpt,/hardware_interface/all_system_statuses,/hardware_interface/as/brake_command_echo,/hardware_interface/as/brake_feedback,/hardware_interface/as/curvature_feedback,/hardware_interface/as/gear_command_echo,/hardware_interface/as/gear_feedback,/hardware_interface/as/gear_select,/hardware_interface/as/pacmod/as_rx/shift_cmd,/hardware_interface/as/pacmod/parsed_tx/global_rpt,/hardware_interface/as/pacmod/parsed_tx/shift_rpt,/hardware_interface/as/pacmod/parsed_tx/wheel_speed_rpt,/hardware_interface/as/speed_pedals,/hardware_interface/as/steering_command_echo,/hardware_interface/as/steering_feedback,/hardware_interface/as/steering_wheel,/hardware_interface/as/throttle_command_echo,/hardware_interface/as/throttle_feedback,/hardware_interface/as/turn_signal_command,/hardware_interface/as/vehicle_platform,/hardware_interface/as/velocity_accel_cov,/hardware_interface/brake_aux_rpt,/hardware_interface/can/brake_position,/hardware_interface/can/steering_wheel_angle,/hardware_interface/can/transmission_state,/hardware_interface/can_rx,/hardware_interface/can_tx,/hardware_interface/hazard_lights_cmd,/hardware_interface/hazard_lights_rpt,/hardware_interface/headlight_aux_rpt,/hardware_interface/headlight_cmd,/hardware_interface/headlight_rpt,/hardware_interface/horn_cmd,/hardware_interface/horn_rpt,/hardware_interface/lat_lon_heading_rpt,/hardware_interface/pacmod/parsed_tx/brake_rpt,/hardware_interface/pacmod/parsed_tx/shift_rpt,/hardware_interface/pacmod/parsed_tx/steer_rpt,/hardware_interface/shift_aux_rpt,/hardware_interface/state,/hardware_interface/steering_aux_rpt,/hardware_interface/turn_aux_rpt,/hardware_interface/turn_rpt,/hardware_interface/vehicle/engage,/hardware_interface/velocity_accel_cov,/hardware_interface/vin_rpt,/localization/current_pose_with_covariance,/localization/gnss_pose,/localization/initialpose,/localization/localization_status,/localization/managed_initialpose,/localization/map_param_loader/georeference,/localization/ndt_pose,/localization/ndt_stat,/localization/selected_pose,/robot_description,/message/j2735_convertor_node/transition_event,/hardware_interface/driver_shutdown_81dc0955537246e38b742b0cc74a1a8d/transition_event,/hardware_interface/driver_shutdown_8d5d9db8edc243aca0016a9d9e60e918/transition_event,/hardware_interface/driver_shutdown_98a97ce77cda4cfbb7e8912c56276e62/transition_event,/hardware_interface/driver_shutdown_c19bda6e09c2471bbb762c8d2f0af03d/transition_event,/guidance/twist_filter_node/limitation_debug/ctrl/lateral_accel,/guidance/twist_filter_node/limitation_debug/ctrl/lateral_jerk,/guidance/twist_filter_node/limitation_debug/twist/lateral_accel,/guidance/twist_filter_node/limitation_debug/twist/lateral_jerk,/hardware_interface/as/pacmod/as_rx/enable,/hardware_interface/as/dbw_enabled_feedback,/hardware_interface/as/module_states CLOUD_BRIDGE_EXCLUSION_LIST= + +## Web server +# Server port +PORT=9010 +#Allowed client url +ALLOW_CLIENT_URL=http://local.cav-telematics.com:8888 + +# DB credentials +DB_HOST=localhost +DB_USER=root +DB_PASSWORD=root +GRAFANA_DB=wfd_grafana +DB_PORT=3307 + +# Security +GRAFANA_HTPASSWORD=/opt/apache2/grafana_htpasswd +SESSION_TIMEOUT=3600 +SECRET=my_secret_telematic_ui + +# File upload settings +### Destination: HOST or S3 +### If S3, files are uploaded to the s3 bucket. Otherwise, files are uploaded to HOST machine with destination path +UPLOAD_DESTINATION=host +UPLOAD_DESTINATION_PATH=/opt/telematics/upload + +# S3 bucket credentials +AWS_ACCESS_KEY_ID= +AWS_SECRET_KEY= +S3_BUCKET= +S3_REGION=us-east-2 + +# File upload service +UPLOAD_HTTP_PORT = 9011 +UPLOAD_TIME_OUT=3600000 # Milliseconds +UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB +CONCURRENT_QUEUE_SIZE=5 # How many parts can be parallel processed +PART_SIZE = 10485760 # The size of each part during a multipart upload, in bytes, at least 10MB + +# NATS config +NATS_SERVERS = localhost:4222 +### This topic is used by processing service. Make sure this is consistent with processing service topic setup +FILE_PROCESSING_SUBJECT=ui.file.procressing + diff --git a/telematic_system/telematic_apps/web_app/server/Dockerfile b/telematic_system/telematic_apps/web_app/server/Dockerfile index 7dc0cdd7..133b2dc0 100755 --- a/telematic_system/telematic_apps/web_app/server/Dockerfile +++ b/telematic_system/telematic_apps/web_app/server/Dockerfile @@ -12,5 +12,6 @@ RUN npm install # Bundle app source COPY . . +RUN chmod +x service.sh -ENTRYPOINT ["npm", "start"] +CMD ./service.sh diff --git a/telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js b/telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js index 0bfde1b8..b2621d9a 100644 --- a/telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js +++ b/telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js @@ -14,7 +14,6 @@ * the License. */ const { file_info } = require("../models"); -const ADMIN_ID = 1; /** * List file info @@ -42,7 +41,7 @@ exports.list = (filterFields) => { return data; }) .catch((err) => { - console.log("Error findAll file info from MYSQL DB!"); + console.error("Error findAll file info from MYSQL DB!"); throw err; }); }; diff --git a/telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js b/telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js index 7fb523a3..0459d50c 100644 --- a/telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js +++ b/telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js @@ -22,7 +22,7 @@ const filterFiles = async (req_fields) => { try { return await fileInfoController.list(req_fields); } catch (err) { - console.log("Cannot filter a list of DB files!"); + console.error("Cannot filter a list of DB files!"); console.trace(); throw err; } @@ -36,7 +36,7 @@ const listAllFiles = async () => { return await listAllDBFiles(); } } catch (err) { - console.log(err); + console.error(err); throw err; } }; @@ -45,7 +45,7 @@ const listAllDBFiles = async () => { try { return await fileInfoController.list({}); } catch (err) { - console.log("Cannot get a list of All DB files!"); + console.error("Cannot get a list of All DB files!"); console.trace(); throw err; } @@ -78,13 +78,12 @@ const listAllDBFilesAndS3Objects = async () => { .upsertFileInfo(newFileFromS3) .catch((error) => console.log(error)); contents.push(newFile); - console.log(newFile); } } } return contents; } catch (err) { - console.log("Cannot get a list of all DB files or S3 objects!"); + console.error("Cannot get a list of all DB files or S3 objects!"); console.trace(); throw err; } diff --git a/telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js b/telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js index b1d92e75..c1ff4ae4 100644 --- a/telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js +++ b/telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js @@ -68,7 +68,7 @@ exports.uploadFile = async (req) => { } }); } catch (error) { - console.log(error); + console.error(error); throw error; } }; //End upload file diff --git a/telematic_system/telematic_apps/web_app/server/service.sh b/telematic_system/telematic_apps/web_app/server/service.sh new file mode 100755 index 00000000..7693a1e1 --- /dev/null +++ b/telematic_system/telematic_apps/web_app/server/service.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Start the main app +npm start & + +# Start the file upload service +npm run file-upload-server & + +# Wait for any process to exit +wait -n + +# Exit with status of process that exited first +exit $? \ No newline at end of file diff --git a/telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js b/telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js index ee5c487b..bfcfc0cf 100644 --- a/telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js +++ b/telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js @@ -22,7 +22,7 @@ describe("POST file upload service", () => { it("/api/upload ", async () => { jest - .spyOn(NATSConnModule, "CreateNatsConn") + .spyOn(NATSConnModule, "createNatsConn") .mockResolvedValueOnce(NATSMock.connect()); let filename = "mock.txt"; let formData = { diff --git a/telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js b/telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js index 6743e9bc..a097a26c 100644 --- a/telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js +++ b/telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js @@ -1,4 +1,4 @@ -const { CreateNatsConn } = require("../../nats_client/nats_connection"); +const { createNatsConn } = require("../../nats_client/nats_connection"); const { pubFileProcessingReq, } = require("../../nats_client/file_processing_nats_publisher"); @@ -7,7 +7,7 @@ require("dotenv").config(); describe("Test NATS publisher", () => { it("Test NATS publish to topic ui.file.procressing", async () => { try { - const natsConn = await CreateNatsConn(); + const natsConn = await createNatsConn(); if (natsConn) { let processingReq = { filepath: "/opt/telematic/test.txt",