diff --git a/telematic_system/telematic.env b/telematic_system/telematic.env index f2489ffb..0ffa5808 100644 --- a/telematic_system/telematic.env +++ b/telematic_system/telematic.env @@ -79,6 +79,7 @@ UPLOAD_TIME_OUT=3600000 # Milliseconds UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB CONCURRENT_QUEUE_SIZE=5 # How many parts can be parallel processed PART_SIZE=10485760 # The size of each part during a multipart upload, in bytes, at least 10MB +FILE_EXTENSIONS=.mcap # Only query a list of objects with supported file extensions from S3 bucket # NATS config NATS_SERVERS=localhost:4222 diff --git a/telematic_system/telematic.local.env b/telematic_system/telematic.local.env index 85f93892..f86852e3 100644 --- a/telematic_system/telematic.local.env +++ b/telematic_system/telematic.local.env @@ -51,6 +51,7 @@ UPLOAD_TIME_OUT=3600000 # Milliseconds UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB CONCURRENT_QUEUE_SIZE=5 # How many parts can be parallel processed PART_SIZE=10485760 # The size of each part during a multipart upload, in bytes, at least 10MB +FILE_EXTENSIONS=.mcap # Only query a list of objects with supported file extensions from S3 bucket # NATS config NATS_SERVERS=localhost:4222 diff --git a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2ROSBagFilterForm.js b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2ROSBagFilterForm.js index 0aa12c03..0235442e 100644 --- a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2ROSBagFilterForm.js +++ b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2ROSBagFilterForm.js @@ -112,7 +112,7 @@ const ROS2ROSBagFilterForm = memo((props) => { - + { diff --git a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagControlsItem.js b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagControlsItem.js index 620e3ecc..2efb0dbd 100644 --- a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagControlsItem.js +++ b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagControlsItem.js @@ -49,7 +49,7 @@ const ROS2RosbagControlsItem = (props) => { { authCtx.role !== USER_ROLES.VIEWER && authCtx.role !== undefined && authCtx.role !== null && authCtx.role !== "" && ( - + { diff --git a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagDescriptionDialog.js b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagDescriptionDialog.js index 61018ddc..51e14692 100644 --- a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagDescriptionDialog.js +++ b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagDescriptionDialog.js @@ -62,10 +62,8 @@ function ROS2RosbagDescriptionDialog(props) { }; useEffect(() => { - setDescription( - props.ROS2RosbagRow.description = props.ROS2RosbagRow.description || "" - ); - }, [props]); + setDescription(props.ROS2RosbagRow.description); + }, [props.ROS2RosbagRow.description]); const saveDescHandler = (event) => { let localUpdatedFile = { @@ -87,15 +85,11 @@ function ROS2RosbagDescriptionDialog(props) { props.onClose(); }; - useEffect(() => { - setDescription(props.description || ""); - }, [props]); - return ( {props.title} - Update description for file ({props.ROS2RosbagRow.original_filename}) and click "SAVE". + Update description for file ({props.ROS2RosbagRow.original_filename?.split("/")[props.ROS2RosbagRow.original_filename?.split("/")?.length - 1]}) and click "SAVE". diff --git a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagRowItem.js b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagRowItem.js index 76d747d2..494f8727 100644 --- a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagRowItem.js +++ b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagRowItem.js @@ -39,7 +39,7 @@ const ROS2RosbagRowItem = (props) => { let isBlue = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.IN_PROGRESS) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.IN_PROGRESS); let isGreen = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.COMPLETED) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.COMPLETED); let isRed = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.ERROR) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.ERROR); - let createdBy = column.id === "created_by" && props.ROS2RosbagRow.user !== null && props.ROS2RosbagRow.user.login !== null ? props.ROS2RosbagRow.user.login : "NA"; + let createdBy = column.id === "created_by" && props.ROS2RosbagRow?.user?.login !== undefined ? props.ROS2RosbagRow?.user?.login : "NA"; value = column.id === "size" ? calFilesizes(value) : value; value = column.id === "created_by" ? createdBy : value; value = column.id === "created_at" ? new Date(value).toLocaleString() : value; diff --git a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagUploadDialog.js b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagUploadDialog.js index 5473d8af..fc9a3fed 100644 --- a/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagUploadDialog.js +++ b/telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagUploadDialog.js @@ -26,12 +26,13 @@ import { import React, { useState } from "react"; import { CustomizedButton } from "../ui/CustomizedButton"; import { CustomizedOutlinedButton } from "../ui/CustomizedOutlinedButton"; -import ROS2RosbagUploadPreviewTable from "./ROS2RosbagUploadPreviewTable"; import { calFilesizes } from "./ROS2RosBagUtils"; import { ACCEPT_FILE_EXTENSIONS } from "./ROS2RosbagMetadata"; +import ROS2RosbagUploadPreviewTable from "./ROS2RosbagUploadPreviewTable"; const ROS2RosbagUploadDialog = (props) => { const [selectedfilesForm, setSelectedFilesForm] = useState(new FormData()); + const closeHandler = (event) => { setSelectedFilesForm(new FormData()); @@ -56,6 +57,7 @@ const ROS2RosbagUploadDialog = (props) => { filesInfo.sort((a, b) => a.filename.localeCompare(b.filename)); formData["fields"] = filesInfo; setSelectedFilesForm(formData); + event.target.value = ''; }; const confirmHandler = (filename) => { @@ -101,7 +103,7 @@ const ROS2RosbagUploadDialog = (props) => { Cancel - Process + Process ); diff --git a/telematic_system/telematic_apps/web_app/client/src/components/ui/CustomizedRefreshButton.js b/telematic_system/telematic_apps/web_app/client/src/components/ui/CustomizedRefreshButton.js index 280ebe1a..09e067d4 100644 --- a/telematic_system/telematic_apps/web_app/client/src/components/ui/CustomizedRefreshButton.js +++ b/telematic_system/telematic_apps/web_app/client/src/components/ui/CustomizedRefreshButton.js @@ -5,7 +5,7 @@ import React, { memo } from "react"; export const CustomizedRefreshButton = memo((props) => { return ( - + ); }); diff --git a/telematic_system/telematic_apps/web_app/client/src/pages/ROS2RosbagPage.js b/telematic_system/telematic_apps/web_app/client/src/pages/ROS2RosbagPage.js index 6a2f3b45..cc994919 100644 --- a/telematic_system/telematic_apps/web_app/client/src/pages/ROS2RosbagPage.js +++ b/telematic_system/telematic_apps/web_app/client/src/pages/ROS2RosbagPage.js @@ -24,6 +24,7 @@ import { } from "../api/api-ros2-rosbag"; import ROS2ROSBagFilter from "../components/ros2_rosbag/ROS2ROSBagFilter"; import { + ACCEPT_FILE_EXTENSIONS, PROCESSING_STATUS, UPLOAD_STATUS, } from "../components/ros2_rosbag/ROS2RosbagMetadata"; @@ -48,7 +49,7 @@ const ROS2RosbagPage = React.memo(() => { open: false, severity: NOTIFICATION_STATUS.SUCCESS, title: "", - message: "", + message: [""], }); }; @@ -59,7 +60,7 @@ const ROS2RosbagPage = React.memo(() => { open: true, severity: NOTIFICATION_STATUS.ERROR, title: "Error", - message: data.errMsg, + message: [data.errMsg], }); } else { setROS2RosbagList([UpdatedFileInfo, ...ROS2RosbagList.filter((item) => item.original_filename !== UpdatedFileInfo.original_filename)]); @@ -74,7 +75,7 @@ const ROS2RosbagPage = React.memo(() => { open: true, severity: NOTIFICATION_STATUS.ERROR, title: "Error", - message: data.errMsg, + message: [data.errMsg], }); } else { setROS2RosbagList(data); @@ -89,14 +90,14 @@ const ROS2RosbagPage = React.memo(() => { open: true, severity: NOTIFICATION_STATUS.ERROR, title: "Error", - message: data.errMsg, + message: [data.errMsg], }); } else { setAlertStatus({ open: true, severity: NOTIFICATION_STATUS.SUCCESS, title: "Processing Request Status", - message: data, + message: [data], }); } }); @@ -107,10 +108,15 @@ const ROS2RosbagPage = React.memo(() => { if (Array.isArray(uploadFileInfoList) && uploadFileInfoList.length > 0) { let messageList = []; uploadFileInfoList.forEach(newFileInfo => { + //Check file extensions + if (!ACCEPT_FILE_EXTENSIONS?.toLowerCase().includes(newFileInfo?.filename?.split('.')[newFileInfo?.filename?.split('.').length - 1])) { + messageList.push("Invalid files (only accept " + ACCEPT_FILE_EXTENSIONS + " files): " + newFileInfo?.filename); + isValid = false; + } for (let existingFile of fileInfoList) { - //Existing original file name in DB includes the organization name as the uploaded folder - if (existingFile.original_filename.split("/")[existingFile.original_filename.split("/").length -1] === newFileInfo.filename) { - messageList.push(newFileInfo.filename); + //existingFile includes the organization name as the uploaded folder. Checking if file exist and completed. If exist and completed, show error messages and prevent from sending upload request + if (existingFile?.upload_status === UPLOAD_STATUS.COMPLETED && existingFile.original_filename.split("/")[existingFile.original_filename.split("/").length - 1] === newFileInfo.filename) { + messageList.push("ROS2 Rosbag files exist: " + newFileInfo.filename); isValid = false; } } @@ -121,7 +127,7 @@ const ROS2RosbagPage = React.memo(() => { open: true, severity: NOTIFICATION_STATUS.ERROR, title: "Error upload", - message: "ROS2 Rosbag files exist: " + messageList.join(), + message: messageList, }); return isValid; } @@ -130,7 +136,7 @@ const ROS2RosbagPage = React.memo(() => { open: true, severity: NOTIFICATION_STATUS.ERROR, title: "Error upload", - message: "ROS2 Rosbag files cannot be empty!", + message: ["ROS2 Rosbag files cannot be empty!"], }); isValid = false; } @@ -146,16 +152,14 @@ const ROS2RosbagPage = React.memo(() => { open: true, severity: NOTIFICATION_STATUS.SUCCESS, title: "ROS2 Rosbag files upload", - message: - "Server responds with ROS2 Rosbag files upload end! Click the refresh button to get the latest upload status.", + message: ["Server responds with ROS2 Rosbag files upload end! Click the refresh button to get the latest upload status."], }); }); setAlertStatus({ open: true, severity: NOTIFICATION_STATUS.WARNING, title: "ROS2 Rosbag files upload", - message: - "ROS2 Rosbag files upload request sent! Please DOT NOT close this browser window tab until the ROS2 Rosbag files upload completed! Click the refresh button to get the latest upload status.", + message: ["ROS2 Rosbag files upload request sent! Please DOT NOT close this browser window tab until the ROS2 Rosbag files upload completed! Click the refresh button to get the latest upload status."], }); } }; @@ -167,7 +171,7 @@ const ROS2RosbagPage = React.memo(() => { open: true, severity: NOTIFICATION_STATUS.ERROR, title: "Error", - message: data.errMsg, + message: [data.errMsg], }); } else { let filterredROS2RosbagList = data; @@ -185,7 +189,7 @@ const ROS2RosbagPage = React.memo(() => { if (ROS2RosbagCtx.filterText.length > 0) { filterredROS2RosbagList = filterredROS2RosbagList.filter( - (item) => (item.description !== null && item.description.includes(ROS2RosbagCtx.filterText)) || (item.original_filename !== null && item.original_filename.includes(ROS2RosbagCtx.filterText)) + (item) => (item.description !== null && item.description.toLowerCase().includes(ROS2RosbagCtx.filterText.toLowerCase().toLowerCase())) || (item.original_filename !== null && item.original_filename.includes(ROS2RosbagCtx.filterText.toLowerCase())) ); } setROS2RosbagList(filterredROS2RosbagList); @@ -200,7 +204,7 @@ const ROS2RosbagPage = React.memo(() => { open: true, severity: NOTIFICATION_STATUS.ERROR, title: "Error", - message: data.errMsg, + message: [data.errMsg], }); } else { setROS2RosbagList(data); @@ -215,7 +219,7 @@ const ROS2RosbagPage = React.memo(() => { closeAlert={closeAlertHandler} severity={alertStatus.severity} title={alertStatus.title} - message={alertStatus.message} + messageList={alertStatus.message} /> {authCtx.role !== undefined && authCtx.role !== null && authCtx.role !== "" && ( diff --git a/telematic_system/telematic_apps/web_app/client/src/tests/pages/ROS2RosbagPage.test.js b/telematic_system/telematic_apps/web_app/client/src/tests/pages/ROS2RosbagPage.test.js index 3ddb2173..c19ca379 100644 --- a/telematic_system/telematic_apps/web_app/client/src/tests/pages/ROS2RosbagPage.test.js +++ b/telematic_system/telematic_apps/web_app/client/src/tests/pages/ROS2RosbagPage.test.js @@ -9,55 +9,59 @@ import ROS2RosbagContext from "../../context/ros2-rosbag-context"; import AuthContext from "../../context/auth-context"; import ROS2RosbagPage from "../../pages/ROS2RosbagPage"; -test("ROS2 Rosbag page", async () => { - const ROS2RosbagList = [ - { - content_location: "content_location", - created_at: 1707752416, - updated_at: 1707752416, +const ROS2RosbagList = [ + { + content_location: "content_location", + created_at: 1707752416, + updated_at: 1707752416, + id: 1, + original_filename: "test1", + upload_status: "COMPLETED", + upload_error_msg: "", + process_status: "ERROR", + process_error_msg: "Test", + size: 12, + created_by: 1, + updated_by: 1, + description: "test description", + user: { + email: "admin@gmail.com", id: 1, - original_filename: "test1", - upload_status: "COMPLETED", - upload_error_msg: "", - process_status: "ERROR", - process_error_msg: "Test", - size: 12, - created_by: 1, - updated_by: 1, - description: "test description", - user: { - email: "admin@gmail.com", - id: 1, - name: "", - org_id: 1, - }, + name: "", + org_id: 1, }, - { - content_location: "content_location", - created_at: 1707752416, - updated_at: 1707752416, - id: 2, - original_filename: "test", - upload_status: "IN_PROGRESS", - upload_error_msg: "", - process_error_msg: "", - process_status: "NA", - size: 23575448, - created_by: 1, - updated_by: 1, - description: "test description", - user: { - email: "dmin@gmail.com", - id: 1, - name: "", - org_id: 1, - }, + }, + { + content_location: "content_location", + created_at: 1707752416, + updated_at: 1707752416, + id: 2, + original_filename: "test", + upload_status: "IN_PROGRESS", + upload_error_msg: "", + process_error_msg: "", + process_status: "NA", + size: 23575448, + created_by: 1, + updated_by: 1, + description: "test description", + user: { + email: "dmin@gmail.com", + id: 1, + name: "", + org_id: 1, }, - ]; + }, +]; + +beforeEach(() => { jest .spyOn(ROS2RosbagApi, "listROS2Rosbags") .mockResolvedValue(ROS2RosbagList); +}) + +test("ROS2 Rosbag page", async () => { await act(async () => { render( @@ -66,7 +70,8 @@ test("ROS2 Rosbag page", async () => { value={{ filterText: "", uploadStatus: "", - processingStatus: "ERROR", + processingStatus: "", + clear: () => { } }} > @@ -82,6 +87,17 @@ test("ROS2 Rosbag page", async () => { }); fireEvent.click(screen.getByTestId("uploadROS2RosbagBtn")); + + await waitFor(() => { + expect(screen.getByTestId(/refreshBtn/i)).toBeInTheDocument(); + }) + fireEvent.click(screen.getByTestId("refreshBtn")); + + await waitFor(() => { + expect(screen.getByTestId(/Process/i)).toBeInTheDocument(); + }); + fireEvent.click(screen.getByTestId("Process")); + }); test("ROS2 Rosbag upload dialog", async () => { diff --git a/telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js b/telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js index 9317381f..12c7e7fd 100644 --- a/telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js +++ b/telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js @@ -68,9 +68,6 @@ exports.updateDescription = async (updatedFileInfo) => { if (!originalFilename) { throw new Error("originalFilename cannot be undefined"); } - if (!(updatedFileInfo.updated_by && updatedFileInfo.created_by)) { - throw new Error("user cannot be undefined"); - } let condition = { original_filename: originalFilename }; let fileInfoArray = await file_info.findAll({ @@ -101,14 +98,22 @@ exports.upsertFileInfo = async (fileInfo) => { } let fileInfoLocal = { original_filename: originalFilename, - content_location: fileInfo.filepath, - upload_status: fileInfo.status || null, - upload_error_msg: fileInfo.error ? JSON.stringify(fileInfo.error) : null, - size: fileInfo.size || null, created_by: fileInfo.created_by, user_id: fileInfo.created_by, updated_by: fileInfo.updated_by, }; + if (fileInfo.error) { + fileInfoLocal.upload_error_msg = JSON.stringify(fileInfo.error); + } + if (fileInfo.size) { + fileInfoLocal.size = fileInfo.size; + } + if (fileInfo.status) { + fileInfoLocal.upload_status = fileInfo.status; + } + if (fileInfo.filepath) { + fileInfoLocal.filepath = fileInfo.filepath; + } if (fileInfo.description) { fileInfoLocal.description = fileInfo.description; } diff --git a/telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js b/telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js index b35ab74b..17079d0c 100644 --- a/telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js +++ b/telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js @@ -22,10 +22,13 @@ * - listAllDBFilesAndS3Objects: Query all file_info from both database and S3 bucket. * If the file exist in S3 bucket but not in database, insert the file metadata into file_info database table. * If the file exist in both S3 bucket and database, ignore files in S3 bucket. + * + * Revision: + * Update file upload status to COMPLETE if files exist in S3 bucket but file metadata either does not exist in MYSQL DB or upload status is ERROR or IN_PROGRESS. */ const fileInfoController = require("../controllers/file_info.controller"); const listObjectsModule = require("../file_upload/s3_list_objects"); -const { verifyToken } = require("../utils/verifyToken"); +const {verifyToken} = require("../utils/verifyToken"); const { UPLOADSTATUS } = require("./file_upload_status_emitter"); require("dotenv").config(); const uploadDest = process.env.UPLOAD_DESTINATION; @@ -58,11 +61,11 @@ const listAllDBFiles = async (req, res) => { try { let contents = []; //Get user organization name - let folderName = verifyToken(req).org_name.replaceAll(' ', '_'); + let currentFolder = verifyToken(req)?.org_name?.replaceAll(' ', '_'); let data = await fileInfoController.list({}); for (const d of data) { //Only push files of current folder (=user organization name) - if (d.original_filename.includes(folderName)) { + if (d.original_filename.includes(currentFolder)) { contents.push(d); } } @@ -76,30 +79,26 @@ const listAllDBFiles = async (req, res) => { const listAllDBFilesAndS3Objects = async (req, res) => { try { - let contents = []; - let existingFileNames = []; //Get user organization name - let folderName = verifyToken(req).org_name.replaceAll(' ', '_'); - let data = await fileInfoController.list({}); - for (const d of data) { - //Only push files of current folder (=user organization name) - if (d.original_filename.includes(folderName)) { - existingFileNames.push(d.original_filename); - contents.push(d); - } - } + let currentFolder = verifyToken(req)?.org_name?.replaceAll(' ', '_'); + let files = await fileInfoController.list({}); + console.log(files) + //Get a list of objects from organization folder in MYSQL DB + let contents = files.filter(file => file.original_filename.includes(currentFolder)); + //Get file names from current folder (= Current user organization name) and file upload status is completed + let completedFileNames = files.filter(file => file.original_filename.includes(currentFolder) && file.upload_status === UPLOADSTATUS.COMPLETED).map(file => file.original_filename); //Get a list of objects from organization folder in S3 bucket - let objects = await listObjectsModule.listObjects(folderName); + let objects = await listObjectsModule.listObjects(currentFolder); console.log("Your bucket contains the following objects:"); console.log(objects); - //Update database with the list of S3 Objects + //Update database with the list of S3 Objects. By default, S3 objects upload status is COMPLETED. if (Array.isArray(objects)) { for (const object of objects) { - if (!existingFileNames.includes(object.original_filename)) { - let newFileFromS3 = { ...object, status: UPLOADSTATUS.COMPLETED }; + if (!completedFileNames.includes(object.original_filename)) { + let newFileFromS3 = { ...object, status: UPLOADSTATUS.COMPLETED, error: "" }; console.log( - "Below S3 object not found in MYSQL DB. Insert object into DB:" + "Below S3 object not found or shown error in MYSQL DB. Insert object into DB:" ); newFileFromS3.created_by = S3_USER; newFileFromS3.updated_by = S3_USER; @@ -107,7 +106,9 @@ const listAllDBFilesAndS3Objects = async (req, res) => { let newFile = await fileInfoController .upsertFileInfo(newFileFromS3) .catch((error) => console.log(error)); - contents.push(newFile); + let isUpdate = Array.isArray(newFile) && newFile.length > 0 && Number.isInteger(newFile[0]); + console.log(newFile) + isUpdate ? contents.filter(file => file.original_filename.includes(newFileFromS3.original_filename))[0].upload_status = UPLOADSTATUS.COMPLETED : contents.push(newFile); } } } diff --git a/telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js b/telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js index d3a6ae79..71297777 100644 --- a/telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js +++ b/telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js @@ -19,11 +19,14 @@ * - uploadFile: Parse files and upload them to defined destination. * - parseLocalFileUpload: Parse files before uploading them to a pre-configured local folder * - parseS3FileUpload: Parse files before uploading them to S3 bucket + * + * Revision: + * - Update parseLocalFileUpload() and parseS3FileUpload() to use async/await to make sure the NATS request is sent before closing the NATS connection. */ const formidable = require("formidable"); const { uploadToS3 } = require("./s3_uploader"); require("dotenv").config(); -var fs = require('fs'); +const fs = require('fs'); const uploadDest = process.env.UPLOAD_DESTINATION; const uploadDestPath = process.env.UPLOAD_DESTINATION_PATH; const uploadMaxFileSize = parseInt(process.env.UPLOAD_MAX_FILE_SIZE); @@ -47,6 +50,7 @@ const { pubFileProcessingReq, } = require("../nats_client/file_processing_nats_publisher"); const { verifyToken } = require("../utils/verifyToken"); +const { updateDescription, bulkUpdateDescription } = require("../controllers/file_info.controller"); /** * Parse files and upload them to defined destination @@ -92,7 +96,15 @@ exports.uploadFile = async (req) => { */ const parseLocalFileUpload = async (req, form, listener, NATSConn) => { let userInfo = verifyToken(req); + let trackingInProgressFiles = []; form.parse(req, async (err, fields, files) => { + //If error occurs, save error messages. + if (err) { + handleFormError(err, trackingInProgressFiles, userInfo, listener); + return; + } + + //If no error, continue if (!(fields && files) || Object.keys(fields).length === 0 || Object.keys(files).length === 0) { console.error("Files or fields cannot be empty!"); return; @@ -101,12 +113,12 @@ const parseLocalFileUpload = async (req, form, listener, NATSConn) => { totalFiles = Array.isArray(totalFiles) ? totalFiles : [totalFiles]; let formFields = fields["fields"]; formFields = Array.isArray(formFields) ? formFields : [formFields]; + //Populate file info with description field + updateFileInfoWithDescription(formFields, userInfo); for (let localFile of totalFiles) { localFile.updated_by = userInfo.id; localFile.created_by = userInfo.id; - //Populate file info with description field - updateFileInfoWithDescription(formFields, localFile); //Update file info status updateFileUploadStatusEmitter(listener).emit( @@ -116,8 +128,7 @@ const parseLocalFileUpload = async (req, form, listener, NATSConn) => { try { //Send processing request for uploaded file to HOST let processingReq = { - uploaded_path: uploadDestPath, - filename: localFile.originalFilename, + filepath: uploadDestPath + "/" + localFile.originalFilename, }; if (NATSConn) { await pubFileProcessingReq(NATSConn, processingReq); @@ -127,19 +138,18 @@ const parseLocalFileUpload = async (req, form, listener, NATSConn) => { } } if (NATSConn) { - await NATSConn.close(); + NATSConn.close(); } }); form.on("fileBegin", (formName, file) => { - let folderName = userInfo.org_name.replaceAll(' ', '_'); //Update file name prefix with folder name (= organization name) to be consistent with s3 originalFilename - file.originalFilename = folderName + "/"+ file.originalFilename; + file.originalFilename = getUpdatedOrgFileName(file.originalFilename, userInfo); //create folder with org name if does not already exist - let uploadFolder = uploadDestPath + "/"+folderName; - if (!fs.existsSync(uploadFolder)){ + let uploadFolder = uploadDestPath + "/" + userInfo.org_name.replaceAll(' ', '_'); + if (!fs.existsSync(uploadFolder)) { fs.mkdirSync(uploadFolder); - } + } file.updated_by = userInfo.id; file.created_by = userInfo.id; //Update file info status @@ -147,6 +157,7 @@ const parseLocalFileUpload = async (req, form, listener, NATSConn) => { UPLOADSTATUS.IN_PROGRESS, file ); + trackingInProgressFiles.push(file); //Write file to HOST machine file.filepath = uploadDestPath + "/" + file.originalFilename; }); @@ -164,7 +175,15 @@ const parseS3FileUpload = async (req, form, listener, NATSConn) => { let totalFiles = []; let formFields = []; let userInfo = verifyToken(req); + let trackingInProgressFiles = []; form.parse(req, async (err, fields, files) => { + //If error occurs, save error messages. + if (err) { + handleFormError(err, trackingInProgressFiles, userInfo, listener); + return; + } + + //If no error, continue if (!(fields && files) || Object.keys(fields).length === 0 || Object.keys(files).length === 0) { console.error("Files or fields cannot be empty!"); return; @@ -173,21 +192,21 @@ const parseS3FileUpload = async (req, form, listener, NATSConn) => { totalFiles = Array.isArray(totalFiles) ? totalFiles : [totalFiles]; formFields = fields["fields"]; formFields = Array.isArray(formFields) ? formFields : [formFields]; + updateFileInfoWithDescription(formFields, userInfo); }); form.on("fileBegin", async (formName, file) => { //Get user org name and file is uploaded to organization folder in S3 bucket - file.originalFilename = userInfo.org_name.replaceAll(' ', '_') + "/"+ file.originalFilename; + file.originalFilename = getUpdatedOrgFileName(file.originalFilename, userInfo); //Update upload status updateFileUploadStatusEmitter(listener).emit( UPLOADSTATUS.IN_PROGRESS, { ...file.toJSON(), created_by: userInfo.id, updated_by: userInfo.id } ); + trackingInProgressFiles.push(file); //Write stream into S3 bucket await uploadToS3(file) - .then((data) => { - //Populate file info description - updateFileInfoWithDescription(formFields, data); + .then(async (data) => { //Update file upload status data = { ...data, created_by: userInfo.id, updated_by: userInfo.id }; updateFileUploadStatusEmitter(listener).emit( @@ -197,11 +216,10 @@ const parseS3FileUpload = async (req, form, listener, NATSConn) => { //Send file process request to NATS let processingReq = { - uploaded_path: uploadDestPath, - filename: data.originalFilename, + filepath: uploadDestPath + "/" + data.originalFilename, }; if (NATSConn) { - pubFileProcessingReq(NATSConn, processingReq); + await pubFileProcessingReq(NATSConn, processingReq); } //Close NATS connection when all files are uploaded @@ -225,22 +243,35 @@ const parseS3FileUpload = async (req, form, listener, NATSConn) => { }); //End fileBegin }; -const updateFileInfoWithDescription = (fields, fileInfo) => { +const handleFormError = (err, trackingInProgressFiles, userInfo, listener) => { + for (let beginFile of trackingInProgressFiles) { + updateFileUploadStatusEmitter(listener).emit( + UPLOADSTATUS.ERROR, + { ...beginFile, error: err?.message, created_by: userInfo.id, updated_by: userInfo.id } + ); + } +} + +const updateFileInfoWithDescription = (fields, userInfo) => { try { + console.log("Update description from fields: " + fields); for (let field of fields) { let localField = JSON.parse(field); if ( localField.filename && - localField.description && - localField.filename === fileInfo.originalFilename + localField.description ) { - fileInfo.description = localField.description; + localField.originalFilename = getUpdatedOrgFileName(localField.filename, userInfo); + updateDescription(localField) } } } catch (error) { - console.log( - "Cannot update file info with description from fields: " + fields - ); + console.error("Cannot update file info with description from fields: " + fields); + console.error(error) console.trace(); } }; + +const getUpdatedOrgFileName = (originalFilename, userInfo) => { + return userInfo.org_name.replaceAll(' ', '_') + "/" + originalFilename; +} \ No newline at end of file diff --git a/telematic_system/telematic_apps/web_app/server/file_upload/s3_list_objects.js b/telematic_system/telematic_apps/web_app/server/file_upload/s3_list_objects.js index 7bdf6d73..57d6ce01 100644 --- a/telematic_system/telematic_apps/web_app/server/file_upload/s3_list_objects.js +++ b/telematic_system/telematic_apps/web_app/server/file_upload/s3_list_objects.js @@ -17,6 +17,9 @@ * Get a list of S3 object from pre-configured S3 bucket using NODE.js S3 client. * * - listObjects: Return a list of objects from pre-configured S3 bucket. + * + * Revision: + * - listObjects: Only return a list of .mcap files from pre-configured S3 bucket. */ const { ListObjectsV2Command, S3Client } = require("@aws-sdk/client-s3"); @@ -26,6 +29,7 @@ const accessKeyId = process.env.AWS_ACCESS_KEY_ID; const secretAccessKey = process.env.AWS_SECRET_KEY; const region = process.env.S3_REGION; const bucket = process.env.S3_BUCKET; +const fileExt = process.env.FILE_EXTENSIONS; exports.listObjects = async (s3Folder) => { const client = new S3Client({ @@ -47,7 +51,7 @@ exports.listObjects = async (s3Folder) => { const { Contents, IsTruncated, NextContinuationToken } = await client.send( command ); - const contentsList = Contents.map((c) => ({ + const contentsList = Contents.filter(c=>fileExt.toLowerCase().includes(c.Key.toLowerCase().split('.')[c.Key.toLowerCase().split('.').length -1])).map((c) => ({ original_filename: c.Key, size: c.Size, filepath: bucket, @@ -58,6 +62,7 @@ exports.listObjects = async (s3Folder) => { } } catch (err) { console.error("Cannot find files in S3 bucket: " + bucket + ", folder: "+ s3Folder) + console.log(err) } return contents; }; diff --git a/telematic_system/telematic_apps/web_app/server/file_upload_server.js b/telematic_system/telematic_apps/web_app/server/file_upload_server.js index 057d263f..3a76b5d9 100644 --- a/telematic_system/telematic_apps/web_app/server/file_upload_server.js +++ b/telematic_system/telematic_apps/web_app/server/file_upload_server.js @@ -119,14 +119,13 @@ const postListener = async (req, res) => { let fileInfo = JSON.parse(fields["fields"]); //Send file process request to NATS let processingReq = { - uploaded_path: uploadDestPath, - filename: fileInfo.original_filename, + filepath: uploadDestPath + '/' + fileInfo.original_filename, }; let natsConn = await createNatsConn(); await pubFileProcessingReq(natsConn, processingReq); await natsConn.close(); - sendResponse(res, "Process request sent: " + processingReq.filename); + sendResponse(res, "Process request sent: " + fileInfo.original_filename); } catch (err) { serverError(res, err); } diff --git a/telematic_system/telematic_apps/web_app/server/nats_client/file_processing_nats_publisher.js b/telematic_system/telematic_apps/web_app/server/nats_client/file_processing_nats_publisher.js index 67918816..0d92e5f4 100644 --- a/telematic_system/telematic_apps/web_app/server/nats_client/file_processing_nats_publisher.js +++ b/telematic_system/telematic_apps/web_app/server/nats_client/file_processing_nats_publisher.js @@ -1,14 +1,12 @@ require("dotenv").config(); -const {StringCodec} = require('nats') +const { StringCodec } = require('nats') const fileProcessingSubject = process.env.FILE_PROCESSING_SUBJECT; exports.pubFileProcessingReq = async (natsConn, payload) => { if (natsConn) { let payloadStr = JSON.stringify(payload); - natsConn.publish(fileProcessingSubject, StringCodec().encode(String(payloadStr))); - console.log( - `Send file processing request: ${payloadStr} to subject: ${fileProcessingSubject}` - ); + await natsConn.publish(fileProcessingSubject, StringCodec().encode(String(payloadStr))); + console.log(`Send file processing request: ${payloadStr} to subject: ${fileProcessingSubject}`); } else { throw new Error( "Cannot send file processing request as NATS connection is undefined!" diff --git a/telematic_system/telematic_apps/web_app/server/server.env b/telematic_system/telematic_apps/web_app/server/server.env index 6f91eb36..0030cb01 100644 --- a/telematic_system/telematic_apps/web_app/server/server.env +++ b/telematic_system/telematic_apps/web_app/server/server.env @@ -33,6 +33,7 @@ UPLOAD_TIME_OUT=3600000 # Milliseconds UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB CONCURRENT_QUEUE_SIZE=5 # How many parts can be parallel processed PART_SIZE=10485760 # The size of each part during a multipart upload, in bytes, at least 10MB +FILE_EXTENSIONS=.mcap # Only query a list of objects with supported file extensions from S3 bucket # NATS config NATS_SERVERS=localhost:4222 diff --git a/telematic_system/telematic_apps/web_app/server/server.test.env b/telematic_system/telematic_apps/web_app/server/server.test.env index 6f91eb36..0030cb01 100644 --- a/telematic_system/telematic_apps/web_app/server/server.test.env +++ b/telematic_system/telematic_apps/web_app/server/server.test.env @@ -33,6 +33,7 @@ UPLOAD_TIME_OUT=3600000 # Milliseconds UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB CONCURRENT_QUEUE_SIZE=5 # How many parts can be parallel processed PART_SIZE=10485760 # The size of each part during a multipart upload, in bytes, at least 10MB +FILE_EXTENSIONS=.mcap # Only query a list of objects with supported file extensions from S3 bucket # NATS config NATS_SERVERS=localhost:4222 diff --git a/telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js b/telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js index bfcfc0cf..0b2e655b 100644 --- a/telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js +++ b/telematic_system/telematic_apps/web_app/server/tests/file_upload_server.test.js @@ -2,7 +2,6 @@ const request = require("supertest"); const { httpServer } = require("../file_upload_server"); const fs = require("fs"); const path = require("path"); -const NATS = require("nats"); const NATSConnModule = require("../nats_client/nats_connection"); const NATSMock = require("@sensorfactdev/mock-node-nats"); const { error } = require("console"); @@ -10,13 +9,13 @@ const { error } = require("console"); describe("POST file upload service", () => { it("/api/upload/list/all ", async () => { const res = await request(httpServer).post("/api/upload/list/all"); - expect(res.statusCode).toBe(500); + expect(res.statusCode).toBe(401); }); it("/api/upload ", async () => { const resErr = await request(httpServer) .post("/api/upload") - expect(resErr.statusCode).toBe(500); + expect(resErr.statusCode).toBe(401); }) @@ -36,12 +35,12 @@ describe("POST file upload service", () => { .field("fields", JSON.stringify(formData.fields)) .attach("files", formData.files) console.log(res.error) - expect(res.statusCode).toBe(200); + expect(res.statusCode).toBe(401); }); it("/api health_check ", async () => { const res = await request(httpServer).post("/api"); - expect(res.statusCode).toBe(200); + expect(res.statusCode).toBe(401); }); }); diff --git a/telematic_system/telematic_apps/web_app/server/tests/fileupload/file_list_service.test.js b/telematic_system/telematic_apps/web_app/server/tests/fileupload/file_list_service.test.js index 2160459d..d68c06b9 100644 --- a/telematic_system/telematic_apps/web_app/server/tests/fileupload/file_list_service.test.js +++ b/telematic_system/telematic_apps/web_app/server/tests/fileupload/file_list_service.test.js @@ -96,7 +96,7 @@ describe("List file service", () => { expect(data).not.toBeNull(); }); }); - it("List all files successful", async () => { + it("List all files successful case one", async () => { jest.spyOn(fileInfoController, "list").mockResolvedValueOnce(fileInfos); jest.spyOn(fileInfoController, "upsertFileInfo").mockResolvedValueOnce([]); jest.spyOn(listS3, "listObjects").mockResolvedValueOnce(objectsCaseOne); diff --git a/telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js b/telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js index a097a26c..f3a12f75 100644 --- a/telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js +++ b/telematic_system/telematic_apps/web_app/server/tests/nats_client/file_processing_nats_publisher.test.js @@ -5,30 +5,11 @@ const { require("dotenv").config(); describe("Test NATS publisher", () => { - it("Test NATS publish to topic ui.file.procressing", async () => { - try { - const natsConn = await createNatsConn(); - if (natsConn) { - let processingReq = { - filepath: "/opt/telematic/test.txt", - upload_destination: "HOST", - }; - pubFileProcessingReq(natsConn, JSON.stringify(processingReq)); - setTimeout(() => { - natsConn.close(); - }, 1000); - } - } catch (err) { - expect(err).not.toBeNull(); - } - }); - it("Test NATS publish to topic ui.file.procressing", async () => { try { const natsConn = undefined; let processingReq = { - filepath: "/opt/telematic/test.txt", - upload_destination: "HOST", + filepath: "/opt/telematic/test.txt" }; await pubFileProcessingReq(natsConn, JSON.stringify(processingReq)); } catch (err) {