Historical data page: Query only list .mcap file from S3 bucket and process request issues

telematic_system/telematic.env

@@ -79,6 +79,7 @@ UPLOAD_TIME_OUT=3600000 # Milliseconds
 UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB
 CONCURRENT_QUEUE_SIZE=5 # How many parts can be parallel processed
 PART_SIZE=10485760 # The size of each part during a multipart upload, in bytes, at least 10MB
+FILE_EXTENSIONS=.mcap # Only query a list of objects with supported file extensions from S3 bucket 
 
 # NATS config
 NATS_SERVERS=localhost:4222

telematic_system/telematic.local.env

@@ -51,6 +51,7 @@ UPLOAD_TIME_OUT=3600000 # Milliseconds
 UPLOAD_MAX_FILE_SIZE=21474836480 #20 GB
 CONCURRENT_QUEUE_SIZE=5 # How many parts can be parallel processed
 PART_SIZE=10485760 # The size of each part during a multipart upload, in bytes, at least 10MB
+FILE_EXTENSIONS=.mcap # Only query a list of objects with supported file extensions from S3 bucket 
 
 # NATS config
 NATS_SERVERS=localhost:4222

telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2ROSBagFilterForm.js

@@ -112,7 +112,7 @@ const ROS2ROSBagFilterForm = memo((props) => {
         </FormControl>
 
         <FormControl sx={{ paddingTop: "10px" }}>
-          <CustomizedRefreshButton title="Reset filters and refresh table" onClick={refreshHandler} ></CustomizedRefreshButton>
+          <CustomizedRefreshButton title="Reset filters and refresh table" data-testid="refreshBtn" onClick={refreshHandler} ></CustomizedRefreshButton>
         </FormControl>
       </Grid>
       {

telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagControlsItem.js

@@ -49,7 +49,7 @@ const ROS2RosbagControlsItem = (props) => {
       {
         authCtx.role !== USER_ROLES.VIEWER && authCtx.role !== undefined && authCtx.role !== null && authCtx.role !== "" && (
           <TableCell key={`controls-${props.ROS2RosbagRow.id}`}>
-            <ROS2RosbagDescriptionDialog open={open} onClose={closeHandler} title={`Edit (${props.ROS2RosbagRow.original_filename}) Description`} ROS2RosbagRow={props.ROS2RosbagRow} OnDescriptionSave={saveDescHandler}></ROS2RosbagDescriptionDialog>
+            <ROS2RosbagDescriptionDialog open={open} onClose={closeHandler} title={`Edit Description`} ROS2RosbagRow={props.ROS2RosbagRow} OnDescriptionSave={saveDescHandler}></ROS2RosbagDescriptionDialog>
             <CustomizedOutlinedButton title={"Edit description"} key={`edit-Rosbag-${props.ROS2RosbagRow.id}`} onClick={openHandler}><EditIcon /></CustomizedOutlinedButton>
 
             {

telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagDescriptionDialog.js

@@ -62,10 +62,8 @@ function ROS2RosbagDescriptionDialog(props) {
   };
 
   useEffect(() => {
-    setDescription(
-      props.ROS2RosbagRow.description = props.ROS2RosbagRow.description || ""
-    );
-  }, [props]);
+    setDescription(props.ROS2RosbagRow.description);
+  }, [props.ROS2RosbagRow.description]);
 
   const saveDescHandler = (event) => {
     let localUpdatedFile = {
@@ -87,15 +85,11 @@ function ROS2RosbagDescriptionDialog(props) {
     props.onClose();
   };
 
-  useEffect(() => {
-    setDescription(props.description || "");
-  }, [props]);
-
   return (
     <Dialog open={props.open} onClose={props.onCloseHandler}>
       <DialogTitle sx={{ fontWeight: "bolder" }}>{props.title}</DialogTitle>
       <DialogContent>
-        <DialogContentText>  Update description for file (<b>{props.ROS2RosbagRow.original_filename}</b>)  and click "SAVE". </DialogContentText>
+        <DialogContentText>  Update description for file (<b>{props.ROS2RosbagRow.original_filename?.split("/")[props.ROS2RosbagRow.original_filename?.split("/")?.length - 1]}</b>)  and click "SAVE". </DialogContentText>
         <FormControl fullWidth>
           <TextField {...register("description")} error={errors.description ? true : false} margin="dense" id="description" label="Description*" variant="standard" value={description} onChange={handleDescChange} sx={{ marginBottom: 5 }} />
         </FormControl>

telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagRowItem.js

@@ -39,7 +39,7 @@ const ROS2RosbagRowItem = (props) => {
           let isBlue = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.IN_PROGRESS) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.IN_PROGRESS);
           let isGreen = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.COMPLETED) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.COMPLETED);
           let isRed = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.ERROR) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.ERROR);
-          let createdBy = column.id === "created_by" && props.ROS2RosbagRow.user !== null && props.ROS2RosbagRow.user.login !== null ? props.ROS2RosbagRow.user.login : "NA";
+          let createdBy = column.id === "created_by" && props.ROS2RosbagRow?.user?.login !== undefined ? props.ROS2RosbagRow?.user?.login : "NA";
           value = column.id === "size" ? calFilesizes(value) : value;
           value = column.id === "created_by" ? createdBy : value;
           value = column.id === "created_at" ? new Date(value).toLocaleString() : value;

telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagUploadDialog.js

@@ -26,12 +26,13 @@ import {
 import React, { useState } from "react";
 import { CustomizedButton } from "../ui/CustomizedButton";
 import { CustomizedOutlinedButton } from "../ui/CustomizedOutlinedButton";
-import ROS2RosbagUploadPreviewTable from "./ROS2RosbagUploadPreviewTable";
 import { calFilesizes } from "./ROS2RosBagUtils";
 import { ACCEPT_FILE_EXTENSIONS } from "./ROS2RosbagMetadata";
+import ROS2RosbagUploadPreviewTable from "./ROS2RosbagUploadPreviewTable";
 
 const ROS2RosbagUploadDialog = (props) => {
   const [selectedfilesForm, setSelectedFilesForm] = useState(new FormData());
+
 
   const closeHandler = (event) => {
     setSelectedFilesForm(new FormData());
@@ -56,6 +57,7 @@ const ROS2RosbagUploadDialog = (props) => {
     filesInfo.sort((a, b) => a.filename.localeCompare(b.filename));
     formData["fields"] = filesInfo;
     setSelectedFilesForm(formData);
+    event.target.value = '';
   };
 
   const confirmHandler = (filename) => {
@@ -101,7 +103,7 @@ const ROS2RosbagUploadDialog = (props) => {
       </DialogContent>
       <DialogActions>
         <CustomizedOutlinedButton onClick={closeHandler}> Cancel </CustomizedOutlinedButton>
-        <CustomizedButton onClick={uploadAndProcessHandler}>Process</CustomizedButton>
+        <CustomizedButton onClick={uploadAndProcessHandler} data-testid="Process">Process</CustomizedButton>
       </DialogActions>
     </Dialog>
   );

telematic_system/telematic_apps/web_app/client/src/components/ui/CustomizedRefreshButton.js

@@ -5,7 +5,7 @@ import React, { memo } from "react";
 export const CustomizedRefreshButton = memo((props) => {
   return (
     <Tooltip title={props.title}>
-      <Button variant="outlined" size="large" onClick={props.onClick} sx={{ borderColor: "#748c93", display: 'inline-block' }}><SyncIcon sx={{ color: "#748c93" }} /></Button>
+      <Button variant="outlined" size="large"  data-testid={props["data-testid"]} onClick={props.onClick} sx={{ borderColor: "#748c93", display: 'inline-block' }}><SyncIcon sx={{ color: "#748c93" }} /></Button>
     </Tooltip>
   );
 });

telematic_system/telematic_apps/web_app/client/src/pages/ROS2RosbagPage.js

@@ -24,6 +24,7 @@ import {
 } from "../api/api-ros2-rosbag";
 import ROS2ROSBagFilter from "../components/ros2_rosbag/ROS2ROSBagFilter";
 import {
+  ACCEPT_FILE_EXTENSIONS,
   PROCESSING_STATUS,
   UPLOAD_STATUS,
 } from "../components/ros2_rosbag/ROS2RosbagMetadata";
@@ -48,7 +49,7 @@ const ROS2RosbagPage = React.memo(() => {
       open: false,
       severity: NOTIFICATION_STATUS.SUCCESS,
       title: "",
-      message: "",
+      message: [""],
     });
   };
 
@@ -59,7 +60,7 @@ const ROS2RosbagPage = React.memo(() => {
           open: true,
           severity: NOTIFICATION_STATUS.ERROR,
           title: "Error",
-          message: data.errMsg,
+          message: [data.errMsg],
         });
       } else {
         setROS2RosbagList([UpdatedFileInfo, ...ROS2RosbagList.filter((item) => item.original_filename !== UpdatedFileInfo.original_filename)]);
@@ -74,7 +75,7 @@ const ROS2RosbagPage = React.memo(() => {
           open: true,
           severity: NOTIFICATION_STATUS.ERROR,
           title: "Error",
-          message: data.errMsg,
+          message: [data.errMsg],
         });
       } else {
         setROS2RosbagList(data);
@@ -89,14 +90,14 @@ const ROS2RosbagPage = React.memo(() => {
           open: true,
           severity: NOTIFICATION_STATUS.ERROR,
           title: "Error",
-          message: data.errMsg,
+          message: [data.errMsg],
         });
       } else {
         setAlertStatus({
           open: true,
           severity: NOTIFICATION_STATUS.SUCCESS,
           title: "Processing Request Status",
-          message: data,
+          message: [data],
         });
       }
     });
@@ -107,10 +108,15 @@ const ROS2RosbagPage = React.memo(() => {
     if (Array.isArray(uploadFileInfoList) && uploadFileInfoList.length > 0) {
       let messageList = [];
       uploadFileInfoList.forEach(newFileInfo => {
+        //Check file extensions
+        if (!ACCEPT_FILE_EXTENSIONS?.toLowerCase().includes(newFileInfo?.filename?.split('.')[newFileInfo?.filename?.split('.').length - 1])) {
+          messageList.push("Invalid files (only accept " + ACCEPT_FILE_EXTENSIONS + " files): " + newFileInfo?.filename);
+          isValid = false;
+        }
         for (let existingFile of fileInfoList) {
-          //Existing original file name in DB includes the organization name as the uploaded folder
-          if (existingFile.original_filename.split("/")[existingFile.original_filename.split("/").length -1] === newFileInfo.filename) {
-            messageList.push(newFileInfo.filename);
+          //existingFile includes the organization name as the uploaded folder. Checking if file exist and completed. If exist and completed, show error messages and prevent from sending upload request
+          if (existingFile?.upload_status === UPLOAD_STATUS.COMPLETED && existingFile.original_filename.split("/")[existingFile.original_filename.split("/").length - 1] === newFileInfo.filename) {
+            messageList.push("ROS2 Rosbag files exist: " + newFileInfo.filename);
             isValid = false;
           }
         }
@@ -121,7 +127,7 @@ const ROS2RosbagPage = React.memo(() => {
           open: true,
           severity: NOTIFICATION_STATUS.ERROR,
           title: "Error upload",
-          message: "ROS2 Rosbag files exist: " + messageList.join(),
+          message: messageList,
         });
         return isValid;
       }
@@ -130,7 +136,7 @@ const ROS2RosbagPage = React.memo(() => {
         open: true,
         severity: NOTIFICATION_STATUS.ERROR,
         title: "Error upload",
-        message: "ROS2 Rosbag files cannot be empty!",
+        message: ["ROS2 Rosbag files cannot be empty!"],
       });
       isValid = false;
     }
@@ -146,16 +152,14 @@ const ROS2RosbagPage = React.memo(() => {
           open: true,
           severity: NOTIFICATION_STATUS.SUCCESS,
           title: "ROS2 Rosbag files upload",
-          message:
-            "Server responds with ROS2 Rosbag files upload end! Click the refresh button to get the latest upload status.",
+          message: ["Server responds with ROS2 Rosbag files upload end! Click the refresh button to get the latest upload status."],
         });
       });
       setAlertStatus({
         open: true,
         severity: NOTIFICATION_STATUS.WARNING,
         title: "ROS2 Rosbag files upload",
-        message:
-          "ROS2 Rosbag files upload request sent! Please DOT NOT close this browser window tab until the ROS2 Rosbag files upload completed! Click the refresh button to get the latest upload status.",
+        message: ["ROS2 Rosbag files upload request sent! Please DOT NOT close this browser window tab until the ROS2 Rosbag files upload completed! Click the refresh button to get the latest upload status."],
       });
     }
   };
@@ -167,7 +171,7 @@ const ROS2RosbagPage = React.memo(() => {
           open: true,
           severity: NOTIFICATION_STATUS.ERROR,
           title: "Error",
-          message: data.errMsg,
+          message: [data.errMsg],
         });
       } else {
         let filterredROS2RosbagList = data;
@@ -185,7 +189,7 @@ const ROS2RosbagPage = React.memo(() => {
 
         if (ROS2RosbagCtx.filterText.length > 0) {
           filterredROS2RosbagList = filterredROS2RosbagList.filter(
-            (item) => (item.description !== null && item.description.includes(ROS2RosbagCtx.filterText)) || (item.original_filename !== null && item.original_filename.includes(ROS2RosbagCtx.filterText))
+            (item) => (item.description !== null && item.description.toLowerCase().includes(ROS2RosbagCtx.filterText.toLowerCase().toLowerCase())) || (item.original_filename !== null && item.original_filename.includes(ROS2RosbagCtx.filterText.toLowerCase()))
           );
         }
         setROS2RosbagList(filterredROS2RosbagList);
@@ -200,7 +204,7 @@ const ROS2RosbagPage = React.memo(() => {
           open: true,
           severity: NOTIFICATION_STATUS.ERROR,
           title: "Error",
-          message: data.errMsg,
+          message: [data.errMsg],
         });
       } else {
         setROS2RosbagList(data);
@@ -215,7 +219,7 @@ const ROS2RosbagPage = React.memo(() => {
         closeAlert={closeAlertHandler}
         severity={alertStatus.severity}
         title={alertStatus.title}
-        message={alertStatus.message}
+        messageList={alertStatus.message}
       />
       {authCtx.role !== undefined && authCtx.role !== null && authCtx.role !== "" && (
         <Grid container columnSpacing={2} rowSpacing={1}>

telematic_system/telematic_apps/web_app/client/src/tests/pages/ROS2RosbagPage.test.js

@@ -9,55 +9,59 @@ import ROS2RosbagContext from "../../context/ros2-rosbag-context";
 import AuthContext from "../../context/auth-context";
 import ROS2RosbagPage from "../../pages/ROS2RosbagPage";
 
-test("ROS2 Rosbag page", async () => {
-  const ROS2RosbagList = [
-    {
-      content_location: "content_location",
-      created_at: 1707752416,
-      updated_at: 1707752416,
+const ROS2RosbagList = [
+  {
+    content_location: "content_location",
+    created_at: 1707752416,
+    updated_at: 1707752416,
+    id: 1,
+    original_filename: "test1",
+    upload_status: "COMPLETED",
+    upload_error_msg: "",
+    process_status: "ERROR",
+    process_error_msg: "Test",
+    size: 12,
+    created_by: 1,
+    updated_by: 1,
+    description: "test description",
+    user: {
+      email: "admin@gmail.com",
       id: 1,
-      original_filename: "test1",
-      upload_status: "COMPLETED",
-      upload_error_msg: "",
-      process_status: "ERROR",
-      process_error_msg: "Test",
-      size: 12,
-      created_by: 1,
-      updated_by: 1,
-      description: "test description",
-      user: {
-        email: "admin@gmail.com",
-        id: 1,
-        name: "",
-        org_id: 1,
-      },
+      name: "",
+      org_id: 1,
     },
-    {
-      content_location: "content_location",
-      created_at: 1707752416,
-      updated_at: 1707752416,
-      id: 2,
-      original_filename: "test",
-      upload_status: "IN_PROGRESS",
-      upload_error_msg: "",
-      process_error_msg: "",
-      process_status: "NA",
-      size: 23575448,
-      created_by: 1,
-      updated_by: 1,
-      description: "test description",
-      user: {
-        email: "dmin@gmail.com",
-        id: 1,
-        name: "",
-        org_id: 1,
-      },
+  },
+  {
+    content_location: "content_location",
+    created_at: 1707752416,
+    updated_at: 1707752416,
+    id: 2,
+    original_filename: "test",
+    upload_status: "IN_PROGRESS",
+    upload_error_msg: "",
+    process_error_msg: "",
+    process_status: "NA",
+    size: 23575448,
+    created_by: 1,
+    updated_by: 1,
+    description: "test description",
+    user: {
+      email: "dmin@gmail.com",
+      id: 1,
+      name: "",
+      org_id: 1,
     },
-  ];
+  },
+];
+
+beforeEach(() => {
 
   jest
     .spyOn(ROS2RosbagApi, "listROS2Rosbags")
     .mockResolvedValue(ROS2RosbagList);
+})
+
+test("ROS2 Rosbag page", async () => {
 
   await act(async () => {
     render(
@@ -66,7 +70,8 @@ test("ROS2 Rosbag page", async () => {
           value={{
             filterText: "",
             uploadStatus: "",
-            processingStatus: "ERROR",
+            processingStatus: "",
+            clear: () => { }
           }}
         >
           <ROS2RosbagPage>
@@ -82,6 +87,17 @@ test("ROS2 Rosbag page", async () => {
   });
 
   fireEvent.click(screen.getByTestId("uploadROS2RosbagBtn"));
+
+  await waitFor(() => {
+    expect(screen.getByTestId(/refreshBtn/i)).toBeInTheDocument();
+  })
+  fireEvent.click(screen.getByTestId("refreshBtn"));
+
+  await waitFor(() => {
+    expect(screen.getByTestId(/Process/i)).toBeInTheDocument();
+  });
+  fireEvent.click(screen.getByTestId("Process"));
+
 });
 
 test("ROS2 Rosbag upload dialog", async () => {