Skip to content

Commit

Permalink
Historical Data Page: File existence validation for file upload (#230)
Browse files Browse the repository at this point in the history 
* add query by filenames

* validate file existence

* return when invalid

* reove logs
  • Loading branch information
@dan-du-car
dan-du-car authored Jun 18, 2024
1 parent 3b57f8a commit 091c8d3
Show file tree
Hide file tree
Showing 7 changed files with 246 additions and 70 deletions.
22 changes: 22 additions & 0 deletions telematic_system/telematic_apps/web_app/client/src/api/api-ros2-rosbag.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,27 @@ const uploadROS2Rosbags = async (ROS2RosbagsFormData) => {
}
};

/***
*@brief Send post request to validate files
@param ROS2RosbagsFormData Request form data with file content.
@return Server response
*/
const validateROS2Rosbags = async (ROS2RosbagsFormData) => {
const URL = `${env.REACT_APP_FILE_UPLOAD_WEB_SERVER_URI}/api/upload/validation`;
try {
let formData = new FormData();
let fields = ROS2RosbagsFormData["fields"];
for (let key in fields) {
formData.append("fields", JSON.stringify(fields[key]));
}
const { data } = await axios.post(URL, formData);
return data;
} catch (err) {
console.log(err)
return constructError(err);
}
};

/**
* @brief Send POST request to process an existing rROS2 rosbag in the server.
* @returns server response with acknowledgement
Expand All @@ -87,4 +108,5 @@ export {
uploadROS2Rosbags,
updateDescription,
sendProcessRequest,
validateROS2Rosbags,
};
2 changes: 1 addition & 1 deletion ...atic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagRowItem.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ const ROS2RosbagRowItem = (props) => {
wordBreak: "break-word",
}}
>
{value}
{value.toLowerCase()}

{column.id === "process_status" && isRed && (
<InfoPopover
Expand Down
6 changes: 3 additions & 3 deletions telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagTable.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ const columns = [
align: "right",
info: "",
},
{ id: "upload_status", label: "Upload Status", minWidth: 100, info: "" },
{ id: "upload_status", label: "Upload Status", minWidth: 120, info: "" },
{
id: "process_status",
label: "Process Status",
minWidth: 100,
minWidth: 120,
info: "",
},
{
Expand All @@ -71,7 +71,7 @@ const columns = [
align: "right",
format: (value) => value.toLocaleString("en-US"),
info: "",
}
},
];

export default function ROS2RosbagTable(props) {
Expand Down
160 changes: 120 additions & 40 deletions telematic_system/telematic_apps/web_app/client/src/pages/ROS2RosbagPage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
sendProcessRequest,
updateDescription,
uploadROS2Rosbags,
validateROS2Rosbags,
} from "../api/api-ros2-rosbag";
import ROS2ROSBagFilter from "../components/ros2_rosbag/ROS2ROSBagFilter";
import {
Expand All @@ -40,8 +41,12 @@ const ROS2RosbagPage = React.memo(() => {
const ROS2RosbagCtx = React.useContext(ROS2RosbagContext);
const authCtx = React.useContext(AuthContext);
const [alertStatus, setAlertStatus] = useState({});
const [uploadStatusList, setUploadStatusList] = useState(Object.keys(UPLOAD_STATUS).map((status) => status));
const [processingStatusList, setProcessingStatusList] = useState(Object.keys(PROCESSING_STATUS).map((status) => status));
const [uploadStatusList, setUploadStatusList] = useState(
Object.keys(UPLOAD_STATUS).map((status) => status)
);
const [processingStatusList, setProcessingStatusList] = useState(
Object.keys(PROCESSING_STATUS).map((status) => status)
);
const [ROS2RosbagList, setROS2RosbagList] = useState([]);

const closeAlertHandler = () => {
Expand All @@ -63,7 +68,13 @@ const ROS2RosbagPage = React.memo(() => {
message: [data.errMsg],
});
} else {
setROS2RosbagList([UpdatedFileInfo, ...ROS2RosbagList.filter((item) => item.original_filename !== UpdatedFileInfo.original_filename)]);
setROS2RosbagList([
UpdatedFileInfo,
...ROS2RosbagList.filter(
(item) =>
item.original_filename !== UpdatedFileInfo.original_filename
),
]);
}
});
};
Expand Down Expand Up @@ -103,26 +114,30 @@ const ROS2RosbagPage = React.memo(() => {
});
};

const validateUpload = (fileInfoList, uploadFileInfoList) => {
const validateUpload = async (ROS2RosBagsFormData) => {
let uploadFileInfoList = ROS2RosBagsFormData["fields"] || [];
let isValid = true;
let messageList = [];
if (Array.isArray(uploadFileInfoList) && uploadFileInfoList.length > 0) {
let messageList = [];
uploadFileInfoList.forEach(newFileInfo => {
uploadFileInfoList.forEach((newFileInfo) => {
//Check file extensions
if (!ACCEPT_FILE_EXTENSIONS?.toLowerCase().includes(newFileInfo?.filename?.split('.')[newFileInfo?.filename?.split('.').length - 1])) {
messageList.push("Invalid files (only accept " + ACCEPT_FILE_EXTENSIONS + " files): " + newFileInfo?.filename);
if (
!ACCEPT_FILE_EXTENSIONS?.toLowerCase().includes(
newFileInfo?.filename?.split(".")[
newFileInfo?.filename?.split(".").length - 1
]
)
) {
messageList.push(
"Invalid files (only accept " +
ACCEPT_FILE_EXTENSIONS +
" files): " +
newFileInfo?.filename
);
isValid = false;
}
for (let existingFile of fileInfoList) {
//existingFile includes the organization name as the uploaded folder. Checking if file exist and completed. If exist and completed, show error messages and prevent from sending upload request
if (existingFile?.upload_status === UPLOAD_STATUS.COMPLETED && existingFile.original_filename.split("/")[existingFile.original_filename.split("/").length - 1] === newFileInfo.filename) {
messageList.push("ROS2 Rosbag files exist: " + newFileInfo.filename);
isValid = false;
}
}
});

if (messageList.length > 0) {
if (!isValid && messageList.length > 0) {
setAlertStatus({
open: true,
severity: NOTIFICATION_STATUS.ERROR,
Expand All @@ -139,27 +154,56 @@ const ROS2RosbagPage = React.memo(() => {
message: ["ROS2 Rosbag files cannot be empty!"],
});
isValid = false;
return isValid;
}

//Server side validation
let data = await validateROS2Rosbags(ROS2RosBagsFormData);
if (data.errCode !== undefined && data.errMsg !== undefined) {
messageList.push(data.errMsg);
isValid = false;
}
if (!isValid && messageList.length > 0) {
setAlertStatus({
open: true,
severity: NOTIFICATION_STATUS.ERROR,
title: "Error upload",
message: messageList,
});
}
return isValid;
}
};

const uploadHandler = (ROS2RosBagsFormData) => {
let fields = ROS2RosBagsFormData["fields"] || [];
const uploadHandler = async (ROS2RosBagsFormData) => {
console.log(ROS2RosBagsFormData);
if (validateUpload(ROS2RosbagList, fields)) {
let isValid = await validateUpload(ROS2RosBagsFormData);
if (isValid) {
uploadROS2Rosbags(ROS2RosBagsFormData).then((data) => {
setAlertStatus({
open: true,
severity: NOTIFICATION_STATUS.SUCCESS,
title: "ROS2 Rosbag files upload",
message: ["Server responds with ROS2 Rosbag files upload end! Click the refresh button to get the latest upload status."],
});
if (data.errCode !== undefined && data.errMsg !== undefined) {
setAlertStatus({
open: true,
severity: NOTIFICATION_STATUS.ERROR,
title: "Error",
message: [data.errMsg],
});
} else {
setAlertStatus({
open: true,
severity: NOTIFICATION_STATUS.SUCCESS,
title: "ROS2 Rosbag files upload",
message: [
"Server responds with ROS2 Rosbag files upload end! Click the refresh button to get the latest upload status.",
],
});
}
});
setAlertStatus({
open: true,
severity: NOTIFICATION_STATUS.WARNING,
title: "ROS2 Rosbag files upload",
message: ["ROS2 Rosbag files upload request sent! Please DOT NOT close this browser window tab until the ROS2 Rosbag files upload completed! Click the refresh button to get the latest upload status."],
message: [
"ROS2 Rosbag files upload request sent! Please DOT NOT close this browser window tab until the ROS2 Rosbag files upload completed! Click the refresh button to get the latest upload status.",
],
});
}
};
Expand All @@ -177,19 +221,41 @@ const ROS2RosbagPage = React.memo(() => {
let filterredROS2RosbagList = data;
if (ROS2RosbagCtx.uploadStatus.length > 0) {
filterredROS2RosbagList = filterredROS2RosbagList.filter(
(item) => (item.upload_status !== null && item.upload_status.toUpperCase().trim() === ROS2RosbagCtx.uploadStatus) || (ROS2RosbagCtx.uploadStatus === UPLOAD_STATUS.NA && (item.upload_status === null || item.upload_status.length === 0))
(item) =>
(item.upload_status !== null &&
item.upload_status.toUpperCase().trim() ===
ROS2RosbagCtx.uploadStatus) ||
(ROS2RosbagCtx.uploadStatus === UPLOAD_STATUS.NA &&
(item.upload_status === null ||
item.upload_status.length === 0))
);
}

if (ROS2RosbagCtx.processingStatus.length > 0) {
filterredROS2RosbagList = filterredROS2RosbagList.filter(
(item) => (item.process_status !== null && item.process_status.toUpperCase().trim() === ROS2RosbagCtx.processingStatus) || (ROS2RosbagCtx.processingStatus === PROCESSING_STATUS.NA && (item.process_status === null || item.process_status.length === 0))
(item) =>
(item.process_status !== null &&
item.process_status.toUpperCase().trim() ===
ROS2RosbagCtx.processingStatus) ||
(ROS2RosbagCtx.processingStatus === PROCESSING_STATUS.NA &&
(item.process_status === null ||
item.process_status.length === 0))
);
}

if (ROS2RosbagCtx.filterText.length > 0) {
filterredROS2RosbagList = filterredROS2RosbagList.filter(
(item) => (item.description !== null && item.description.toLowerCase().includes(ROS2RosbagCtx.filterText.toLowerCase().toLowerCase())) || (item.original_filename !== null && item.original_filename.includes(ROS2RosbagCtx.filterText.toLowerCase()))
(item) =>
(item.description !== null &&
item.description
.toLowerCase()
.includes(
ROS2RosbagCtx.filterText.toLowerCase().toLowerCase()
)) ||
(item.original_filename !== null &&
item.original_filename.includes(
ROS2RosbagCtx.filterText.toLowerCase()
))
);
}
setROS2RosbagList(filterredROS2RosbagList);
Expand Down Expand Up @@ -221,16 +287,30 @@ const ROS2RosbagPage = React.memo(() => {
title={alertStatus.title}
messageList={alertStatus.message}
/>
{authCtx.role !== undefined && authCtx.role !== null && authCtx.role !== "" && (
<Grid container columnSpacing={2} rowSpacing={1}>
<PageAvatar icon={<WorkHistorySharpIcon />} title="ROS2 Rosbag" />
<Grid item xs={4}></Grid>
<ROS2ROSBagFilter uploadStatusList={uploadStatusList} processingStatusList={processingStatusList} onRefresh={refreshHandler} filter={filterHandler} onUpload={uploadHandler} />
<Grid container item xs={12}>
<ROS2RosbagTable ROS2RosbagList={ROS2RosbagList} onSaveDescription={saveDescriptionHandler} onProcessReq={(ROS2RosBagInfo) => processReqHandler(ROS2RosBagInfo)} />
{authCtx.role !== undefined &&
authCtx.role !== null &&
authCtx.role !== "" && (
<Grid container columnSpacing={2} rowSpacing={1}>
<PageAvatar icon={<WorkHistorySharpIcon />} title="ROS2 Rosbag" />
<Grid item xs={4}></Grid>
<ROS2ROSBagFilter
uploadStatusList={uploadStatusList}
processingStatusList={processingStatusList}
onRefresh={refreshHandler}
filter={filterHandler}
onUpload={uploadHandler}
/>
<Grid container item xs={12}>
<ROS2RosbagTable
ROS2RosbagList={ROS2RosbagList}
onSaveDescription={saveDescriptionHandler}
onProcessReq={(ROS2RosBagInfo) =>
processReqHandler(ROS2RosBagInfo)
}
/>
</Grid>
</Grid>
</Grid>
)}
)}
</React.Fragment>
);
});
Expand Down
21 changes: 20 additions & 1 deletion telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
* - upsertFileInfo: Update the file_info table based on the unique filename if file_info exist in the database table, and return the number
* of records updated. Otherwise, insert a new file_info record into the database table and return the new record.
*/
const { Op } = require("sequelize");
const { file_info, user } = require("../models");

/**
Expand Down Expand Up @@ -119,11 +120,29 @@ exports.upsertFileInfo = async (fileInfo) => {
}
let condition = { original_filename: originalFilename };
return await file_info
.upsert(fileInfoLocal, condition)
.upsert(fileInfoLocal, condition)
.then(async (data) => {
return data[0];
})
.catch((err) => {
throw new Error("Error updating file info record: " + err);
});
};

exports.findByFilenames = async (filenames) => {
return await file_info
.findAll({
where: {
original_filename: {
[Op.in]: filenames,
},
},
})
.then((data) => {
console.log(data);
return data;
})
.catch((err) => {
throw new Error("Error find all files by filenames: " + err);
});
};
Loading

0 comments on commit 091c8d3

Please sign in to comment.