Historical data page: Files should be upload to individual folder of S3 bucket based on user organization

telematic_system/telematic_apps/web_app/client/src/components/ros2_rosbag/ROS2RosbagRowItem.js

@@ -39,10 +39,11 @@ const ROS2RosbagRowItem = (props) => {
           let isBlue = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.IN_PROGRESS) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.IN_PROGRESS);
           let isGreen = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.COMPLETED) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.COMPLETED);
           let isRed = (column.id === "process_status" && props.ROS2RosbagRow.process_status === PROCESSING_STATUS.ERROR) || (column.id === "upload_status" && props.ROS2RosbagRow.upload_status === UPLOAD_STATUS.ERROR);
-          let createdBy = column.id === "created_by" && props.ROS2RosbagRow.user !== null && props.ROS2RosbagRow.user.email !== null ? props.ROS2RosbagRow.user.email : "NA";
+          let createdBy = column.id === "created_by" && props.ROS2RosbagRow.user !== null && props.ROS2RosbagRow.user.login !== null ? props.ROS2RosbagRow.user.login : "NA";
           value = column.id === "size" ? calFilesizes(value) : value;
           value = column.id === "created_by" ? createdBy : value;
           value = column.id === "created_at" ? new Date(value).toLocaleString() : value;
+          value = column.id === "original_filename" && value.includes("/") ? value.split("/")[value.split("/").length-1]: value;
 
           return (
             <TableCell key={`ros2-rosbag-cell-${props.ROS2RosbagRow.id}-${column.id}`} align={column.align}

telematic_system/telematic_apps/web_app/client/src/context/auth-context.js

@@ -29,7 +29,7 @@ const AuthContext = React.createContext({
   user_id: null,
   role: null,
   view_count: null,
-  login: (id, username, sessionToken, sessionExpiredAt, email, last_seen_at, org_id, name, is_admin) => { },
+  login: (id, username, sessionToken, sessionExpiredAt, email, last_seen_at, org_id,org_name, name, is_admin) => { },
   logout: () => { },
   updateRole: (role) => { },
   updateOrg: (org_id, org_name) => { },
@@ -51,7 +51,7 @@ export const AuthContextProvider = (props) => {
   const clearSessionStorage = useClearSessionStorage();
   const [sessionExpiredAt, setSessionExpiredAt] = useSessionStorageNumber("sessionExpiredAt",0);
 
-  const loginHandler = (user_id, username, sessionToken, sessionExpiredAt, email, last_seen_at, org_id, name, is_admin) => {
+  const loginHandler = (user_id, username, sessionToken, sessionExpiredAt, email, last_seen_at, org_id,org_name, name, is_admin) => {
     if (username !== undefined && username !== ""
       && sessionToken !== undefined && sessionToken !== "") {
       setUserId(user_id);
@@ -62,6 +62,7 @@ export const AuthContextProvider = (props) => {
       setSessionExpiredAt(sessionExpiredAt);
       setLastSeenAt(last_seen_at);
       setOrgId(org_id);
+      setOrgName(org_name)
       setName(name);
       setIsAdmin(is_admin);
       return true;

telematic_system/telematic_apps/web_app/client/src/pages/Login.js

@@ -82,6 +82,7 @@ const Login = React.memo(() => {
                 resData.email,
                 resData.last_seen_at,
                 resData.org_id,
+                resData.org_name,
                 resData.name,
                 resData.is_admin);
             setLoginState(true);

telematic_system/telematic_apps/web_app/client/src/pages/ROS2RosbagPage.js

@@ -67,10 +67,6 @@ const ROS2RosbagPage = React.memo(() => {
     });
   };
 
-  const isInCurrentOrg = (ROS2RosbagInfo, authCtx) => {
-    return (ROS2RosbagInfo.user !== null && ROS2RosbagInfo.user.org_id === parseInt(authCtx.org_id));
-  };
-
   const refreshHandler = () => {
     listROS2Rosbags().then((data) => {
       if (data.errCode !== undefined && data.errMsg !== undefined) {
@@ -81,7 +77,7 @@ const ROS2RosbagPage = React.memo(() => {
           message: data.errMsg,
         });
       } else {
-        setROS2RosbagList(data.filter((item) => isInCurrentOrg(item, authCtx)));
+        setROS2RosbagList(data);
       }
     });
   };
@@ -112,7 +108,8 @@ const ROS2RosbagPage = React.memo(() => {
       let messageList = [];
       uploadFileInfoList.forEach(newFileInfo => {
         for (let existingFile of fileInfoList) {
-          if (existingFile.original_filename === newFileInfo.filename) {
+          //Existing original file name in DB includes the organization name as the uploaded folder
+          if (existingFile.original_filename.split("/")[existingFile.original_filename.split("/").length -1] === newFileInfo.filename) {
             messageList.push(newFileInfo.filename);
             isValid = false;
           }
@@ -173,9 +170,7 @@ const ROS2RosbagPage = React.memo(() => {
           message: data.errMsg,
         });
       } else {
-        let filterredROS2RosbagList = data.filter((item) =>
-          isInCurrentOrg(item, authCtx)
-        );
+        let filterredROS2RosbagList = data;
         if (ROS2RosbagCtx.uploadStatus.length > 0) {
           filterredROS2RosbagList = filterredROS2RosbagList.filter(
             (item) => (item.upload_status !== null && item.upload_status.toUpperCase().trim() === ROS2RosbagCtx.uploadStatus) || (ROS2RosbagCtx.uploadStatus === UPLOAD_STATUS.NA && (item.upload_status === null || item.upload_status.length === 0))
@@ -208,7 +203,7 @@ const ROS2RosbagPage = React.memo(() => {
           message: data.errMsg,
         });
       } else {
-        setROS2RosbagList((data) => data.filter((item) => isInCurrentOrg(item, authCtx)));
+        setROS2RosbagList(data);
       }
     });
   }, [authCtx]);

telematic_system/telematic_apps/web_app/server/controllers/file_info.controller.js

@@ -46,7 +46,7 @@ exports.list = (filterFields) => {
       include: [
         {
           model: user,
-          attributes: ["id", "name", "org_id", "email"],
+          attributes: ["id", "name", "org_id", "email", "login"],
         },
       ],
     })

telematic_system/telematic_apps/web_app/server/controllers/user.controller.js

@@ -15,7 +15,7 @@
  */
 var manager = require('htpasswd-mgr');
 const saltHash = require('password-salt-and-hash')
-const { org_user, user, Sequelize } = require("../models");
+const { org_user, org, user, Sequelize } = require("../models");
 const { Op } = require("sequelize");
 const { addOrgUser } = require('./org.controller');
 const getUuid = require('uuid-by-string');
@@ -196,24 +196,30 @@ exports.loginUser = (req, res) => {
             if (is_pwd_match) {
                 //Update user credential file
                 htpasswordManager.upsertUser(req.body.username, req.body.password).then((status) => {
-                    let result = {
-                        id: data[0].id,
-                        last_seen_at: data[0].last_seen_at,
-                        is_admin: data[0].is_admin,
-                        email: data[0].email,
-                        name: data[0].name,
-                        org_id: data[0].org_id,
-                        login: data[0].login,
-                        username: data[0].login,
-                    }
-                    //Creating jwt token, and the token expire in an hour
-                    let token = jwt.sign(
-                        result,
-                        process.env.SECRET,
-                        { expiresIn: "1h" });
-                    result.token = token;
-                    result.tokenExpiredAt = Math.round(new Date().getTime()/1000) + 3600;
-                    res.status(200).send(result);
+                    //get org name
+                    org.findAll({ where: { id: data[0].org_id } }).then(org_data => {
+                        if (org_data.length > 0) {
+                            let result = {
+                                id: data[0].id,
+                                last_seen_at: data[0].last_seen_at,
+                                is_admin: data[0].is_admin,
+                                email: data[0].email,
+                                name: data[0].name,
+                                org_id: data[0].org_id,
+                                login: data[0].login,
+                                username: data[0].login,
+                                org_name: org_data[0].name
+                            }
+                            //Creating jwt token, and the token expire in an hour
+                            let token = jwt.sign(
+                                result,
+                                process.env.SECRET,
+                                { expiresIn: "1h" });
+                            result.token = token;
+                            result.tokenExpiredAt = Math.round(new Date().getTime() / 1000) + 3600;
+                            res.status(200).send(result);
+                        }
+                    });
                 }).catch((err) => {
                     console.error(err)
                     res.status(500).send({ message: "Server error while user login." });

telematic_system/telematic_apps/web_app/server/file_upload/file_list_service.js

@@ -25,6 +25,7 @@
  */
 const fileInfoController = require("../controllers/file_info.controller");
 const listObjectsModule = require("../file_upload/s3_list_objects");
+const { verifyToken } = require("../utils/verifyToken");
 const { UPLOADSTATUS } = require("./file_upload_status_emitter");
 require("dotenv").config();
 const uploadDest = process.env.UPLOAD_DESTINATION;
@@ -55,7 +56,17 @@ const listAllFiles = async (req, res) => {
 
 const listAllDBFiles = async (req, res) => {
   try {
-    return await fileInfoController.list({});
+    let contents = [];
+    //Get user organization name
+    let folderName = verifyToken(req).org_name.replaceAll(' ', '_');
+    let data = await fileInfoController.list({});
+    for (const d of data) {
+      //Only push files of current folder (=user organization name)
+      if (d.original_filename.includes(folderName)) {
+        contents.push(d);
+      }
+    }
+    return contents;
   } catch (err) {
     console.error("Cannot get a list of All DB files!");
     console.trace();
@@ -67,13 +78,18 @@ const listAllDBFilesAndS3Objects = async (req, res) => {
   try {
     let contents = [];
     let existingFileNames = [];
+    //Get user organization name
+    let folderName = verifyToken(req).org_name.replaceAll(' ', '_');
     let data = await fileInfoController.list({});
-    contents.push(...data);
     for (const d of data) {
-      existingFileNames.push(d.original_filename);
+      //Only push files of current folder (=user organization name)
+      if (d.original_filename.includes(folderName)) {
+        existingFileNames.push(d.original_filename);
+        contents.push(d);
+      }
     }
-
-    let objects = await listObjectsModule.listObjects();
+    //Get a list of objects from organization folder in S3 bucket
+    let objects = await listObjectsModule.listObjects(folderName);
     console.log("Your bucket contains the following objects:");
     console.log(objects);
 

telematic_system/telematic_apps/web_app/server/file_upload/file_upload_service.js

@@ -23,6 +23,7 @@
 const formidable = require("formidable");
 const { uploadToS3 } = require("./s3_uploader");
 require("dotenv").config();
+var fs = require('fs');
 const uploadDest = process.env.UPLOAD_DESTINATION;
 const uploadDestPath = process.env.UPLOAD_DESTINATION_PATH;
 const uploadMaxFileSize = parseInt(process.env.UPLOAD_MAX_FILE_SIZE);
@@ -131,6 +132,14 @@ const parseLocalFileUpload = async (req, form, listener, NATSConn) => {
   });
 
   form.on("fileBegin", (formName, file) => {
+    let folderName = userInfo.org_name.replaceAll(' ', '_');
+    //Update file name prefix with folder name (= organization name) to be consistent with s3 originalFilename
+    file.originalFilename = folderName + "/"+ file.originalFilename;
+    //create folder with org name if does not already exist
+    let uploadFolder = uploadDestPath + "/"+folderName;
+    if (!fs.existsSync(uploadFolder)){
+      fs.mkdirSync(uploadFolder);
+  }
     file.updated_by = userInfo.id;
     file.created_by = userInfo.id;
     //Update file info status
@@ -167,6 +176,8 @@ const parseS3FileUpload = async (req, form, listener, NATSConn) => {
   });
 
   form.on("fileBegin", async (formName, file) => {
+    //Get user org name and file is uploaded to organization folder in S3 bucket
+    file.originalFilename = userInfo.org_name.replaceAll(' ', '_') + "/"+ file.originalFilename;
     //Update upload status
     updateFileUploadStatusEmitter(listener).emit(
       UPLOADSTATUS.IN_PROGRESS,

telematic_system/telematic_apps/web_app/server/file_upload/s3_list_objects.js

@@ -27,31 +27,37 @@ const secretAccessKey = process.env.AWS_SECRET_KEY;
 const region = process.env.S3_REGION;
 const bucket = process.env.S3_BUCKET;
 
-exports.listObjects = async () => {
+exports.listObjects = async (s3Folder) => {
   const client = new S3Client({
     credentials: {
       accessKeyId,
       secretAccessKey,
     },
     region,
   });
-  const command = new ListObjectsV2Command({
-    Bucket: bucket,
-  });
   let contents = [];
-  let isTruncated = true;
-  while (isTruncated) {
-    const { Contents, IsTruncated, NextContinuationToken } = await client.send(
-      command
-    );
-    const contentsList = Contents.map((c) => ({
-      original_filename: c.Key,
-      size: c.Size,
-      filepath: bucket,
-    }));
-    isTruncated = IsTruncated;
-    contents.push(...contentsList);
-    command.input.ContinuationToken = NextContinuationToken;
+  try {
+    const command = new ListObjectsV2Command({
+      Bucket: bucket,
+      Prefix: s3Folder + '/',
+      Delimiter: '/',
+    });
+    let isTruncated = true;
+    while (isTruncated) {
+      const { Contents, IsTruncated, NextContinuationToken } = await client.send(
+        command
+      );
+      const contentsList = Contents.map((c) => ({
+        original_filename: c.Key,
+        size: c.Size,
+        filepath: bucket,
+      }));
+      isTruncated = IsTruncated;
+      contents.push(...contentsList);
+      command.input.ContinuationToken = NextContinuationToken;
+    }
+  } catch (err) {
+    console.error("Cannot find files in S3 bucket: " + bucket + ", folder:  "+ s3Folder)
   }
   return contents;
 };

telematic_system/telematic_apps/web_app/server/models/index.js

@@ -66,7 +66,8 @@ grafana_db.event_units.belongsTo(grafana_db.units);
 grafana_db.user.hasMany(grafana_db.file_info, {
     foreignKey: {
         name: 'userId',
-        field: 'user_id'
+        field: 'user_id',
+        type: Sequelize.BIGINT(20)
     }
 });
 grafana_db.file_info.belongsTo(grafana_db.user);