diff --git a/src/components/ExportToCognitoS3Dialog/index.js b/src/components/ExportToCognitoS3Dialog/index.js index f6f37832..fd52652c 100644 --- a/src/components/ExportToCognitoS3Dialog/index.js +++ b/src/components/ExportToCognitoS3Dialog/index.js @@ -5,8 +5,7 @@ import useActiveDatasetManager from "../../hooks/use-active-dataset-manager" import isEmpty from "lodash/isEmpty" import datasetManagerCognito from "udt-dataset-managers/dist/CognitoDatasetManager" import useAuth from "../../utils/auth-handlers/use-auth" -import { TextField, Grid } from "@material-ui/core" - +import { Grid, TextField } from "@material-ui/core" const redText = { color: "orange" } const expandedAnnotationsColumns = [ @@ -133,15 +132,11 @@ export default ({ open, onClose }) => { }, [nameProjectToCreate, projects]) const handleCreateProject = async () => { - if (nameProjectExist) await dm.removeProject(nameProjectToCreate) - var dataset = currentDataset + var dataset = await activeDatasetManager.getDataset() dataset = dataset.setIn(["name"], nameProjectToCreate) - await dm.createProject({ - name: nameProjectToCreate, - interface: dataset.interface, - }) + if (nameProjectExist) await dm.removeSamplesFolder(nameProjectToCreate) await dm.setDataset(dataset) - await activeDatasetManager.setDatasetProperty("name", nameProjectToCreate) + await activeDatasetManager.setDataset(dataset) await getProjects() onClose() } diff --git a/src/components/ImportFromCognitoS3Dialog/get-annotation-from-aws.js b/src/components/ImportFromCognitoS3Dialog/get-annotation-from-aws.js deleted file mode 100644 index 7afdd667..00000000 --- a/src/components/ImportFromCognitoS3Dialog/get-annotation-from-aws.js +++ /dev/null @@ -1,57 +0,0 @@ -import isEmpty from "lodash/isEmpty" -import Amplify, { Storage } from "aws-amplify" -import * as datasetHelper from "../../utils//dataset-helper" -function CheckIfAnnotationExist(result, folderToFetch) { - return result.find( - (element) => element.key === `${folderToFetch}/annotations/annotations.json` - ) -} - -function GetSampleFromAnnotation(json, samples) { - if (isEmpty(json.content.samples)) return - var newSamples = [] - for (var i = 0; i < json.content.samples.length; i++) { - var sampleName = datasetHelper.getSampleName(json.content.samples[i]) - var annotation = json.content.samples[i].annotation - var sampleFound = datasetHelper.getSampleWithThisSampleName( - sampleName, - samples - ) - var url - if (sampleFound === null) { - url = datasetHelper.getSampleUrl(samples) - } else { - url = datasetHelper.getSampleUrl(sampleFound) - } - newSamples.push( - datasetHelper.createOneNewSample(sampleName, url, annotation) - ) - } - json.content.samples = newSamples -} - -export default async (result, samples, folderToFetch, authConfig) => { - Amplify.configure(authConfig) - - var json = null - if (CheckIfAnnotationExist(result, folderToFetch)) { - await Storage.get(`${folderToFetch}/annotations/annotations.json`, { - expires: 24 * 60 * 60 * 2000, - level: "private", - }) - .then(async (result) => { - await fetch(result).then(async (data) => { - return await data.json().then(async (result) => { - if (typeof result.content === "undefined") return - json = result - GetSampleFromAnnotation(json, samples) - }) - }) - }) - .catch((err) => { - console.log("error getting link for s3 image", err) - return null - }) - } - return json -} diff --git a/src/components/ImportFromCognitoS3Dialog/get-images-from-aws.js b/src/components/ImportFromCognitoS3Dialog/get-images-from-aws.js deleted file mode 100644 index dc3ceb4c..00000000 --- a/src/components/ImportFromCognitoS3Dialog/get-images-from-aws.js +++ /dev/null @@ -1,77 +0,0 @@ -import RecognizeFileExtension from "../../utils/RecognizeFileExtension" -import Amplify, { Storage } from "aws-amplify" - -async function setUrl(result, configImport) { - if (configImport.loadProjectIsSelected) { - if (RecognizeFileExtension(result) === "Image") { - return { imageUrl: `${result}` } - } else if (RecognizeFileExtension(result) === "Video") { - return { videoUrl: `${result}` } - } else if (RecognizeFileExtension(result) === "Audio") { - return { audioUrl: `${result}` } - } else if (RecognizeFileExtension(result) === "PDF") { - return { pdfUrl: `${result}` } - } - } else { - if ( - RecognizeFileExtension(result) === configImport.typeOfFileToLoad && - configImport.typeOfFileToLoad === "Image" - ) { - return { imageUrl: `${result}` } - } else if ( - RecognizeFileExtension(result) === configImport.typeOfFileToLoad && - configImport.typeOfFileToLoad === "Video" - ) { - return { videoUrl: `${result}` } - } else if ( - RecognizeFileExtension(result) === configImport.typeOfFileToLoad && - configImport.typeOfFileToLoad === "Audio" - ) { - return { audioUrl: `${result}` } - } else if ( - RecognizeFileExtension(result) === configImport.typeOfFileToLoad && - configImport.typeOfFileToLoad === "PDF" - ) { - return { pdfUrl: `${result}` } - } else if ( - RecognizeFileExtension(result) === configImport.typeOfFileToLoad && - configImport.typeOfFileToLoad === "Texte" - ) { - var texte = await fetchTextInFile(result) - return { document: `${texte}` } - } - } -} -async function fetchTextInFile(urlSource) { - var proxyUrl = "https://cors-anywhere.herokuapp.com/" - var response - var url - url = proxyUrl + urlSource - response = await fetch(url, { - method: "GET", - }).catch((error) => { - console.log("Looks like there was a problem: \n", error) - }) - const texte = await response.text() - return texte -} -export default async (result, folderToFetch, configImport, authConfig) => { - Amplify.configure(authConfig) - var samples = [] - for (let i = 0; i < result.length; i++) { - if (result[i].key.match(`(${folderToFetch}/data).*(\\.).*`)) { - await Storage.get(result[i].key, { - expires: 24 * 60 * 60, - level: "private", - }) - .then(async (result) => { - samples.push(await setUrl(result, configImport)) - }) - .catch((err) => { - console.log("error getting link for s3 image", err) - return null - }) - } - } - return samples -} diff --git a/src/components/ImportFromCognitoS3Dialog/get-sources.js b/src/components/ImportFromCognitoS3Dialog/get-sources.js new file mode 100644 index 00000000..5acd820a --- /dev/null +++ b/src/components/ImportFromCognitoS3Dialog/get-sources.js @@ -0,0 +1,6 @@ +const getSources = (annotations) => { + return annotations + .map((obj) => obj.source) + .filter((source, index, self) => self.indexOf(source) === index) +} +export default getSources diff --git a/src/components/ImportFromCognitoS3Dialog/header-table-import.js b/src/components/ImportFromCognitoS3Dialog/header-table-import.js new file mode 100644 index 00000000..bf7529f0 --- /dev/null +++ b/src/components/ImportFromCognitoS3Dialog/header-table-import.js @@ -0,0 +1,61 @@ +import React from "react" +import { + Settings as SettingsIcon, + Storage as StorageIcon, +} from "@material-ui/icons/" + +import { Button, IconButton } from "@material-ui/core/" + +const selectedStyle = { color: "DodgerBlue" } + +const headerTable = ({ configImport, setConfigImport }) => { + const loadAssetsOrAnnotations = () => { + setConfigImport({ + ...configImport, + loadAssetsIsSelected: !configImport.loadAssetsIsSelected, + }) + } + return ( + + + {configImport.loadAssetsIsSelected ? ( + + ) : ( + + )} + {configImport.loadAssetsIsSelected ? ( + + ) : ( + + )} + { + setConfigImport({ + ...configImport, + contentDialogBoxIsSetting: !configImport.contentDialogBoxIsSetting, + }) + }} + > + {configImport.contentDialogBoxIsSetting ? ( + + ) : ( + + )} + + + + ) +} +export default headerTable diff --git a/src/components/ImportFromCognitoS3Dialog/index.js b/src/components/ImportFromCognitoS3Dialog/index.js index 25680e2b..a247477e 100644 --- a/src/components/ImportFromCognitoS3Dialog/index.js +++ b/src/components/ImportFromCognitoS3Dialog/index.js @@ -1,111 +1,55 @@ -import React, { useEffect, useState } from "react" +import React, { useEffect, useRef, useState, useCallback } from "react" import SimpleDialog from "../SimpleDialog" import DataTable from "react-data-table-component" import useDataset from "../../hooks/use-dataset" import isEmpty from "lodash/isEmpty" import datasetManagerCognito from "udt-dataset-managers/dist/CognitoDatasetManager" import useAuth from "../../utils/auth-handlers/use-auth" +import setTypeOfFileToLoadAndDisable from "./set-type-of-file-to-load-and-disable" import initConfigImport from "./init-config-import" +import datasetHasChanged from "../../utils//dataset-helper/get-files-differences" import setUrl from "./set-url" import { setIn } from "seamless-immutable" -import { - Settings as SettingsIcon, - Storage as StorageIcon, -} from "@material-ui/icons/" -import { - Radio, - RadioGroup, - FormControl, - FormControlLabel, - Button, - FormLabel, - IconButton, - Grid, -} from "@material-ui/core/" - -const selectedStyle = { color: "DodgerBlue" } - -const expandedAssetsColumns = [ - { name: "Assets", selector: "assets", sortable: true }, - { name: "Last Modified", selector: "lastModified", sortable: true }, -] - -const expandedAnnotationsColumns = [ - { name: "Annotations", selector: "annotation" }, - { name: "Last Modified", selector: "lastModified", sortable: true }, -] +import ExpandedRow from "./table-expanded-row" +import SettingImport from "./interface-setting-import" +import HeaderTableImport from "./header-table-import" +import { Radio, Grid } from "@material-ui/core/" +import getSources from "./get-sources" const columns = [{ name: "Projects", selector: "folder", sortable: true }] -const customStyles = { - headCells: { - style: { - paddingLeft: "10px", - "font-weight": "bold", - }, - }, - cells: { - style: { - paddingLeft: "25px", - }, - }, -} - -const ExpandedRow = ({ data }) => { - const { rowAssets, rowAnnotations } = data - return ( - <> - 10} - paginationPerPage={10} - paginationRowsPerPageOptions={[10, 20, 25, 50, 100, 200]} - customStyles={customStyles} - /> - 10} - paginationPerPage={10} - paginationRowsPerPageOptions={[10, 20, 25, 50, 100, 200]} - customStyles={customStyles} - /> - - ) -} - export default ({ open, onClose, onAddSamples }) => { const [dm, setDm] = useState(null) const [oldData] = useDataset() const { authConfig } = useAuth() - const [projects, setProjects] = useState() - const [projectToFetch, setProjectToFetch] = useState() - const [configImport, setConfigImport] = useState({}) + const [projects, setProjects] = useState(null) + const [projectToFetch, setProjectToFetch] = useState("") + const [configImport, setConfigImport] = useState(initConfigImport(oldData)) + const lastObjectRef = useRef({}) - useEffect(() => { + const hasProjectStarted = useCallback(() => { if (!open) return - if (!oldData) return - setConfigImport(initConfigImport(oldData)) - }, [oldData, setConfigImport, open]) + if ( + isEmpty(oldData) || + (isEmpty(oldData.interface) && isEmpty(oldData.samples)) + ) + return false + return true + }, [oldData, open]) + + useEffect(() => { + if (oldData === lastObjectRef.current) return + var configToSet = configImport + const changes = datasetHasChanged(lastObjectRef.current, oldData) + if (changes.interface.type || changes.samples) { + configToSet = setTypeOfFileToLoadAndDisable(configToSet, oldData) + } + setConfigImport({ + ...configToSet, + projectStarted: hasProjectStarted(), + }) + lastObjectRef.current = oldData + }, [oldData, configImport, setConfigImport, hasProjectStarted]) const handleRowSelected = (whatsChanging) => { if (!open) return @@ -126,13 +70,6 @@ export default ({ open, onClose, onAddSamples }) => { } } - const loadAssetsOrAnnotations = () => { - setConfigImport({ - ...configImport, - loadAssetsIsSelected: !configImport.loadAssetsIsSelected, - }) - } - useEffect(() => { if (!open) return if (!authConfig) return @@ -202,25 +139,39 @@ export default ({ open, onClose, onAddSamples }) => { setProject() // eslint-disable-next-line }, [projectToFetch, open]) + const createJsonFromAsset = async () => { - if ( - !configImport.typeOfFileToLoad || - configImport.typeOfFileToLoad === "None" - ) - return var jsons = await Promise.all( projectToFetch.rowAssetsUrl.map(async (obj) => { - var url = await dm.getDataUrl(obj.split("/assets/")[1]) - var json = setUrl(url, configImport) - if (json) json = setIn(json, ["_id"], obj.split("/assets/")[1]) - return json + return await createJsonFromUrlAWS( + dm.projectName, + obj.split("/assets/")[1] + ) }) ) onAddSamples(jsons) } + const createJsonFromUrlAWS = async (projectName, imageName) => { + var url = await dm.getAssetUrl(imageName, projectName) + var json = setUrl(url, configImport) + if (json) json = setIn(json, ["_id"], imageName) + if (json) json = setIn(json, ["source"], projectName) + return json + } + const createJsonFromAnnotation = async () => { - var jsons = await dm.readJSONAllSample(projectToFetch.rowAnnotationsUrl) + var jsons = await dm.readJSONAllSamples(projectToFetch.rowAnnotationsUrl) + var sources = getSources(jsons) + if (sources) { + jsons = await Promise.all( + jsons.map(async (json) => { + if (json.source) + json = await createJsonFromUrlAWS(json.source, json._id) + return json + }) + ) + } onAddSamples(jsons) } @@ -244,55 +195,28 @@ export default ({ open, onClose, onAddSamples }) => { onClick: () => { handleAddSample() }, + disabled: configImport.contentDialogBoxIsSetting, }, ]} > - {configImport.loadAssetsIsSelected ? ( - - ) : ( - - )} - {configImport.loadAssetsIsSelected ? ( - - ) : ( - - )} - { - setConfigImport({ - ...configImport, - contentDialogBoxIsSetting: !configImport.contentDialogBoxIsSetting, - }) - }} - > - {configImport.contentDialogBoxIsSetting ? ( - - ) : ( - - )} - + - {!configImport.contentDialogBoxIsSetting ? ( !isEmpty(projects) && ( } + expandableRowsComponent={ + + } selectableRows selectableRowsHighlight selectableRowsNoSelectAll @@ -312,88 +236,10 @@ export default ({ open, onClose, onAddSamples }) => { ) ) : ( - {!configImport.loadAssetsIsSelected ? ( - - Annotation processing - { - setConfigImport({ - ...configImport, - annotationToKeep: event.target.value, - }) - }} - > - } - label="Keep both annotations" - checked={configImport.annotationToKeep === "both"} - /> - } - label="Keep incoming annotations" - checked={configImport.annotationToKeep === "incoming"} - /> - } - label="Keep current annotations" - checked={configImport.annotationToKeep === "current"} - /> - - - ) : ( - - Choose file type - { - setConfigImport({ - ...configImport, - typeOfFileToLoad: event.target.value, - }) - }} - > - } - label="Load image file" - disabled={configImport.typeOfFileToDisable.Image} - checked={configImport.typeOfFileToLoad === "Image"} - /> - } - label="Load video file" - disabled={configImport.typeOfFileToDisable.Video} - checked={configImport.typeOfFileToLoad === "Video"} - /> - } - label="Load audio file" - disabled={configImport.typeOfFileToDisable.Audio} - checked={configImport.typeOfFileToLoad === "Audio"} - /> - } - label="Load PDF file" - disabled={configImport.typeOfFileToDisable.PDF} - checked={configImport.typeOfFileToLoad === "PDF"} - /> - } - label="Load texte file" - disabled={configImport.typeOfFileToDisable.Texte} - checked={configImport.typeOfFileToLoad === "Texte"} - /> - - - )} + )} diff --git a/src/components/ImportFromCognitoS3Dialog/interface-setting-import.js b/src/components/ImportFromCognitoS3Dialog/interface-setting-import.js new file mode 100644 index 00000000..7c5bedb4 --- /dev/null +++ b/src/components/ImportFromCognitoS3Dialog/interface-setting-import.js @@ -0,0 +1,99 @@ +import React from "react" +import { + Radio, + RadioGroup, + FormControl, + FormControlLabel, + FormLabel, +} from "@material-ui/core/" +const setting = ({ configImport, setConfigImport }) => { + return ( + + + {!configImport.loadAssetsIsSelected ? ( + + Annotation processing + { + setConfigImport({ + ...configImport, + annotationToKeep: event.target.value, + }) + }} + > + } + label="Keep both annotations" + checked={configImport.annotationToKeep === "both"} + /> + } + label="Keep incoming annotations" + checked={configImport.annotationToKeep === "incoming"} + /> + } + label="Keep current annotations" + checked={configImport.annotationToKeep === "current"} + /> + + + ) : ( + + Choose file type + { + setConfigImport({ + ...configImport, + typeOfFileToLoad: event.target.value, + }) + }} + > + } + label="Load image file" + disabled={configImport.typeOfFileToDisable.Image} + checked={configImport.typeOfFileToLoad === "Image"} + /> + } + label="Load video file" + disabled={configImport.typeOfFileToDisable.Video} + checked={configImport.typeOfFileToLoad === "Video"} + /> + } + label="Load audio file" + disabled={configImport.typeOfFileToDisable.Audio} + checked={configImport.typeOfFileToLoad === "Audio"} + /> + } + label="Load PDF file" + disabled={configImport.typeOfFileToDisable.PDF} + checked={configImport.typeOfFileToLoad === "PDF"} + /> + } + label="Load texte file" + disabled={configImport.typeOfFileToDisable.Texte} + checked={configImport.typeOfFileToLoad === "Texte"} + /> + + + )} + + + ) +} +export default setting diff --git a/src/components/ImportFromCognitoS3Dialog/set-annotation-from-aws.js b/src/components/ImportFromCognitoS3Dialog/set-annotation-from-aws.js deleted file mode 100644 index 5d5d9630..00000000 --- a/src/components/ImportFromCognitoS3Dialog/set-annotation-from-aws.js +++ /dev/null @@ -1,21 +0,0 @@ -import { setIn } from "seamless-immutable" -import * as datasetHelper from "../../utils//dataset-helper" -import isEmpty from "lodash/isEmpty" -export default (file, json, configImport) => { - var contentOldFile = file.content - contentOldFile = setIn( - contentOldFile, - ["samples"], - datasetHelper.concatSample( - file.content.samples, - json.content.samples, - configImport.annotationToKeep - ) - ) - - contentOldFile = setIn(contentOldFile, ["interface"], json.content.interface) - file = setIn(file, ["content"], contentOldFile) - if (isEmpty(file.fileName) || file.fileName === "unnamed") - file = setIn(file, ["fileName"], json.fileName) - return file -} diff --git a/src/components/ImportFromCognitoS3Dialog/set-button-add-sample-name.js b/src/components/ImportFromCognitoS3Dialog/set-button-add-sample-name.js deleted file mode 100644 index f7ed5e89..00000000 --- a/src/components/ImportFromCognitoS3Dialog/set-button-add-sample-name.js +++ /dev/null @@ -1,30 +0,0 @@ -import RecognizeFileExtension from "../../utils/RecognizeFileExtension" -import isEmpty from "lodash/isEmpty" -export default (loadProjectIsSelected, typeOfFileToLoad, dataForTable) => { - var numberOfSamples = 0 - var textToSet = "" - var folderName - if (!isEmpty(dataForTable)) { - for (var i = 0; i < dataForTable.length; i++) { - if (dataForTable[i].isSelected) { - folderName = dataForTable[i].folder - if (!isEmpty(dataForTable[i].rowData)) { - for (var y = 0; y < dataForTable[i].rowData.length; y++) { - if ( - RecognizeFileExtension(dataForTable[i].rowData[y].data) === - typeOfFileToLoad - ) { - numberOfSamples++ - } - } - } - } - } - if (loadProjectIsSelected) { - textToSet = "Load " + folderName - } else { - textToSet = "Add " + numberOfSamples + " " + typeOfFileToLoad - } - } - return textToSet -} diff --git a/src/components/ImportFromCognitoS3Dialog/set-type-of-file-to-load-and-disable.js b/src/components/ImportFromCognitoS3Dialog/set-type-of-file-to-load-and-disable.js new file mode 100644 index 00000000..c3d5da35 --- /dev/null +++ b/src/components/ImportFromCognitoS3Dialog/set-type-of-file-to-load-and-disable.js @@ -0,0 +1,38 @@ +import isEmpty from "lodash/isEmpty" +import checkInterfaceAndAssets from "./check-interface-and-sample-type" +export default (configImport, dataset) => { + return { + ...configImport, + typeOfFileToLoad: + !isEmpty(configImport) && + !isEmpty(configImport.typeOfFileToLoad) && + checkInterfaceAndAssets([configImport.typeOfFileToLoad, "Empty"], dataset) + ? configImport.typeOfFileToLoad + : checkInterfaceAndAssets(["Image", "Empty"], dataset) + ? "Image" + : checkInterfaceAndAssets(["Video", "Empty"], dataset) + ? "Video" + : checkInterfaceAndAssets(["Audio", "Empty"], dataset) + ? "Audio" + : checkInterfaceAndAssets(["PDF", "Empty"], dataset) + ? "PDF" + : checkInterfaceAndAssets(["Texte", "Empty"], dataset) + ? "Texte" + : "None", + typeOfFileToDisable: { + Image: checkInterfaceAndAssets(["Image", "Empty"], dataset) + ? false + : true, + Video: checkInterfaceAndAssets(["Video", "Empty"], dataset) + ? false + : true, + Audio: checkInterfaceAndAssets(["Audio", "Empty"], dataset) + ? false + : true, + PDF: checkInterfaceAndAssets(["PDF", "Empty"], dataset) ? false : true, + Texte: checkInterfaceAndAssets(["Texte", "Empty"], dataset) + ? false + : true, + }, + } +} diff --git a/src/components/ImportFromCognitoS3Dialog/set-url.js b/src/components/ImportFromCognitoS3Dialog/set-url.js index 6e66d52e..5a9f22fc 100644 --- a/src/components/ImportFromCognitoS3Dialog/set-url.js +++ b/src/components/ImportFromCognitoS3Dialog/set-url.js @@ -9,6 +9,9 @@ const setUrl = (result, configImport) => { return { audioUrl: `${result}` } } else if (RecognizeFileExtension(result) === "PDF") { return { pdfUrl: `${result}` } + } else if (RecognizeFileExtension(result) === "Texte") { + //var texte = await fetchTextInFile(result) + return { document: `Is not supported` /*${texte}`*/ } } } else { if ( diff --git a/src/components/ImportFromCognitoS3Dialog/table-expanded-row.js b/src/components/ImportFromCognitoS3Dialog/table-expanded-row.js new file mode 100644 index 00000000..7d3ff395 --- /dev/null +++ b/src/components/ImportFromCognitoS3Dialog/table-expanded-row.js @@ -0,0 +1,72 @@ +import React from "react" +import DataTable from "react-data-table-component" + +const expandedAssetsColumns = [ + { name: "Assets", selector: "assets", sortable: true }, + { name: "Last Modified", selector: "lastModified", sortable: true }, +] + +const expandedAnnotationsColumns = [ + { name: "Annotations", selector: "annotation" }, + { name: "Last Modified", selector: "lastModified", sortable: true }, +] + +const customStyles = { + headCells: { + style: { + paddingLeft: "10px", + "font-weight": "bold", + }, + }, + cells: { + style: { + paddingLeft: "25px", + }, + }, +} + +const ExpandedRow = ({ data, loadAssetIsSelected }) => { + const { rowAssets, rowAnnotations } = data + return ( + <> + {!loadAssetIsSelected ? ( + 10} + paginationPerPage={10} + paginationRowsPerPageOptions={[10, 20, 25, 50, 100, 200]} + customStyles={customStyles} + /> + ) : ( + 10} + paginationPerPage={10} + paginationRowsPerPageOptions={[10, 20, 25, 50, 100, 200]} + customStyles={customStyles} + /> + )} + + ) +} +export default ExpandedRow diff --git a/src/components/ImportPage/index.js b/src/components/ImportPage/index.js index 1f578c44..994ae55a 100644 --- a/src/components/ImportPage/index.js +++ b/src/components/ImportPage/index.js @@ -258,19 +258,16 @@ export default ({ isDesktop, authConfig, user }) => { > {t("export-to-cognito-s3")} - - { - - } +