diff --git a/pephub/routers/api/v1/project.py b/pephub/routers/api/v1/project.py index 5947993b..6862dac7 100644 --- a/pephub/routers/api/v1/project.py +++ b/pephub/routers/api/v1/project.py @@ -53,9 +53,12 @@ ProjectHistoryResponse, SamplesResponseModel, ConfigResponseModel, + StandardizerResponse, ) from .helpers import verify_updated_project +from attribute_standardizer.attr_standardizer_class import AttrStandardizer + _LOGGER = logging.getLogger(__name__) load_dotenv() @@ -1140,3 +1143,42 @@ def delete_full_history( status_code=400, detail="Could not delete history. Server error.", ) + + +@project.post( + "/standardize", + summary="Standardize PEP metadata column headers", + response_model=StandardizerResponse, +) +async def get_standardized_cols( + pep: peppy.Project = Depends(get_project), + schema: str = "", +): + """ + Standardize PEP metadata column headers using BEDmess. + + :param namespace: pep: PEP string to be standardized + :param schema: Schema for AttrStandardizer + + :return dict: Standardized results + """ + + if schema == "": + raise HTTPException( + code=500, + detail="Schema is required! Available schemas are ENCODE and Fairtracks", + ) + return {} + + prj = peppy.Project.from_dict(pep) + model = AttrStandardizer(schema) + + try: + results = model.standardize(pep=prj) + except Exception: + raise HTTPException( + code=400, + detail=f"Error standardizing PEP.", + ) + + return StandardizerResponse(results=results) diff --git a/pephub/routers/models.py b/pephub/routers/models.py index d9907abe..12939b5b 100644 --- a/pephub/routers/models.py +++ b/pephub/routers/models.py @@ -159,3 +159,7 @@ class SchemaGetResponse(BaseModel): description: Optional[str] = None last_update_date: str = "" submission_date: str = "" + + +class StandardizerResponse(BaseModel): + results: dict = {} diff --git a/web/package.json b/web/package.json index 9dbf595d..30c06791 100644 --- a/web/package.json +++ b/web/package.json @@ -22,7 +22,7 @@ "@typescript-eslint/eslint-plugin": "^5.56.0", "@typescript-eslint/parser": "^5.56.0", "axios": "^1.3.4", - "bootstrap": "^5.2.3", + "bootstrap": "^5.3.3", "bootstrap-icons": "^1.10.3", "eslint": "^8.36.0", "eslint-config-prettier": "^8.8.0", @@ -57,6 +57,7 @@ "devDependencies": { "@mdx-js/rollup": "^3.0.0", "@trivago/prettier-plugin-sort-imports": "^4.1.1", + "@types/js-yaml": "^4.0.9", "@types/react": "^18.0.28", "@types/react-dom": "^18.0.11", "@vitejs/plugin-react-swc": "^3.0.0", diff --git a/web/src/api/project.ts b/web/src/api/project.ts index 18927cda..540c1dde 100644 --- a/web/src/api/project.ts +++ b/web/src/api/project.ts @@ -85,6 +85,14 @@ export type RestoreProjectFromHistoryResponse = { registry: string; }; +export type StandardizeColsResponse = { + results: { + [key: string]: { + [key: string]: number; + }; + }; +}; + export const getProject = ( namespace: string, projectName: string, @@ -404,3 +412,16 @@ export const restoreProjectFromHistory = ( const url = `${API_BASE}/projects/${namespace}/${name}/history/${historyId}/restore?tag=${tag}`; return axios.post(url, {}, { headers: { Authorization: `Bearer ${jwt}` } }); }; + +export const getStandardizedCols = ( + namespace: string, + name: string, + tag: string, + jwt: string | null, + schema: string, +) => { + const url = `${API_BASE}/projects/${namespace}/${name}/standardize?schema=${schema}&tag=${tag}`; + return axios + .post(url, { headers: { Authorization: `Bearer ${jwt || 'NO_AUTHORIZATION'}` } }) + .then((res) => res.data); +}; diff --git a/web/src/components/forms/blank-project-form.tsx b/web/src/components/forms/blank-project-form.tsx index b72f437f..dd490375 100644 --- a/web/src/components/forms/blank-project-form.tsx +++ b/web/src/components/forms/blank-project-form.tsx @@ -50,7 +50,7 @@ const CombinedErrorMessage = (props: CombinedErrorMessageProps) => { } if (nameError || tagError) { - return

{msg}

; + return

{msg}

; } return null; @@ -106,8 +106,8 @@ sample_table: samples.csv const { isPending: isSubmitting, submit } = useBlankProjectFormMutation(namespace); return ( -
-
+ +
-
-
- - - +
+ + +
-
+
:
- +
-

{message}

} /> + + - +
- + -
+
-
+
{ setValue('config', data); }} - height={300} + height={295} />
+

+ * Namespace and Project Name are required. A tag value of "default" will be supplied if the Tag input is left empty. +

+
+ ), { + duration: 16000, + position: 'top-right', + }); return; } }} @@ -241,7 +272,7 @@ sample_table: samples.csv
*/} -
- - +
+ +
@@ -94,9 +124,15 @@ export const CreateSchemaForm = (props: Props) => { // dont allow any whitespace {...register('name', { required: true, + required: { + value: true, + message: "empty", + }, pattern: { value: /^\S+$/, message: 'No spaces allowed.', + value: /^[a-zA-Z0-9_-]+$/, + message: "invalid", }, })} id="schema-name" @@ -106,18 +142,15 @@ export const CreateSchemaForm = (props: Props) => { />
-
- - - + {/* Add a dropdown here */}
= ({ onHide, defaultNamespace }) => { })}
) : null} +

+ * Namespace and Project Name are required. A tag value of "default" will be supplied if the Tag input is left empty. +

+ +
+ +
- - - - ); }; diff --git a/web/src/components/modals/standardize-metadata.tsx b/web/src/components/modals/standardize-metadata.tsx new file mode 100644 index 00000000..d3c8998c --- /dev/null +++ b/web/src/components/modals/standardize-metadata.tsx @@ -0,0 +1,383 @@ +import { HotTable } from '@handsontable/react'; +import Handsontable from 'handsontable'; +import 'handsontable/dist/handsontable.full.css'; +import React, { FormEvent, useState, useEffect, useCallback, useMemo } from 'react'; +import { Modal } from 'react-bootstrap'; +import ReactSelect from 'react-select'; + +import { useEditProjectMetaMutation } from '../../hooks/mutations/useEditProjectMetaMutation'; +import { useProjectAnnotation } from '../../hooks/queries/useProjectAnnotation'; +import { useSampleTable } from '../../hooks/queries/useSampleTable'; +import { useStandardize } from '../../hooks/queries/useStandardize'; +import { arraysToSampleList, sampleListToArrays } from '../../utils/sample-table'; +import { ProjectMetaEditForm } from '../forms/edit-project-meta'; +import { LoadingSpinner } from '../spinners/loading-spinner' +import { formatToPercentage } from '../../utils/etc' + +type Props = { + namespace: string; + project: string; + tag: string; + show: boolean; + onHide: () => void; + sampleTable: ReturnType['data']; + sampleTableIndex: string; + newSamples: any[][]; + setNewSamples: (samples: any[][]) => void; + resetStandardizedData: boolean; + setResetStandardizedData: (boolean) => void; +}; + +type TabDataRow = string[]; +type TabData = TabDataRow[]; +type SelectedValues = Record; +type AvailableSchemas = 'ENCODE' | 'FAIRTRACKS'; + +type StandardizedData = Record>; + +export const StandardizeMetadataModal = (props: Props) => { + const { namespace, project, tag, show, onHide, sampleTable, sampleTableIndex, newSamples, setNewSamples, resetStandardizedData, setResetStandardizedData } = props; + + const tabDataRaw = newSamples; + const tabData = tabDataRaw[0] + .map((_, colIndex) => tabDataRaw.map((row) => row[colIndex])) + .reduce((obj, row) => { + const [key, ...values] = row; + obj[key as string] = values; + return obj; + }, {} as Record); + + const originalCols: string[] = useMemo(() => Object.keys(tabData), []); + const newCols: string[] = Object.keys(tabData); + + const [selectedOption, setSelectedOption] = useState<{ value: AvailableSchemas; label: string } | null>(null); + const [selectedValues, setSelectedValues] = useState({}); + const [whereDuplicates, setWhereDuplicates] = useState(null); + + const { + isFetching, + isError, + error, + data: rawData, + refetch: standardize, + } = useStandardize(namespace, project, tag, selectedOption?.value); + + const data = resetStandardizedData ? rawData : null; + + const standardizedData = data?.results as StandardizedData | undefined; + + const getOriginalColValues = (key: string): string | undefined => { + const oldColIndex = originalCols.indexOf(key); + if (oldColIndex !== -1 && oldColIndex < newCols.length) { + return newCols[oldColIndex]; + } + return undefined; + }; + + const handleRadioChange = (key: string, value: string | null) => { + setSelectedValues((prev) => { + const newValues = { + ...prev, + [key]: value === null ? key : value, + }; + setWhereDuplicates(checkForDuplicates(newValues)); + return newValues; + }); + }; + + const checkForDuplicates = (values: SelectedValues): number[] | null => { + const valueArray = Object.values(values); + const duplicates: Record = {}; + const result: number[] = []; + + for (let i = 0; i < valueArray.length; i++) { + const value = valueArray[i]; + if (value in duplicates) { + if (duplicates[value].length === 1) { + result.push(duplicates[value][0]); + } + result.push(i); + } else { + duplicates[value] = [i]; + } + } + + return result.length > 0 ? result : null; + }; + + const getDefaultSelections = (standardizedData: StandardizedData): SelectedValues => { + const defaultSelections: SelectedValues = {}; + Object.keys(standardizedData).forEach((key) => { + defaultSelections[key] = key; + }); + return defaultSelections; + }; + + const updateTabDataRaw = (tabDataRaw: TabData, selectedValues: SelectedValues): TabData => { + if (tabDataRaw.length === 0) return tabDataRaw; + + const updatedTabDataRaw: TabData = [tabDataRaw[0].slice(), ...tabDataRaw.slice(1)]; + + Object.entries(selectedValues).forEach(([key, value]) => { + let columnIndex = updatedTabDataRaw[0].indexOf(key); + if (columnIndex === -1) { + const originalValue = getOriginalColValues(key) + if (originalValue) { + columnIndex = updatedTabDataRaw[0].indexOf(originalValue) + } + } + if (columnIndex !== -1) { + updatedTabDataRaw[0][columnIndex] = value; + } + }); + + return updatedTabDataRaw; + }; + + const handleSubmit = async (event: FormEvent) => { + event.preventDefault(); + setResetStandardizedData(true); + await standardize(); + }; + + const prepareHandsontableData = useCallback((key: string) => { + const selectedValue = selectedValues[key] || ''; + const originalValue = getOriginalColValues(key) + let topValues: string[][] = [] + if (originalValue) { + topValues = tabData[key]?.slice(0, 6).map((item) => [item]) || tabData[originalValue]?.slice(0, 6).map((item) => [item]) || []; + } + const emptyRows = Array(Math.max(0, 6 - topValues.length)).fill(['']); + + return [[selectedValue], ...topValues, ...emptyRows]; + }, [selectedValues, tabData]); + + useEffect(() => { + if (standardizedData) { + const defaultSelections = getDefaultSelections(standardizedData); + setWhereDuplicates(checkForDuplicates(defaultSelections)); + setSelectedValues(defaultSelections); + } + }, [standardizedData]); + + useEffect(() => { + if (!resetStandardizedData) { + setSelectedOption(null); + setSelectedValues({}); + setWhereDuplicates(null); + } + }, [resetStandardizedData]); + + + return ( + + +
+

Metadata Standardizer

+ +

+ Use the metadata standardizer powered by BEDmess to bring consistency across metadata columns in all of + your projects. After choosing a standardizer schema below, compare predicted suggestions (confidence indicated in parenthesis) and choose whether + to keep or discard them. Column contents are not modified by the standardizer. + After accepting the changes, save your project for them to take effect. +

+
+ +
+
+

Standardizer Schema

+ +
+
+
+ ({ + ...provided, + borderRadius: '.333333em', // Left radii set to 0, right radii kept at 4px + }), + }} + options={[ + // @ts-ignore + { value: 'ENCODE', label: 'ENCODE' }, + // @ts-ignore + { value: 'FAIRTRACKS', label: 'Fairtracks' }, + ]} + defaultValue={selectedOption} + value={selectedOption} + onChange={(selectedOption) => { + if (selectedOption === null) { + return; + } + setSelectedOption(selectedOption); + }} + /> +
+
+ +
+
+
+
+
+ + {standardizedData && !isFetching ? ( + <> +
+ +
+
+

Original Column

+
+
+

Predicted Column Header

+
+
+ +
+ {Object.keys(standardizedData).map((key, index) => ( +
+
+ {key === sampleTableIndex ?

SampleTableIndex must also be updated in project config!

: null} +
+
+ +
+
+
+
+
+ handleRadioChange(key, null)} + /> + + + {Object.entries(standardizedData[key]).map(([subKey, value], index, array) => ( + + handleRadioChange(key, subKey)} + /> + + + ))} +
+
+
+
+
+
+ ))} +
+ + ) : isFetching ? + +
+ +
+ : + null + } + +
+
+ {whereDuplicates !== null && ( +
Warning: ensure no duplicate column names have been selected.
+ )} + + +
+
+ + + ); +}; diff --git a/web/src/components/modals/validation-result.tsx b/web/src/components/modals/validation-result.tsx index 8ee0859c..92ac581b 100644 --- a/web/src/components/modals/validation-result.tsx +++ b/web/src/components/modals/validation-result.tsx @@ -11,7 +11,7 @@ type Props = { show: boolean; onHide: () => void; validationResult: ReturnType['data']; - currentSchema: string; + currentSchema: string | undefined; }; type FormProps = { @@ -19,7 +19,7 @@ type FormProps = { }; export const ValidationResultModal = (props: Props) => { - const { show, onHide, validationResult } = props; + const { show, onHide, validationResult, currentSchema } = props; const { namespace, projectName, tag } = useProjectPage(); @@ -33,9 +33,15 @@ export const ValidationResultModal = (props: Props) => { const newSchema = updateForm.watch('schema'); const handleSubmit = () => { - submit({ - newSchema, - }); + if (newSchema === '') { + submit({ + newSchema: undefined, + }); + } else { + submit({ + newSchema: newSchema, + }); + } }; return ( @@ -49,33 +55,51 @@ export const ValidationResultModal = (props: Props) => { >

- {validationResult?.valid ? ( - - - Validation Passed - + {currentSchema ? ( + <> + {validationResult?.valid ? ( + + + Validation Passed + + ) : ( + + + Validation Failed + + )} + ) : ( - - - Validation Failed + + Select a Schema )}

- {validationResult?.valid ? ( -

Your PEP is valid against the schema.

- ) : ( - -

You PEP is invalid against the schema.

-

Validation result:

- 
-              {JSON.stringify(validationResult, null, 2)}
-
- + {currentSchema ? ( + <> + {validationResult?.valid ? ( +

Your PEP is valid against the schema.

+ ) : ( + +

You PEP is invalid against the schema.

+

Validation result:

+ 
+                {JSON.stringify(validationResult, null, 2)}
+
+ + )} + + ) : ( null )} -
- + + + {currentSchema ? ( + + ) : ( + + )}
void; setShowDeletePEPModal: (show: boolean) => void; setShowForkPEPModal: (show: boolean) => void; + starNumber: number; } export const ProjectCardDropdown: FC = (props) => { - const { project, isStarred, copied, setCopied, setShowDeletePEPModal, setShowForkPEPModal } = props; + const { project, isStarred, copied, setCopied, setShowDeletePEPModal, setShowForkPEPModal, starNumber } = props; const { user } = useSession(); const { isPending: isAddingStar, addStar } = useAddStar(user?.login); const { isPending: isRemovingStar, removeStar } = useRemoveStar(user?.login); + const [localStarred, setLocalStarred] = useState(false); + return ( - - + + View diff --git a/web/src/components/namespace/project-cards/project-card.tsx b/web/src/components/namespace/project-cards/project-card.tsx index 179cea3c..6cc45507 100644 --- a/web/src/components/namespace/project-cards/project-card.tsx +++ b/web/src/components/namespace/project-cards/project-card.tsx @@ -30,12 +30,11 @@ export const ProjectCard: FC = ({ project }) => { return (
- + {project.namespace}/{project.name}:{project.tag} {project.is_private ? ( @@ -63,59 +62,46 @@ export const ProjectCard: FC = ({ project }) => { setCopied={setCopied} setShowDeletePEPModal={setShowDeletePEPModal} setShowForkPEPModal={setShowForkPEPModal} + starNumber={project.stars_number || 0} />
-
-
-
- - {project.stars_number || 0} -
-
- - {project.number_of_samples} -
-
- - {project.pep_schema || 'No schema'} -
-
-
- {project.description ? ( - {project.description} - ) : ( - - No description - - )} -
+
+ {project.description ? ( + {project.description} + ) : ( + + No description + + )}
-
-
- - - - Created: - {dateStringToDateTime(project.submission_date)} - - Updated: - {dateStringToDateTime(project.last_update_date)} - - - - {project?.forked_from && ( - - - - Forked from - + +
+
+ + Created: + {dateStringToDateTime(project.submission_date)} + + + Updated: + {dateStringToDateTime(project.last_update_date)} + + {project?.forked_from && ( +
+ + Forked from + -
+
Search diff --git a/web/src/components/namespace/view-selector.tsx b/web/src/components/namespace/view-selector.tsx index 790e6403..6a09bec9 100644 --- a/web/src/components/namespace/view-selector.tsx +++ b/web/src/components/namespace/view-selector.tsx @@ -27,7 +27,7 @@ export const NamespaceViewSelector: FC = (props) => { }; return ( -
+