Merge pull request #388 from pepkit/dev

Release 0.14.0

README.md

@@ -19,6 +19,6 @@
 
 **Source Code**: <a href="https://github.com/pepkit/pephub" target="_blank">https://github.com/pepkit/pephub</a>
 
----
+--  -
 
 

pephub/_version.py

@@ -1 +1 @@
-__version__ = "0.13.0"
+__version__ = "0.14.0"

pephub/const.py

@@ -138,3 +138,5 @@
 DEFAULT_QDRANT_SCORE_THRESHOLD = (
     0.72  # empirical value, highly dependent on the model used
 )
+
+ARCHIVE_URL_PATH = "https://cloud2.databio.org/pephub/"

pephub/dependencies.py

@@ -149,8 +149,10 @@ def read_authorization_header(authorization: str = Header(None)) -> Union[dict,
     except jwt.exceptions.DecodeError as e:
         _LOGGER_PEPHUB.error(e)
         return None
-    except jwt.exceptions.ExpiredSignatureError:
-        raise HTTPException(status_code=401, detail="JWT has expired")
+    except jwt.exceptions.ExpiredSignatureError as e:
+        # raise HTTPException(status_code=401, detail="JWT has expired")
+        _LOGGER_PEPHUB.error(e)
+        return None
     return session_info
 
 

pephub/helpers.py

@@ -96,6 +96,29 @@ def zip_conv_result(conv_result: dict, filename: str = "project.zip") -> Respons
     return resp
 
 
+def download_yaml(content: dict, file_name: str = "unnamed.yaml") -> Response:
+    """
+    Convert json/dict to downloading io format
+
+    :param content: content of the file
+    :param file_name: name of the file
+    return Response: response object
+    """
+
+    yaml_string = yaml.dump(content)
+
+    yaml_bytes = io.BytesIO()
+    yaml_bytes.write(yaml_string.encode("utf-8"))
+    yaml_bytes.seek(0)  # Move the pointer to the start of the stream
+
+    # Create a streaming response with the YAML data
+    return Response(
+        yaml_bytes.getvalue(),
+        media_type="application/x-yaml",
+        headers={"Content-Disposition": f"attachment; filename={file_name}"},
+    )
+
+
 def build_authorization_url(
     client_id: str,
     redirect_uri: str,

pephub/routers/api/v1/helpers.py

@@ -1,30 +1,48 @@
 import logging
 
 import eido
+from eido.validation import validate_config
+from eido.exceptions import EidoValidationError
 import peppy
 import yaml
 from fastapi.exceptions import HTTPException
 from peppy import Project
 from peppy.const import (
     CONFIG_KEY,
-    SAMPLE_NAME_ATTR,
     SAMPLE_RAW_DICT_KEY,
-    SAMPLE_TABLE_INDEX_KEY,
     SUBSAMPLE_RAW_LIST_KEY,
 )
+from ....dependencies import (
+    get_db,
+)
 
 _LOGGER = logging.getLogger(__name__)
+DEFAULT_SCHEMA_NAMESPACE = "databio"
+DEFAULT_SCHEMA_NAME = "pep-2.1.0"
 
 
 async def verify_updated_project(updated_project) -> peppy.Project:
     new_raw_project = {}
 
+    agent = get_db()
+    default_schema = agent.schema.get(
+        namespace=DEFAULT_SCHEMA_NAMESPACE, name=DEFAULT_SCHEMA_NAME
+    )
+
     if not updated_project.sample_table or not updated_project.project_config_yaml:
         raise HTTPException(
             status_code=400,
             detail="Please provide a sample table and project config yaml to update project",
         )
-
+    try:
+        validate_config(
+            yaml.safe_load(updated_project.project_config_yaml), default_schema
+        )
+    except EidoValidationError as e:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Config structure error: {', '.join(list(e.errors_by_type.keys()))}. Please check schema definition and try again.",
+        )
     # sample table update
     new_raw_project[SAMPLE_RAW_DICT_KEY] = updated_project.sample_table
 
@@ -64,7 +82,7 @@ async def verify_updated_project(updated_project) -> peppy.Project:
 
     try:
         # validate project (it will also validate samples)
-        eido.validate_project(new_project, "http://schema.databio.org/pep/2.1.0.yaml")
+        eido.validate_project(new_project, default_schema)
     except Exception as _:
         raise HTTPException(
             status_code=400,

pephub/routers/api/v1/namespace.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 from typing import List, Literal, Optional, Union
+import os
 
 import peppy
 from dotenv import load_dotenv
@@ -20,13 +21,15 @@
     ListOfNamespaceInfo,
     Namespace,
     NamespaceStats,
+    TarNamespaceModelReturn,
 )
 from peppy import Project
 from peppy.const import DESC_KEY, NAME_KEY
 from typing_extensions import Annotated
 
 from ....const import (
     DEFAULT_TAG,
+    ARCHIVE_URL_PATH,
 )
 from ....dependencies import (
     get_db,
@@ -39,6 +42,8 @@
 from ....helpers import parse_user_file_upload, split_upload_files_on_init_file
 from ...models import FavoriteRequest, ProjectJsonRequest, ProjectRawModel
 
+from bedms.const import AVAILABLE_SCHEMAS
+
 load_dotenv()
 
 namespaces = APIRouter(prefix="/api/v1/namespaces", tags=["namespace"])
@@ -77,7 +82,9 @@ async def get_namespace_projects(
     offset: int = 0,
     query: str = None,
     admin_list: List[str] = Depends(get_namespace_access_list),
-    order_by: str = "update_date",
+    order_by: Optional[
+        Literal["update_date", "name", "submission_date", "stars"]
+    ] = "update_date",
     order_desc: bool = False,
     filter_by: Annotated[
         Optional[Literal["submission_date", "last_update_date"]],
@@ -175,7 +182,6 @@ async def create_pep(
             p = Project(f"{dirpath}/{init_file.filename}")
             p.name = name
             p.description = description
-            p.pep_schema = pep_schema
             try:
                 agent.project.create(
                     p,
@@ -184,7 +190,7 @@ async def create_pep(
                     tag=tag,
                     description=description,
                     is_private=is_private,
-                    pep_schema=pep_schema,
+                    pep_schema=pep_schema or "databio/pep-2.1.0",
                 )
             except ProjectUniqueNameError:
                 raise HTTPException(
@@ -228,7 +234,7 @@ async def upload_raw_pep(
         is_private = project_from_json.is_private
         tag = project_from_json.tag
         overwrite = project_from_json.overwrite
-        pep_schema = project_from_json.pep_schema
+        pep_schema = project_from_json.pep_schema or "databio/pep-2.1.0"
         pop = project_from_json.pop or False
         if hasattr(project_from_json, NAME_KEY):
             if project_from_json.name:
@@ -436,3 +442,27 @@ def remove_user(
         },
         status_code=202,
     )
+
+
+@namespace.get(
+    "/archive",
+    summary="Get metadata of all archived files of all projects in the namespace",
+    response_model=TarNamespaceModelReturn,
+)
+async def get_archive(namespace: str, agent: PEPDatabaseAgent = Depends(get_db)):
+
+    result = agent.namespace.get_tar_info(namespace)
+
+    for item in result.results:
+        item.file_path = os.path.join(ARCHIVE_URL_PATH, item.file_path)
+
+    return result
+
+
+@namespace.get(
+    "/standardizer-schemas",
+    summary="Get all available schemas from BEDMS",
+)
+async def get_schemas(namespace: str, agent: PEPDatabaseAgent = Depends(get_db)):
+
+    return AVAILABLE_SCHEMAS

pephub/routers/api/v1/project.py

@@ -53,9 +53,12 @@
     ProjectHistoryResponse,
     SamplesResponseModel,
     ConfigResponseModel,
+    StandardizerResponse,
 )
 from .helpers import verify_updated_project
 
+from bedms import AttrStandardizer
+
 _LOGGER = logging.getLogger(__name__)
 
 load_dotenv()
@@ -159,7 +162,10 @@ async def update_pep(
     )
     if new_project:
         update_dict.update(project=new_project)
-        new_name = new_project.name or project
+        try:
+            new_name = new_project.name or project
+        except NotImplementedError:
+            new_name = new_project.name = project
     else:
         new_name = project
     agent.project.update(
@@ -171,7 +177,6 @@ async def update_pep(
     )
 
     # fetch latest name and tag
-    new_name = updated_project.name or new_name
     tag = updated_project.tag or tag
 
     return JSONResponse(
@@ -1138,3 +1143,42 @@ def delete_full_history(
             status_code=400,
             detail="Could not delete history. Server error.",
         )
+
+
+@project.post(
+    "/standardize",
+    summary="Standardize PEP metadata column headers",
+    response_model=StandardizerResponse,
+)
+async def get_standardized_cols(
+    pep: peppy.Project = Depends(get_project),
+    schema: str = "",
+):
+    """
+    Standardize PEP metadata column headers using BEDmess.
+
+    :param namespace: pep: PEP string to be standardized
+    :param schema: Schema for AttrStandardizer
+
+    :return dict: Standardized results
+    """
+
+    if schema == "":
+        raise HTTPException(
+            code=500,
+            detail="Schema is required! Available schemas are ENCODE and Fairtracks",
+        )
+        return {}
+
+    prj = peppy.Project.from_dict(pep)
+    model = AttrStandardizer(schema)
+
+    try:
+        results = model.standardize(pep=prj)
+    except Exception:
+        raise HTTPException(
+            code=400,
+            detail=f"Error standardizing PEP.",
+        )
+
+    return StandardizerResponse(results=results)

pephub/routers/api/v1/schemas.py

@@ -1,4 +1,5 @@
-from typing import Optional
+from typing import Optional, Union, Literal
+from starlette.responses import Response
 
 import yaml
 from dotenv import load_dotenv
@@ -27,6 +28,7 @@
     SchemaGroupAssignRequest,
     SchemaGetResponse,
 )
+from ....helpers import download_yaml
 from ....dependencies import (
     get_db,
     get_namespace_access_list,
@@ -168,26 +170,45 @@ async def create_schema_for_namespace_by_file(
         )
 
 
-@schemas.get("/{namespace}/{schema}", response_model=SchemaGetResponse)
+@schemas.get("/{namespace}/{schema}", response_model=Union[SchemaGetResponse, dict])
 async def get_schema(
     namespace: str,
     schema: str,
     agent: PEPDatabaseAgent = Depends(get_db),
+    return_type: Optional[Literal["yaml", "json"]] = "json",
 ):
     try:
         schema_dict = agent.schema.get(namespace=namespace, name=schema)
+    except SchemaDoesNotExistError:
+        raise HTTPException(
+            status_code=404, detail=f"Schema {schema}/{namespace} not found."
+        )
+    if return_type == "yaml":
+
         info = agent.schema.info(namespace=namespace, name=schema)
         return SchemaGetResponse(
             schema=yaml.dump(schema_dict),
             description=info.description,
             last_update_date=info.last_update_date,
             submission_date=info.submission_date,
         )
+    else:
+        return schema_dict
+
 
+@schemas.get("/{namespace}/{schema}/file")
+async def download_schema(
+    namespace: str,
+    schema: str,
+    agent: PEPDatabaseAgent = Depends(get_db),
+) -> Response:
+    try:
+        schema_dict = agent.schema.get(namespace=namespace, name=schema)
     except SchemaDoesNotExistError:
         raise HTTPException(
             status_code=404, detail=f"Schema {schema}/{namespace} not found."
         )
+    return download_yaml(schema_dict, file_name=f"{namespace}/{schema}.yaml")
 
 
 @schemas.delete("/{namespace}/{schema}")

pephub/routers/models.py

@@ -159,3 +159,7 @@ class SchemaGetResponse(BaseModel):
     description: Optional[str] = None
     last_update_date: str = ""
     submission_date: str = ""
+
+
+class StandardizerResponse(BaseModel):
+    results: dict = {}

requirements/requirements-all.txt

@@ -1,9 +1,9 @@
 fastapi>=0.108.0
 psycopg>=3.1.15
-pepdbagent>=0.11.0
+pepdbagent>=0.11.1
 # pepdbagent @ git+https://github.com/pepkit/pepdbagent.git@schams2.0#egg=pepdbagent
-peppy>=0.40.5
-eido>=0.2.2
+peppy>=0.40.6
+eido>=0.2.3
 jinja2>=3.1.2
 python-multipart>=0.0.5
 uvicorn

web/package.json

@@ -22,7 +22,7 @@
     "@typescript-eslint/eslint-plugin": "^5.56.0",
     "@typescript-eslint/parser": "^5.56.0",
     "axios": "^1.3.4",
-    "bootstrap": "^5.2.3",
+    "bootstrap": "^5.3.3",
     "bootstrap-icons": "^1.10.3",
     "eslint": "^8.36.0",
     "eslint-config-prettier": "^8.8.0",
@@ -57,6 +57,7 @@
   "devDependencies": {
     "@mdx-js/rollup": "^3.0.0",
     "@trivago/prettier-plugin-sort-imports": "^4.1.1",
+    "@types/js-yaml": "^4.0.9",
     "@types/react": "^18.0.28",
     "@types/react-dom": "^18.0.11",
     "@vitejs/plugin-react-swc": "^3.0.0",