diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml new file mode 100644 index 0000000..5f6a6d9 --- /dev/null +++ b/.github/workflows/ci-backend.yml @@ -0,0 +1,38 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./xas-standards-api + strategy: + matrix: + python-version: ["3.10", "3.11"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install .[dev] + - name: Lint + run: tox -e pre-commit + - name: Test with pytest + run: | + python3 -m pytest diff --git a/xas-standards-api/.devcontainer/devcontainer.json b/xas-standards-api/.devcontainer/devcontainer.json index 3739d7b..245dd91 100644 --- a/xas-standards-api/.devcontainer/devcontainer.json +++ b/xas-standards-api/.devcontainer/devcontainer.json @@ -23,7 +23,8 @@ "ms-python.python", "tamasfe.even-better-toml", "redhat.vscode-yaml", - "ryanluker.vscode-coverage-gutters" + "ryanluker.vscode-coverage-gutters", + "charliermarsh.ruff" ] } }, @@ -39,9 +40,6 @@ // map in home directory - not strictly necessary but useful "source=${localEnv:HOME},target=${localEnv:HOME},type=bind,consistency=cached" ], - // make the workspace folder the same inside and outside of the container - "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind", - "workspaceFolder": "${localWorkspaceFolder}", // After the container is created, install the python project in editable form "postCreateCommand": "pip install -e '.[dev]'" -} \ No newline at end of file +} diff --git a/xas-standards-api/.github/actions/install_requirements/action.yml b/xas-standards-api/.github/actions/install_requirements/action.yml deleted file mode 100644 index 79d1a71..0000000 --- a/xas-standards-api/.github/actions/install_requirements/action.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Install requirements -description: Run pip install with requirements and upload resulting requirements -inputs: - requirements_file: - description: Name of requirements file to use and upload - required: true - install_options: - description: Parameters to pass to pip install - required: true - artifact_name: - description: A user friendly name to give the produced artifacts - required: true - python_version: - description: Python version to install - default: "3.x" - -runs: - using: composite - - steps: - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - - name: Pip install - run: | - touch ${{ inputs.requirements_file }} - # -c uses requirements.txt as constraints, see 'Validate requirements file' - pip install -c ${{ inputs.requirements_file }} ${{ inputs.install_options }} - shell: bash - - - name: Create lockfile - run: | - mkdir -p lockfiles - pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} - # delete the self referencing line and make sure it isn't blank - sed -i'' -e '/file:/d' lockfiles/${{ inputs.requirements_file }} - shell: bash - - - name: Upload lockfiles - uses: actions/upload-artifact@v4.0.0 - with: - name: lockfiles-${{ inputs.python_version }}-${{ inputs.artifact_name }}-${{ github.sha }} - path: lockfiles - - # This eliminates the class of problems where the requirements being given no - # longer match what the packages themselves dictate. E.g. In the rare instance - # where I install some-package which used to depend on vulnerable-dependency - # but now uses good-dependency (despite being nominally the same version) - # pip will install both if given a requirements file with -r - - name: If requirements file exists, check it matches pip installed packages - run: | - if [ -s ${{ inputs.requirements_file }} ]; then - if ! diff -u ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }}; then - echo "Error: ${{ inputs.requirements_file }} need the above changes to be exhaustive" - exit 1 - fi - fi - shell: bash diff --git a/xas-standards-api/.github/dependabot.yml b/xas-standards-api/.github/dependabot.yml deleted file mode 100644 index 2d1af87..0000000 --- a/xas-standards-api/.github/dependabot.yml +++ /dev/null @@ -1,20 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - groups: - github-artifacts: - patterns: - - actions/*-artifact - - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "weekly" diff --git a/xas-standards-api/.github/workflows/code.yml b/xas-standards-api/.github/workflows/code.yml deleted file mode 100644 index 3e0c4ec..0000000 --- a/xas-standards-api/.github/workflows/code.yml +++ /dev/null @@ -1,251 +0,0 @@ -name: Code CI - -on: - push: - pull_request: -env: - # The target python version, which must match the Dockerfile version - CONTAINER_PYTHON: "3.11" - DIST_WHEEL_PATH: dist-${{ github.sha }} - -defaults: - run: - working-directory: ./xas-standards-api - -jobs: - lint: - # pull requests are a duplicate of a branch push if within the same repo. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install python packages - uses: ./.github/actions/install_requirements - with: - requirements_file: requirements-dev-3.x.txt - install_options: -e .[dev] - artifact_name: lint - - - name: Lint - run: tox -e pre-commit,mypy - - test: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] # can add windows-latest, macos-latest - python: ["3.10", "3.11"] - install: ["-e .[dev]"] - # Make one version be non-editable to test both paths of version code - include: - - os: "ubuntu-latest" - python: "3.7" - install: ".[dev]" - - runs-on: ${{ matrix.os }} - env: - # https://github.com/pytest-dev/pytest/issues/2042 - PY_IGNORE_IMPORTMISMATCH: "1" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # Need this to get version number from last tag - fetch-depth: 0 - - - name: Install python packages - uses: ./.github/actions/install_requirements - with: - python_version: ${{ matrix.python }} - requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt - install_options: ${{ matrix.install }} - artifact_name: tests - - - name: List dependency tree - run: pipdeptree - - - name: Run tests - run: tox -e pytest - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - name: ${{ matrix.python }}/${{ matrix.os }} - files: cov.xml - - dist: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: "ubuntu-latest" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # Need this to get version number from last tag - fetch-depth: 0 - - - name: Build sdist and wheel - run: | - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ - pipx run build - - - name: Upload sdist and wheel as artifacts - uses: actions/upload-artifact@v4.0.0 - with: - name: ${{ env.DIST_WHEEL_PATH }} - path: dist - - - name: Check for packaging errors - run: pipx run twine check --strict dist/* - - - name: Install python packages - uses: ./.github/actions/install_requirements - with: - python_version: ${{env.CONTAINER_PYTHON}} - requirements_file: requirements.txt - install_options: dist/*.whl - artifact_name: dist - - - name: Test module --version works using the installed wheel - # If more than one module in src/ replace with module name to test - run: python -m $(ls src | head -1) --version - - container: - needs: [lint, dist, test] - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - env: - TEST_TAG: "testing" - - steps: - - name: Checkout - uses: actions/checkout@v4 - - # image names must be all lower case - - name: Generate image repo name - run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV - - - name: Set lockfile location in environment - run: | - echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV - - - name: Download wheel and lockfiles - uses: actions/download-artifact@v4.1.0 - with: - path: artifacts/ - pattern: "*dist*" - - - name: Log in to GitHub Docker Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and export to Docker local cache - uses: docker/build-push-action@v5 - with: - # Note build-args, context, file, and target must all match between this - # step and the later build-push-action, otherwise the second build-push-action - # will attempt to build the image again - build-args: | - PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - load: true - tags: ${{ env.TEST_TAG }} - # If you have a long docker build (2+ minutes), uncomment the - # following to turn on caching. For short build times this - # makes it a little slower - #cache-from: type=gha - #cache-to: type=gha,mode=max - - - name: Test cli works in cached runtime image - run: docker run docker.io/library/${{ env.TEST_TAG }} --version - - - name: Create tags for publishing image - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.IMAGE_REPOSITORY }} - tags: | - type=ref,event=tag - type=raw,value=latest, enable=${{ github.ref_type == 'tag' }} - # type=edge,branch=main - # Add line above to generate image for every commit to given branch, - # and uncomment the end of if clause in next step - - - name: Push cached image to container registry - if: github.ref_type == 'tag' # || github.ref_name == 'main' - uses: docker/build-push-action@v5 - # This does not build the image again, it will find the image in the - # Docker cache and publish it - with: - # Note build-args, context, file, and target must all match between this - # step and the previous build-push-action, otherwise this step will - # attempt to build the image again - build-args: | - PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - release: - # upload to PyPI and make a release on every tag - needs: [lint, dist, test] - if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} - runs-on: ubuntu-latest - env: - HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }} - - steps: - - name: Download wheel and lockfiles - uses: actions/download-artifact@v4.1.0 - with: - pattern: "*dist*" - - - name: Rename lockfiles and dist - run: | - mv lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }} lockfiles - mv ${{ env.DIST_WHEEL_PATH }} dist - - - name: Fixup blank lockfiles - # Github release artifacts can't be blank - run: for f in lockfiles/*; do [ -s $f ] || echo '# No requirements' >> $f; done - - - name: Github Release - # We pin to the SHA, not the tag, for security reasons. - # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 - with: - prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} - files: | - dist/* - lockfiles/* - generate_release_notes: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Publish to PyPI - if: ${{ env.HAS_PYPI_TOKEN }} - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_TOKEN }} diff --git a/xas-standards-api/nexus_to_xdi.py b/xas-standards-api/nexus_to_xdi.py index c612022..59e4dd2 100644 --- a/xas-standards-api/nexus_to_xdi.py +++ b/xas-standards-api/nexus_to_xdi.py @@ -1,37 +1,35 @@ -import numpy as np import h5py - -#Example - - # XDI/1.0 GSE/1.0 - # Column.1: energy eV - # Column.2: i0 - # Column.3: itrans - # Column.4: mutrans - # Element.edge: K - # Element.symbol: Cu - # Scan.edge_energy: 8980.0 - # Mono.name: Si 111 - # Mono.d_spacing: 3.13553 - # Beamline.name: 13ID - # Beamline.collimation: none - # Beamline.focusing: yes - # Beamline.harmonic_rejection: rhodium-coated mirror - # Facility.name: APS - # Facility.energy: 7.00 GeV - # Facility.xray_source: APS Undulator A - # Scan.start_time: 2001-06-26T22:27:31 - # Detector.I0: 10cm N2 - # Detector.I1: 10cm N2 - # Sample.name: Cu - # Sample.prep: Cu metal foil - # GSE.EXTRA: config 1 - # /// - # Cu foil Room Temperature - # measured at beamline 13-ID - #---- - - +import numpy as np + +# Example + +# XDI/1.0 GSE/1.0 +# Column.1: energy eV +# Column.2: i0 +# Column.3: itrans +# Column.4: mutrans +# Element.edge: K +# Element.symbol: Cu +# Scan.edge_energy: 8980.0 +# Mono.name: Si 111 +# Mono.d_spacing: 3.13553 +# Beamline.name: 13ID +# Beamline.collimation: none +# Beamline.focusing: yes +# Beamline.harmonic_rejection: rhodium-coated mirror +# Facility.name: APS +# Facility.energy: 7.00 GeV +# Facility.xray_source: APS Undulator A +# Scan.start_time: 2001-06-26T22:27:31 +# Detector.I0: 10cm N2 +# Detector.I1: 10cm N2 +# Sample.name: Cu +# Sample.prep: Cu metal foil +# GSE.EXTRA: config 1 +# /// +# Cu foil Room Temperature +# measured at beamline 13-ID +# ---- def main(): @@ -55,7 +53,7 @@ def main(): facility = "Facility.name: DLS" xray = "Facility.xray_source: DLS Bending Magnet B18" start = "Scan.start_time: 2001-06-26T22:27:31" - name = "Sample.name: Cu Formate" + name = "Sample.name: Cu Formate" prep = "Sample.prep: Pressed pellet" stoich = "Sample.stoichiometry: C2 H2 Cu O4" @@ -69,7 +67,7 @@ def main(): column5 = "Column.5: i0" column6 = "Column.6: irefer" - with h5py.File("163245_Cu_formate_1.nxs") as fh, open("test.xdi",'w') as xdi: + with h5py.File("163245_Cu_formate_1.nxs") as fh, open("test.xdi", "w") as xdi: xdi.write(header_symbol + version + lf) xdi.write(header_symbol + column1 + lf) @@ -84,12 +82,11 @@ def main(): xdi.write(header_symbol + beamline + lf) xdi.write(header_symbol + facility + lf) xdi.write(header_symbol + xray + lf) - + xdi.write(header_symbol + start + lf) xdi.write(header_symbol + name + lf) xdi.write(header_symbol + prep + lf) xdi.write(header_symbol + stoich + lf) - xdi.write(header_symbol + field_end + lf) xdi.write(header_symbol + comment1 + lf) @@ -107,11 +104,8 @@ def main(): datasets.append(fh[irp][...]) all = np.vstack(datasets) - np.savetxt(xdi,all.T) - - - + np.savetxt(xdi, all.T) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/xas-standards-api/src/xas_standards_api/__main__.py b/xas-standards-api/src/xas_standards_api/__main__.py index 313e363..f4cf13b 100644 --- a/xas-standards-api/src/xas_standards_api/__main__.py +++ b/xas-standards-api/src/xas_standards_api/__main__.py @@ -1,9 +1,9 @@ from argparse import ArgumentParser -from . import __version__ - import uvicorn +from . import __version__ + __all__ = ["main"] @@ -12,9 +12,9 @@ def main(args=None): parser.add_argument("-v", "--version", action="version", version=__version__) args = parser.parse_args(args) - uvicorn.run("xas_standards_api.app:app", port=5000, log_level="info", host='0.0.0.0') - - + uvicorn.run( + "xas_standards_api.app:app", port=5000, log_level="info", host="0.0.0.0" + ) # test with: python -m xas_standards_api diff --git a/xas-standards-api/src/xas_standards_api/app.py b/xas-standards-api/src/xas_standards_api/app.py index bd91562..890cd26 100644 --- a/xas-standards-api/src/xas_standards_api/app.py +++ b/xas-standards-api/src/xas_standards_api/app.py @@ -1,47 +1,48 @@ -import os import datetime -from contextlib import asynccontextmanager -from typing import Annotated,List, Optional, Union +import os +from typing import Annotated, List, Optional, Union -from fastapi import Depends, FastAPI, Form, UploadFile, File, Query +from fastapi import Depends, FastAPI, File, Form, Query, UploadFile from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles from fastapi_pagination import add_pagination from fastapi_pagination.cursor import CursorPage from fastapi_pagination.ext.sqlalchemy import paginate -from sqlmodel import Session, SQLModel, create_engine, select - -from fastapi.staticfiles import StaticFiles - -from .crud import (add_new_standard, get_data, get_standard, update_review, select_all,select_or_create_person, get_file) +from sqlmodel import Session, create_engine, select + +from .crud import ( + add_new_standard, + get_data, + get_file, + get_standard, + select_all, + select_or_create_person, + update_review, +) from .schemas import ( - Review, - XASStandard, - Element, - Edge, Beamline, BeamlineResponse, - XASStandardResponse, + Edge, + Element, + LicenceType, + Review, + XASStandard, XASStandardAdminResponse, XASStandardInput, - LicenceType + XASStandardResponse, ) dev = False lifespan = None -if dev: - engine = create_engine( - "sqlite:///standards.db", connect_args={"check_same_thread": False}, echo=True - ) +url = os.environ.get("POSTGRESURL") +build_dir = os.environ.get("FRONTEND_BUILD_DIR") - @asynccontextmanager - async def lifespan(app: FastAPI): - SQLModel.metadata.create_all(engine) - yield -else: - url = os.environ.get("POSTGRESURL") +if url: engine = create_engine(url) +else: + print("URL not set - unit tests only") def get_session(): @@ -59,44 +60,57 @@ def get_session(): add_pagination(app) -@app.get("/api/licences") +@app.get("/api/licences") def read_licences(session: Session = Depends(get_session)) -> List[LicenceType]: return list(LicenceType) -@app.get("/api/beamlines") -def read_beamlines(session: Session = Depends(get_session))-> List[BeamlineResponse]: - bl = select_all(session,Beamline) + +@app.get("/api/beamlines") +def read_beamlines(session: Session = Depends(get_session)) -> List[BeamlineResponse]: + bl = select_all(session, Beamline) return bl -@app.get("/api/elements") -def read_elements(session: Session = Depends(get_session))-> List[Element]: + +@app.get("/api/elements") +def read_elements(session: Session = Depends(get_session)) -> List[Element]: e = select_all(session, Element) return e -@app.get("/api/edges") -def read_edges(session: Session = Depends(get_session))-> List[Edge]: + +@app.get("/api/edges") +def read_edges(session: Session = Depends(get_session)) -> List[Edge]: e = select_all(session, Edge) return e + @app.get("/api/standards") -def read_standards(session: Session = Depends(get_session), - element: str | None = None, - admin: bool = False, - response_model=Union[XASStandardResponse, XASStandardAdminResponse]) -> CursorPage[XASStandardAdminResponse | XASStandardResponse]: - - #CHECK HEADER FOR ADMIN QUERY +def read_standards( + session: Session = Depends(get_session), + element: str | None = None, + admin: bool = False, + response_model=Union[XASStandardResponse, XASStandardAdminResponse], +) -> CursorPage[XASStandardAdminResponse | XASStandardResponse]: statement = select(XASStandard) if element: - statement = statement.join(Element, XASStandard.element_z==Element.z).where(Element.symbol == element) + statement = statement.join(Element, XASStandard.element_z == Element.z).where( + Element.symbol == element + ) if admin: - transformer = lambda x: [XASStandardAdminResponse.model_validate(i) for i in x] + + def transformer(x): + return [XASStandardAdminResponse.model_validate(i) for i in x] + else: - transformer = lambda x: [XASStandardResponse.model_validate(i) for i in x] - return paginate(session, statement.order_by(XASStandard.id), transformer=transformer) + def transformer(x): + return [XASStandardResponse.model_validate(i) for i in x] + + return paginate( + session, statement.order_by(XASStandard.id), transformer=transformer + ) @app.get("/api/standards/{id}") @@ -109,39 +123,41 @@ async def read_standard( @app.post("/api/standards") def add_standard_file( xdi_file: UploadFile, - element_id: Annotated[str, Form()], - edge_id: Annotated[str, Form()], - beamline_id: Annotated[int, Form()], - sample_name: Annotated[str, Form()], + element_id: Annotated[str, Form()], + edge_id: Annotated[str, Form()], + beamline_id: Annotated[int, Form()], + sample_name: Annotated[str, Form()], sample_prep: Annotated[str, Form()], doi: Annotated[str, Form()], citation: Annotated[str, Form()], comments: Annotated[str, Form()], date: Annotated[str, Form()], - licence: Annotated[str, Form()], - additional_files: Optional[list[UploadFile]]= Form(None), - sample_comp: Optional[str] = Form(None), - session: Session = Depends(get_session) + licence: Annotated[str, Form()], + additional_files: Optional[list[UploadFile]] = Form(None), + sample_comp: Optional[str] = Form(None), + session: Session = Depends(get_session), ) -> XASStandard: - + if additional_files: print(f"Additional files {len(additional_files)}") person = select_or_create_person(session, "test1234") - form_input = XASStandardInput(submitter_id=person.id, - beamline_id=beamline_id, - doi=doi, - element_z=element_id, - edge_id=edge_id, - sample_name=sample_name, - sample_prep=sample_prep, - submitter_comments= comments, - citation=citation, - licence=licence, - collection_date=date, - submission_date=datetime.datetime.now(), - sample_comp=sample_comp) + form_input = XASStandardInput( + submitter_id=person.id, + beamline_id=beamline_id, + doi=doi, + element_z=element_id, + edge_id=edge_id, + sample_name=sample_name, + sample_prep=sample_prep, + submitter_comments=comments, + citation=citation, + licence=licence, + collection_date=date, + submission_date=datetime.datetime.now(), + sample_comp=sample_comp, + ) return add_new_standard(session, xdi_file, form_input, additional_files) @@ -152,12 +168,12 @@ def submit_review(review: Review, session: Session = Depends(get_session)): @app.get("/api/data/{id}") -async def read_data(id: int, - format: Optional[str] = "json", - session: Session = Depends(get_session)): - +async def read_data( + id: int, format: Optional[str] = "json", session: Session = Depends(get_session) +): + if format == "xdi": - return get_file(session,id) + return get_file(session, id) return get_data(session, id) @@ -183,4 +199,6 @@ async def main(): """ return HTMLResponse(content=content) -app.mount("/", StaticFiles(directory="/client/dist", html = True), name="site") \ No newline at end of file + +if build_dir: + app.mount("/", StaticFiles(directory="/client/dist", html=True), name="site") diff --git a/xas-standards-api/src/xas_standards_api/crud.py b/xas-standards-api/src/xas_standards_api/crud.py index a97a8d4..08282b5 100644 --- a/xas-standards-api/src/xas_standards_api/crud.py +++ b/xas-standards-api/src/xas_standards_api/crud.py @@ -1,34 +1,35 @@ -import os import uuid from fastapi import HTTPException from fastapi.responses import FileResponse from larch.io import xdi +from larch.xafs import pre_edge, set_xafsGroup from sqlmodel import select -from larch.xafs import pre_edge,set_xafsGroup - from .schemas import ( Beamline, Person, PersonInput, XASStandard, XASStandardData, + XASStandardDataInput, XASStandardInput, - XASStandardDataInput ) pvc_location = "/scratch/xas-standards-pretend-pvc/" + def get_beamline_names(session): - results = session.exec(select(Beamline.name, Beamline.id)).all(); + results = session.exec(select(Beamline.name, Beamline.id)).all() return results + def select_all(session, sql_model): statement = select(sql_model) results = session.exec(statement) return results.unique().all() + def get_standard(session, id) -> XASStandard: standard = session.get(XASStandard, id) if standard: @@ -45,6 +46,7 @@ def update_review(session, review): session.refresh(standard) return standard + def select_or_create_person(session, identifier): p = PersonInput(identifier=identifier) @@ -52,7 +54,7 @@ def select_or_create_person(session, identifier): person = session.exec(statement).first() if person is None: - new_person = Person.from_orm(p) + new_person = Person.model_validate(p) session.add(new_person) session.commit() session.refresh(new_person) @@ -61,7 +63,7 @@ def select_or_create_person(session, identifier): return person -def add_new_standard(session, file1, xs_input : XASStandardInput, additional_files): +def add_new_standard(session, file1, xs_input: XASStandardInput, additional_files): tmp_filename = pvc_location + str(uuid.uuid4()) @@ -76,11 +78,13 @@ def add_new_standard(session, file1, xs_input : XASStandardInput, additional_fil transmission = "mutrans" in set_labels emission = "mutey" in set_labels - xsd = XASStandardDataInput(fluorescence=fluorescence, - location=tmp_filename, - original_filename=file1.filename, - emission=emission, - transmission=transmission) + xsd = XASStandardDataInput( + fluorescence=fluorescence, + location=tmp_filename, + original_filename=file1.filename, + emission=emission, + transmission=transmission, + ) new_standard = XASStandard.model_validate(xs_input) new_standard.xas_standard_data = XASStandardData.model_validate(xsd) @@ -96,21 +100,23 @@ def get_filepath(session, id): standard = session.get(XASStandard, id) if not standard: raise HTTPException(status_code=404, detail=f"No standard with id={id}") - + standard_data = session.get(XASStandardData, standard.data_id) if not standard_data: - raise HTTPException(status_code=404, detail=f"No standard data for standard with id={id}") - + raise HTTPException( + status_code=404, detail=f"No standard data for standard with id={id}" + ) + return standard_data.location def get_file(session, id): - xdi_location = get_filepath(session,id) + xdi_location = get_filepath(session, id) return FileResponse(xdi_location) -def get_norm(energy,group,type): +def get_norm(energy, group, type): if type in group: r = group[type] @@ -122,6 +128,7 @@ def get_norm(energy,group,type): return [] + def get_data(session, id): xdi_location = get_filepath(session, id) @@ -130,14 +137,19 @@ def get_data(session, id): if "energy" not in xdi_data: raise HTTPException(status_code=404, detail=f"No energy in file with id={id}") - + if "mutrans" not in xdi_data and "": raise HTTPException(status_code=404, detail=f"No itrans in file with id={id}") e = xdi_data["energy"] - trans_out = get_norm(e,xdi_data,"mutrans") - fluor_out = get_norm(e,xdi_data,"mufluor") - ref_out = get_norm(e,xdi_data,"murefer") + trans_out = get_norm(e, xdi_data, "mutrans") + fluor_out = get_norm(e, xdi_data, "mufluor") + ref_out = get_norm(e, xdi_data, "murefer") - return {"energy": e.tolist(), "mutrans": trans_out, "mufluor":fluor_out, "murefer": ref_out} + return { + "energy": e.tolist(), + "mutrans": trans_out, + "mufluor": fluor_out, + "murefer": ref_out, + } diff --git a/xas-standards-api/src/xas_standards_api/schemas.py b/xas-standards-api/src/xas_standards_api/schemas.py index bfc6e5b..2476b07 100644 --- a/xas-standards-api/src/xas_standards_api/schemas.py +++ b/xas-standards-api/src/xas_standards_api/schemas.py @@ -1,47 +1,55 @@ -from typing import Optional, List -from pydantic import BaseModel -from sqlmodel import Field, SQLModel, Enum, Column, Relationship - import datetime - import enum +from typing import List, Optional + +from pydantic import BaseModel +from sqlmodel import Column, Enum, Field, Relationship, SQLModel class Review(BaseModel): id: int review_status: str + class Mono(BaseModel): name: Optional[str] = None d_spacing: Optional[str] = None + class Sample(BaseModel): name: Optional[str] = None prep: Optional[str] = None + class PersonInput(SQLModel): identifier: str = Field(index=True, unique=True) + class Person(PersonInput, table=True): id: int | None = Field(primary_key=True, default=None) + class ElementInput(SQLModel): symbol: str = Field(unique=True) + class Element(ElementInput, table=True): __tablename__: str = "element" z: int = Field(primary_key=True, unique=True) name: str = Field(unique=True) + class EdgeInput(SQLModel): - name: str = Field(unique=True) + name: str = Field(unique=True) + class Edge(EdgeInput, table=True): __tablename__: str = "edge" id: int = Field(primary_key=True) - level:str = Field(unique=True) + level: str = Field(unique=True) + class Facility(SQLModel, table=True): __tablename__: str = "facility" @@ -55,7 +63,10 @@ class Facility(SQLModel, table=True): region: str country: str - beamlines: List["Beamline"] = Relationship(back_populates="facility", sa_relationship_kwargs={"lazy": "joined"}) + beamlines: List["Beamline"] = Relationship( + back_populates="facility", sa_relationship_kwargs={"lazy": "joined"} + ) + class Beamline(SQLModel, table=True): __tablename__: str = "beamline" @@ -66,7 +77,9 @@ class Beamline(SQLModel, table=True): xray_source: str | None facility_id: int = Field(foreign_key="facility.id") - facility: Facility = Relationship(back_populates="beamlines", sa_relationship_kwargs={"lazy": "joined"}) + facility: Facility = Relationship( + back_populates="beamlines", sa_relationship_kwargs={"lazy": "joined"} + ) class FacilityResponse(SQLModel): @@ -75,6 +88,7 @@ class FacilityResponse(SQLModel): city: str country: str + class BeamlineResponse(SQLModel): id: int name: str @@ -89,6 +103,7 @@ class XASStandardDataInput(SQLModel): emission: bool location: str + class XASStandardData(XASStandardDataInput, table=True): __tablename__: str = "xas_standard_data" @@ -102,6 +117,7 @@ class ReviewStatus(enum.Enum): approved = "approved" rejected = "rejected" + class LicenceType(enum.Enum): cc_by = "cc_by" cc_0 = "cc_0" @@ -121,7 +137,8 @@ class XASStandardInput(SQLModel): sample_prep: Optional[str] sample_comp: Optional[str] beamline_id: int = Field(foreign_key="beamline.id") - licence : LicenceType = Field(sa_column=Column(Enum(LicenceType))) + licence: LicenceType = Field(sa_column=Column(Enum(LicenceType))) + class XASStandard(XASStandardInput, table=True): __tablename__: str = "xas_standard" @@ -129,24 +146,25 @@ class XASStandard(XASStandardInput, table=True): data_id: int | None = Field(foreign_key="xas_standard_data.id", default=None) reviewer_id: Optional[int] = Field(foreign_key="person.id", default=None) reviewer_comments: Optional[str] = None - review_status: Optional[ReviewStatus] = Field(sa_column=Column(Enum(ReviewStatus)), default=ReviewStatus.pending) + review_status: Optional[ReviewStatus] = Field( + sa_column=Column(Enum(ReviewStatus)), default=ReviewStatus.pending + ) xas_standard_data: XASStandardData = Relationship(back_populates="xas_standard") element: Element = Relationship(sa_relationship_kwargs={"lazy": "joined"}) edge: Edge = Relationship(sa_relationship_kwargs={"lazy": "joined"}) beamline: Beamline = Relationship(sa_relationship_kwargs={"lazy": "selectin"}) + class XASStandardResponse(XASStandardInput): - id : int | None + id: int | None element: ElementInput edge: EdgeInput beamline: BeamlineResponse - submitter_id: int + submitter_id: int class XASStandardAdminResponse(XASStandardResponse): reviewer_id: Optional[int] = None reviewer_comments: Optional[str] = None review_status: Optional[ReviewStatus] = None - - diff --git a/xas-standards-api/tests/test_cli.py b/xas-standards-api/tests/test_cli.py index 8e7051a..083a97c 100644 --- a/xas-standards-api/tests/test_cli.py +++ b/xas-standards-api/tests/test_cli.py @@ -3,6 +3,7 @@ from xas_standards_api import __version__ + def test_cli_version(): cmd = [sys.executable, "-m", "xas_standards_api", "--version"] assert subprocess.check_output(cmd).decode().strip() == __version__ diff --git a/xas-standards-api/tests/test_crud.py b/xas-standards-api/tests/test_crud.py index 7aa93b3..37870c8 100644 --- a/xas-standards-api/tests/test_crud.py +++ b/xas-standards-api/tests/test_crud.py @@ -1,11 +1,11 @@ -from sqlmodel import Session +from unittest.mock import Mock, call, create_autospec -from unittest.mock import call, create_autospec, Mock import pytest +from sqlmodel import Session +from xas_standards_api import crud from xas_standards_api.schemas import XASStandard -from xas_standards_api import crud def test_get_standard(): @@ -15,15 +15,15 @@ def test_get_standard(): result = XASStandard() mock_session.get = Mock(return_value=None) - expected_session_calls = [call.get(XASStandard,test_id)] + expected_session_calls = [call.get(XASStandard, test_id)] with pytest.raises(Exception): - crud.get_standard(mock_session,test_id) + crud.get_standard(mock_session, test_id) mock_session.get.assert_has_calls(expected_session_calls) mock_session.get = Mock(return_value=result) - output = crud.get_standard(mock_session,test_id) + output = crud.get_standard(mock_session, test_id) - assert output == result \ No newline at end of file + assert output == result