From edd7aaeee104d5e6d76484b24c8fb572114a4b99 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 3 Sep 2024 13:51:01 -0700 Subject: [PATCH 1/3] Adds local tests with mock auth and updates readme docs --- .gitignore | 135 +++++ README.md | 85 +-- gen3_writer/README.md | 101 +++- .../DocumentReference.in.edge.json | 1 + .../DocumentReference.out.edge.json | 1 + .../DocumentReference.vertex.json | 1 + .../Observation.in.edge.json | 8 + .../Observation.out.edge.json | 8 + .../Observation.vertex.json | 3 + .../Organization.vertex.json | 1 + .../combio-examples-grip/Patient.vertex.json | 1 + .../ResearchStudy.vertex.json | 1 + .../ResearchSubject.in.edge.json | 2 + .../ResearchSubject.out.edge.json | 2 + .../ResearchSubject.vertex.json | 1 + .../Specimen.in.edge.json | 1 + .../Specimen.out.edge.json | 1 + .../combio-examples-grip/Specimen.vertex.json | 1 + gen3_writer/gen3_test.go | 539 ++++++++++-------- gen3_writer/handler.go | 32 +- graphql_gen3/README.md | 18 +- gripgraphql/handler.go | 13 +- gripgraphql/js_client.go | 1 - middleware/gen3_caching.go | 55 +- 24 files changed, 641 insertions(+), 371 deletions(-) create mode 100644 .gitignore create mode 100644 gen3_writer/fixtures/combio-examples-grip/DocumentReference.in.edge.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/DocumentReference.out.edge.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/DocumentReference.vertex.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/Observation.in.edge.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/Observation.out.edge.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/Observation.vertex.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/Organization.vertex.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/Patient.vertex.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/ResearchStudy.vertex.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/ResearchSubject.in.edge.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/ResearchSubject.out.edge.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/ResearchSubject.vertex.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/Specimen.in.edge.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/Specimen.out.edge.json create mode 100644 gen3_writer/fixtures/combio-examples-grip/Specimen.vertex.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b5ef840 --- /dev/null +++ b/.gitignore @@ -0,0 +1,135 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# IDE +.idea/ + +# scratch +tmp/ \ No newline at end of file diff --git a/README.md b/README.md index 23c5712..ec19aa1 100644 --- a/README.md +++ b/README.md @@ -1,84 +1,11 @@ +### Grip Graphql Plugins -# GRIP GraphQL Endpoint -Configurable GraphQL endpoint for the GRaph Integration Platform +grip-graphql is a collection of go plugins designed and implemented for connecting a [Grip](https://github.com/bmeg/grip) server to other microservices in a modified [Gen3](https://gen3.org/) software stack. +gen3_writer directory contains a [Gin](https://github.com/gin-gonic/gin) go server plugin that is used for Writing / Deleting data from Graphs on a grip server -## Build +gripgraphql directory contains a graphql based read query plugin that uses a [goja](https://github.com/dop251/goja) engine to read from a static schema defined as a config file to create custom graphql queries that can be used to abstract Grip's complex query language into a more digestible query format for the frontend to use. -Run `make` to build both the plugin (grip-graphql-endpoint.so) and proxy server (grip-graphql-proxy). +graphql_gen3 is a legacy implementation of a reader reader plugin using a more traditional graphql schema builder. - -## Plugin -Run as a shared object within the GRIP server -``` -grip server -w graphql=grip-graphql-endpoint.so -l graphql:config=./config/config.js -l graphql:graph=test-db -``` - -## Proxy -Run the server as a proxy endpoint connected to an external GRIP service -``` -./grip-graphql-proxy -``` - -## Example configuration file - -```javascript - -endpoint.add({ - name: "projects", - schema: [ - "String" - ], - handler: (G, args) => { - return G.V().hasLabel("Project").render("_gid").toList() - } -}) - -endpoint.add({ - name: "cases", - schema: [ - "String" - ], - args: { - offset: "Int", - limit: "Int", - project_id : "String" - }, - defaults: { - offset: 0, - limit: 100 - }, - handler: (G, args) => { - if (args.project_id === undefined) { - return G.V().hasLabel("Case").skip(args.offset).limit(args.limit).render("_gid").toList() - } else { - return G.V().hasLabel("Case").has(gripql.eq("project_id", args.project_id)).skip(args.offset).limit(args.limit).render("_gid").toList() - } - } -}) - -endpoint.add({ - name: "caseCounts", - schema: { - cases: "Int", - samples: "Int", - aliquots: "Int" - }, - args: { - project_id: "String" - }, - handler: (G, args) => { - return { - "cases": G.V().hasLabel("Case").has(gripql.eq("project_id", args.project_id)).count().toList()[0], - "samples": G.V().hasLabel("Case").has(gripql.eq("project_id", args.project_id)).out("samples").count().toList()[0], - "aliquots": G.V().hasLabel("Case").has(gripql.eq("project_id", args.project_id)).out("samples").out("aliquots").count().toList()[0], - } - } -}) -``` - - -## GRIP setup with frontend framework - -1. setup frontendframework see docs -2. grip server -c mongo.yml -w graphql=grip-graphql-endpoint.so -l graphql:config=config/gen3.js -l graphql:graph=gdc +See ./gen3_writer for tests and additional documentation diff --git a/gen3_writer/README.md b/gen3_writer/README.md index df97a38..6340571 100644 --- a/gen3_writer/README.md +++ b/gen3_writer/README.md @@ -1,59 +1,102 @@ # GRIP FHIR RESTFUL API -## Setup - -Create a grip executable or build one locally by cloning grip main branch and -do a: - -``` -export PATH=$PATH:$HOME/go/bin -go build . -go install . -``` +## Local Setup ``` +go install github.com/bmeg/grip go build --buildmode=plugin ./gen3_writer -grip server -w api/writer=gen3_writer.so +grip server -w graphql=gen3_writer.so ``` -Note: the -w api/writer=gen3_writer.so option can be stacked to include multiple endpoints and long as you have -built the .so file to the corresponding endplint as shown above. +Note: the -w graphql=gen3_writer.so option can be stacked to include multiple endpoints and long as you have +built the .so file to the corresponding endplint as shown above. Ex: + +`grip server -c mongo.yml -w graphql=gen3_writer.so -w reader=grip-graphql-endpoint.so -l reader:config=./config/gen3.js -l reader:graph=CALIPER` + +## Example queries: + +Note: ENV var ACCESS_TOKEN is a valid Gen3 jwt token. An access token is needed for all queries except GET \_status and GET list-graphs -## Example queries: +### Delete an edge and then grep for it to see if it has been deleted or not. Format: + +_http://localhost:8201/graphql/[graph-name]/del-edge/[edge-id]/[gen3-project-id]_ -Delete an edge and then grep for it to see if it has been deleted or not ``` -curl -X DELETE -H "Content-Type: applicationjson" http://localhost:8201/api/writer/test/del-edge/2XM1hqErQvIw9s0cUWDuRpKPbQR -grip query test "E()" | grep 2XM1hqErQvIw9s0cUWDuRpKPbQR +curl -X DELETE http://localhost:8201/graphql/CALIPER/del-edge/fb60e763-e799-4d59-82a3-66977cc6696c/ohsu-test +-H "Content-Type: applicationjson" \ +-H "Authorization: bearer $ACCESS_TOKEN" + +grip query CALIPER "E()" | grep fb60e763-e799-4d59-82a3-66977cc6696c ``` -Bulk load some edges from an edge file: +### Bulk load some edges from a file: + +_http://localhost:8201/graphql/[graph-name]/bulk-load/[gen3-project-id]_ + ``` -curl -X POST -H "Content-Type: applicationjson" -d '{"edge": "../../aced-data/grip-aced-data/edge.ndjson"}' http://localhost:8201/api/writer/test/bulk-load +curl -X POST "http://localhost:8201/graphql/CALIPER/bulk-load/ohsu-test" \ + -H "Authorization: bearer $ACCESS_TOKEN" \ + -F "types=edge" \ + -F "file=@edge.ndjson" ``` -Note: Each edge in edge file above will be of format: +Newline delimited edges should be of form: ``` { - "label": "custodian", + "gid": "bee5bd86-4f06-5eb2-b71a-f62110cf5aa9", + "label": "specimen_observation", "from": "9bc10566-5d7e-4a53-bbc0-6fe9700584a5", - "to": "Organization/ea4ea5e7-2780-46cf-8cc4-fbb40ad63928" + "to": "ea4ea5e7-2780-46cf-8cc4-fbb40ad63928" } ``` -With required keys "label", "from" and "to" -Get the value of the vertex with id 302324d5-1d92-5425-80d5-ac6c63af84b6 +With required keys "label", "from", "to", and "gid" and optional key "data" with value of type dict. + +### Get the data from a vertex given a known vertex id + +_http://localhost:8201/graphql/[graph-name]/get-vertex/[vertex-id]/[gen3-project-id]_ + +``` +curl -X GET http://localhost:8201/graphql/CALIPER/get-vertex/875ddaf8-42da-5d72-b5c5-39c2b16151cd/ohsu-test \ +-H "Authorization: bearer $ACCESS_TOKEN" + ``` -curl -X GET http://localhost:8201/api/graphql/test/get-vertex/302324d5-1d92-5425-80d5-ac6c63af84b6 + +### Get the list of graphs present + ``` +curl http://localhost:8201/graphql/list-graphs +``` + +### Revproxy Setup -- + +The above curl commands assume that you are acessing this grip plugin from within the cluster. equivalent queries can be used from outside the cluster by changing the nginx paths to the form: + +`https://[your_instance_endpoint]/grip/writer/graphql/list-graphs` for the writer or +`https://[your_instance_endpoint]/grip/reader` for the reader api + +These paths assume you have checked out to the grip branch of helm and reployed + +## Tests: + +Tests can be run locally by specifying that you want to turn on the plugin in testing mode using the `TEST` Graph. For example: -Get the list of graphs present ``` -curl -X GET http://localhost:8201/api/graphql/list-graphs +grip server -w graphql=gen3_writer.so \ + -w reader=grip-graphql-endpoint.so \ + -l reader:config=./config/gen3.js \ + -l reader:graph=TEST \ + -l graphql:test=true \ + -l reader:test=true ``` +then cd to gen3_writer directory and run: + +`go test` or `go test -v` for logs or `go test -run [specific_test_name]` to run only a specific test + +If the graph name is `TEST` and the config is setup for test=true, mock auth will be used, and these tests can be run locally outside of a gen3 instance. + ## Version -go version go1.21.3 -## Tests: not currently functional. +go version go1.22.6 diff --git a/gen3_writer/fixtures/combio-examples-grip/DocumentReference.in.edge.json b/gen3_writer/fixtures/combio-examples-grip/DocumentReference.in.edge.json new file mode 100644 index 0000000..776afc1 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/DocumentReference.in.edge.json @@ -0,0 +1 @@ +{"gid":"11e86b6c-4e7d-5b94-a892-216e29baf0cb","label":"subject_Specimen","from":"9ae7e542-767f-4b03-a854-7ceed17152cb","to":"60c67a06-ea2d-4d24-9249-418dc77a16a9"} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/DocumentReference.out.edge.json b/gen3_writer/fixtures/combio-examples-grip/DocumentReference.out.edge.json new file mode 100644 index 0000000..3a4ce10 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/DocumentReference.out.edge.json @@ -0,0 +1 @@ +{"gid":"655b13f1-1955-510f-bac8-5934777099e4","label":"document_reference","from":"60c67a06-ea2d-4d24-9249-418dc77a16a9","to":"9ae7e542-767f-4b03-a854-7ceed17152cb"} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/DocumentReference.vertex.json b/gen3_writer/fixtures/combio-examples-grip/DocumentReference.vertex.json new file mode 100644 index 0000000..33fc9fc --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/DocumentReference.vertex.json @@ -0,0 +1 @@ +{"gid":"9ae7e542-767f-4b03-a854-7ceed17152cb","label":"DocumentReference","data":{"auth_resource_path":"/programs/ohsu/projects/test","content":[{"attachment":{"contentType":"text/fastq","creation":"2024-08-21T10:53:00+00:00","extension":[{"url":"http://aced-idp.org/fhir/StructureDefinition/md5","valueString":"227f0a5379362d42eaa1814cfc0101b8"},{"url":"http://aced-idp.org/fhir/StructureDefinition/source_path","valueUrl":"file:///home/LabA/specimen_1234_labA.fq.gz"}],"size":5595609484,"title":"specimen_1234_labA.fq.gz","url":"file:///home/LabA/specimen_1234_labA.fq.gz"}}],"date":"2024-08-21T10:53:00+00:00","docStatus":"final","id":"9ae7e542-767f-4b03-a854-7ceed17152cb","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"9ae7e542-767f-4b03-a854-7ceed17152cb"}],"resourceType":"DocumentReference","status":"current","subject":{"reference":"Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9"}}} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/Observation.in.edge.json b/gen3_writer/fixtures/combio-examples-grip/Observation.in.edge.json new file mode 100644 index 0000000..795fd7b --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/Observation.in.edge.json @@ -0,0 +1,8 @@ +{"gid":"f043f4b2-6eed-5431-aed9-f875390799c2","label":"focus_DocumentReference","from":"cec32723-9ede-5f24-ba63-63cb8c6a02cf","to":"9ae7e542-767f-4b03-a854-7ceed17152cb"} +{"gid":"0130767a-71ff-5677-aeb9-b1ce9f37f25c","label":"specimen_Specimen","from":"cec32723-9ede-5f24-ba63-63cb8c6a02cf","to":"60c67a06-ea2d-4d24-9249-418dc77a16a9"} +{"gid":"7471e5a6-d026-5311-b79a-0a3f7b9e9893","label":"subject_Patient","from":"cec32723-9ede-5f24-ba63-63cb8c6a02cf","to":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920"} +{"gid":"4d3335be-7333-5c61-a534-57e2148f3df0","label":"focus_Specimen","from":"4e3c6b59-b1fd-5c26-a611-da4cde9fd061","to":"60c67a06-ea2d-4d24-9249-418dc77a16a9"} +{"gid":"e2867c6d-db7e-5d6e-87d8-b1c293f5b47e","label":"subject_Patient","from":"4e3c6b59-b1fd-5c26-a611-da4cde9fd061","to":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920"} +{"gid":"2bc87039-7ec6-539a-b73b-158fbb4ea778","label":"focus_DocumentReference","from":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f","to":"9ae7e542-767f-4b03-a854-7ceed17152cb"} +{"gid":"72ec2c73-7dd7-5fc7-97f1-cc4a53707159","label":"specimen_Specimen","from":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f","to":"60c67a06-ea2d-4d24-9249-418dc77a16a9"} +{"gid":"a6268b43-7100-5663-bbbb-60dd736873a0","label":"subject_Patient","from":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f","to":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920"} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/Observation.out.edge.json b/gen3_writer/fixtures/combio-examples-grip/Observation.out.edge.json new file mode 100644 index 0000000..06da3e7 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/Observation.out.edge.json @@ -0,0 +1,8 @@ +{"gid":"b51c4080-6c87-548e-ae5a-27cff55ab422","label":"focus_observation","from":"9ae7e542-767f-4b03-a854-7ceed17152cb","to":"cec32723-9ede-5f24-ba63-63cb8c6a02cf"} +{"gid":"63c39ac6-a32d-5ec6-9a25-fbb37b035300","label":"specimen_observation","from":"60c67a06-ea2d-4d24-9249-418dc77a16a9","to":"cec32723-9ede-5f24-ba63-63cb8c6a02cf"} +{"gid":"2201f601-22b6-514f-9059-ffb17ec11961","label":"subject_observation","from":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920","to":"cec32723-9ede-5f24-ba63-63cb8c6a02cf"} +{"gid":"6d4b193a-4a6d-5a0e-904f-933dd6fe1d48","label":"focus_observation","from":"60c67a06-ea2d-4d24-9249-418dc77a16a9","to":"4e3c6b59-b1fd-5c26-a611-da4cde9fd061"} +{"gid":"872b44a7-4bf6-5169-a861-d5b66328a474","label":"subject_observation","from":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920","to":"4e3c6b59-b1fd-5c26-a611-da4cde9fd061"} +{"gid":"4579f383-75d0-5cc5-9c5b-bb4e920225bc","label":"focus_observation","from":"9ae7e542-767f-4b03-a854-7ceed17152cb","to":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f"} +{"gid":"dcdd2268-d0e8-5d20-81cd-dd341e0d23cf","label":"specimen_observation","from":"60c67a06-ea2d-4d24-9249-418dc77a16a9","to":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f"} +{"gid":"7b701827-a13f-5fe9-a700-54ffb651b955","label":"subject_observation","from":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920","to":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f"} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/Observation.vertex.json b/gen3_writer/fixtures/combio-examples-grip/Observation.vertex.json new file mode 100644 index 0000000..213ed7b --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/Observation.vertex.json @@ -0,0 +1,3 @@ +{"gid":"cec32723-9ede-5f24-ba63-63cb8c6a02cf","label":"Observation","data":{"auth_resource_path":"/programs/ohsu/projects/test","category":[{"coding":[{"code":"laboratory","display":"Laboratory","system":"http://terminology.hl7.org/CodeSystem/observation-category"}]}],"code":{"coding":[{"code":"Gen3 Sequencing Metadata","display":"Gen3 Sequencing Metadata","system":"https://my_demo.org/labA"}]},"component":[{"code":{"coding":[{"code":"sequencer","display":"sequencer","system":"https://my_demo.org/labA"}],"text":"sequencer"},"valueString":"Illumina Seq 1000"},{"code":{"coding":[{"code":"index","display":"index","system":"https://my_demo.org/labA"}],"text":"index"},"valueString":"100bp Single index"},{"code":{"coding":[{"code":"type","display":"type","system":"https://my_demo.org/labA"}],"text":"type"},"valueString":"Exome"},{"code":{"coding":[{"code":"project_id","display":"project_id","system":"https://my_demo.org/labA"}],"text":"project_id"},"valueString":"labA_projectXYZ"},{"code":{"coding":[{"code":"read_length","display":"read_length","system":"https://my_demo.org/labA"}],"text":"read_length"},"valueString":"100"},{"code":{"coding":[{"code":"instrument_run_id","display":"instrument_run_id","system":"https://my_demo.org/labA"}],"text":"instrument_run_id"},"valueString":"234_ABC_1_8899"},{"code":{"coding":[{"code":"capture_bait_set","display":"capture_bait_set","system":"https://my_demo.org/labA"}],"text":"capture_bait_set"},"valueString":"Human Exom 2X"},{"code":{"coding":[{"code":"end_type","display":"end_type","system":"https://my_demo.org/labA"}],"text":"end_type"},"valueString":"Paired-End"},{"code":{"coding":[{"code":"capture","display":"capture","system":"https://my_demo.org/labA"}],"text":"capture"},"valueString":"emitter XT"},{"code":{"coding":[{"code":"sequencing_site","display":"sequencing_site","system":"https://my_demo.org/labA"}],"text":"sequencing_site"},"valueString":"AdvancedGeneExom"},{"code":{"coding":[{"code":"construction","display":"construction","system":"https://my_demo.org/labA"}],"text":"construction"},"valueString":"library_construction"}],"focus":[{"reference":"DocumentReference/9ae7e542-767f-4b03-a854-7ceed17152cb"}],"id":"cec32723-9ede-5f24-ba63-63cb8c6a02cf","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"patientX_1234-9ae7e542-767f-4b03-a854-7ceed17152cb-sequencer"}],"resourceType":"Observation","specimen":{"reference":"Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9"},"status":"final","subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"}}} +{"gid":"4e3c6b59-b1fd-5c26-a611-da4cde9fd061","label":"Observation","data":{"auth_resource_path":"/programs/ohsu/projects/test","category":[{"coding":[{"code":"laboratory","display":"Laboratory","system":"http://terminology.hl7.org/CodeSystem/observation-category"}],"text":"Laboratory"}],"code":{"coding":[{"code":"labA specimen metadata","display":"labA specimen metadata","system":"https://my_demo.org/labA"}],"text":"sample type abc"},"component":[{"code":{"coding":[{"code":"sample_type","display":"sample_type","system":"https://my_demo.org/labA"}],"text":"sample_type"},"valueString":"Primary Solid Tumor"},{"code":{"coding":[{"code":"library_id","display":"library_id","system":"https://my_demo.org/labA"}],"text":"library_id"},"valueString":"12345"},{"code":{"coding":[{"code":"tissue_type","display":"tissue_type","system":"https://my_demo.org/labA"}],"text":"tissue_type"},"valueString":"Tumor"},{"code":{"coding":[{"code":"treatments","display":"treatments","system":"https://my_demo.org/labA"}],"text":"treatments"},"valueString":"Trastuzumab"},{"code":{"coding":[{"code":"allocated_for_site","display":"allocated_for_site","system":"https://my_demo.org/labA"}],"text":"allocated_for_site"},"valueString":"TEST Clinical Research"},{"code":{"coding":[{"code":"pathology_data","display":"pathology_data","system":"https://my_demo.org/labA"}],"text":"pathology_data"}},{"code":{"coding":[{"code":"clinical_event","display":"clinical_event","system":"https://my_demo.org/labA"}],"text":"clinical_event"}},{"code":{"coding":[{"code":"indexed_collection_date","display":"indexed_collection_date","system":"https://my_demo.org/labA"}],"text":"indexed_collection_date"},"valueInteger":365},{"code":{"coding":[{"code":"biopsy_specimens_bems_id","display":"biopsy_specimens_bems_id","system":"https://my_demo.org/labA"}],"text":"biopsy_specimens"},"valueString":"specimenA, specimenB, specimenC"},{"code":{"coding":[{"code":"biopsy_procedure_type","display":"biopsy_procedure_type","system":"https://my_demo.org/labA"}],"text":"biopsy_procedure_type"},"valueString":"Biopsy - Core"},{"code":{"coding":[{"code":"biopsy_anatomical_location","display":"biopsy_anatomical_location","system":"https://my_demo.org/labA"}],"text":"biopsy_anatomical_location"},"valueString":"top axillary lymph node"},{"code":{"coding":[{"code":"percent_tumor","display":"percent_tumor","system":"https://my_demo.org/labA"}],"text":"percent_tumor"},"valueString":"30"}],"focus":[{"reference":"Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9"}],"id":"4e3c6b59-b1fd-5c26-a611-da4cde9fd061","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"patientX_1234-specimen_1234_labA-sample_type"}],"resourceType":"Observation","status":"final","subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"}}} +{"gid":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f","label":"Observation","data":{"auth_resource_path":"/programs/ohsu/projects/test","category":[{"coding":[{"code":"laboratory","display":"Laboratory","system":"http://terminology.hl7.org/CodeSystem/observation-category"}]}],"code":{"coding":[{"code":"81247-9","display":"Genomic structural variant copy number","system":"https://loinc.org"}]},"component":[{"code":{"coding":[{"code":"Gene","display":"Gene","system":"https://my_demo.org/labA"}],"text":"Gene"},"valueString":"TP53"},{"code":{"coding":[{"code":"Chromosome","display":"Chromosome","system":"https://my_demo.org/labA"}],"text":"Chromosome"},"valueString":"chr17"},{"code":{"coding":[{"code":"result","display":"result","system":"https://my_demo.org/labA"}],"text":"result"},"valueString":"gain of function (GOF)"}],"focus":[{"reference":"DocumentReference/9ae7e542-767f-4b03-a854-7ceed17152cb"}],"id":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"patientX_1234-9ae7e542-767f-4b03-a854-7ceed17152cb-Gene"}],"resourceType":"Observation","specimen":{"reference":"Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9"},"status":"final","subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"}}} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/Organization.vertex.json b/gen3_writer/fixtures/combio-examples-grip/Organization.vertex.json new file mode 100644 index 0000000..4fae180 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/Organization.vertex.json @@ -0,0 +1 @@ +{"gid":"89c8dc4c-2d9c-48c7-8862-241a49a78f14","label":"Organization","data":{"auth_resource_path":"/programs/ohsu/projects/test","id":"89c8dc4c-2d9c-48c7-8862-241a49a78f14","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"LabA_ORGANIZATION"}],"resourceType":"Organization","type":[{"coding":[{"code":"prov","display":"Healthcare Provider","system":"http://terminology.hl7.org/CodeSystem/organization-type"}],"text":"An organization that provides healthcare services."},{"coding":[{"code":"edu","display":"Educational Institute","system":"http://terminology.hl7.org/CodeSystem/organization-type"}],"text":"An educational institution that provides education or research facilities."}]}} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/Patient.vertex.json b/gen3_writer/fixtures/combio-examples-grip/Patient.vertex.json new file mode 100644 index 0000000..b070579 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/Patient.vertex.json @@ -0,0 +1 @@ +{"gid":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920","label":"Patient","data":{"active":true,"auth_resource_path":"/programs/ohsu/projects/test","id":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"patientX_1234"}],"resourceType":"Patient"}} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/ResearchStudy.vertex.json b/gen3_writer/fixtures/combio-examples-grip/ResearchStudy.vertex.json new file mode 100644 index 0000000..c3586b4 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/ResearchStudy.vertex.json @@ -0,0 +1 @@ +{"gid":"7dacd4d0-3c8e-470b-bf61-103891627d45","label":"ResearchStudy","data":{"auth_resource_path":"/programs/ohsu/projects/test","description":"LabA Clinical Trial Study: FHIR Schema Chorot Integration","id":"7dacd4d0-3c8e-470b-bf61-103891627d45","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"labA"}],"name":"LabA","resourceType":"ResearchStudy","status":"active"}} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.in.edge.json b/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.in.edge.json new file mode 100644 index 0000000..563e993 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.in.edge.json @@ -0,0 +1,2 @@ +{"gid":"9eb4d599-8963-56f0-ab37-991e6dd12b1f","label":"study","from":"2fc448d6-a23b-4b94-974b-c66110164851","to":"7dacd4d0-3c8e-470b-bf61-103891627d45"} +{"gid":"50aa2059-b71e-5d48-a1fe-6e6465257133","label":"subject_Patient","from":"2fc448d6-a23b-4b94-974b-c66110164851","to":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920"} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.out.edge.json b/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.out.edge.json new file mode 100644 index 0000000..e5081c9 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.out.edge.json @@ -0,0 +1,2 @@ +{"gid":"3fdd2f49-409b-5677-9687-71d1c3d8c63f","label":"research_subject","from":"7dacd4d0-3c8e-470b-bf61-103891627d45","to":"2fc448d6-a23b-4b94-974b-c66110164851"} +{"gid":"b633871f-f369-507c-8b93-21dc1eb1eb63","label":"research_subject","from":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920","to":"2fc448d6-a23b-4b94-974b-c66110164851"} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.vertex.json b/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.vertex.json new file mode 100644 index 0000000..d84bacd --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/ResearchSubject.vertex.json @@ -0,0 +1 @@ +{"gid":"2fc448d6-a23b-4b94-974b-c66110164851","label":"ResearchSubject","data":{"auth_resource_path":"/programs/ohsu/projects/test","id":"2fc448d6-a23b-4b94-974b-c66110164851","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"subjectX_1234"}],"resourceType":"ResearchSubject","status":"active","subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"}}} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/Specimen.in.edge.json b/gen3_writer/fixtures/combio-examples-grip/Specimen.in.edge.json new file mode 100644 index 0000000..f8179ce --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/Specimen.in.edge.json @@ -0,0 +1 @@ +{"gid":"e02b3a82-40d1-5252-bf63-73db4472306f","label":"subject_Patient","from":"60c67a06-ea2d-4d24-9249-418dc77a16a9","to":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920"} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/Specimen.out.edge.json b/gen3_writer/fixtures/combio-examples-grip/Specimen.out.edge.json new file mode 100644 index 0000000..5ef3962 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/Specimen.out.edge.json @@ -0,0 +1 @@ +{"gid":"45ef3f12-11f5-5739-9c86-80711d158653","label":"specimen","from":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920","to":"60c67a06-ea2d-4d24-9249-418dc77a16a9"} \ No newline at end of file diff --git a/gen3_writer/fixtures/combio-examples-grip/Specimen.vertex.json b/gen3_writer/fixtures/combio-examples-grip/Specimen.vertex.json new file mode 100644 index 0000000..49118a5 --- /dev/null +++ b/gen3_writer/fixtures/combio-examples-grip/Specimen.vertex.json @@ -0,0 +1 @@ +{"gid":"60c67a06-ea2d-4d24-9249-418dc77a16a9","label":"Specimen","data":{"auth_resource_path":"/programs/ohsu/projects/test","collection":{"bodySite":{"concept":{"coding":[{"code":"76752008","display":"Breast","system":"http://snomed.info/sct"}],"text":"Breast"}},"collector":{"reference":"Organization/89c8dc4c-2d9c-48c7-8862-241a49a78f14"}},"id":"60c67a06-ea2d-4d24-9249-418dc77a16a9","identifier":[{"system":"https://my_demo.org/labA","use":"official","value":"specimen_1234_labA"}],"processing":[{"method":{"coding":[{"code":"117032008","display":"Spun specimen (procedure)","system":"http://snomed.info/sct"},{"code":"Double-Spun","display":"Double-Spun","system":"https://my_demo.org/labA"}],"text":"Spun specimen (procedure)"}}],"resourceType":"Specimen","subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"}}} \ No newline at end of file diff --git a/gen3_writer/gen3_test.go b/gen3_writer/gen3_test.go index 8af4496..1d0b6cb 100644 --- a/gen3_writer/gen3_test.go +++ b/gen3_writer/gen3_test.go @@ -1,29 +1,71 @@ package main -//grip query synthea 'V().hasLabel("DocumentReference").out("subject")' -/*documentReference (filter:$filter) { - subject{ - id - } -}*/ import ( "bytes" "encoding/json" + "fmt" + "io" + "mime/multipart" "net/http" - "os/exec" - "reflect" + "os" + "path/filepath" "strings" "testing" + "time" + + "github.com/golang-jwt/jwt/v5" ) -func HTTP_REQUEST(graph_name string, url string, payload []byte, t *testing.T) (response_json map[string]any, status bool) { - req, err := http.NewRequest("POST", url+graph_name, bytes.NewBuffer(payload)) +type Request struct { + url string + method string + headers map[string]any + body []byte +} + +func createToken(expired bool, writer bool, reader bool) string { + var create string + timeNow := time.Now() + time_exp := timeNow + if !expired { + time_exp = timeNow.Add(time.Minute * 20) + } + if writer { + create = "create" + } + if reader { + create = "reader" + } + if writer && reader { + create = "create-reader" + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, + jwt.MapClaims{ + "iss": "https://foobar-domain/user", + "username": "foobar-user", + "iat": timeNow.Unix(), + "exp": time_exp.Unix(), + "jti": "foobbar-jti", + "sub": create, + }) + tokenString, err := token.SignedString([]byte("foo-bar-signature")) + if err != nil { + fmt.Println("Error creating signed string: ", err) + } + return tokenString +} + +func TemplateRequest(request *Request, t *testing.T) (response_json map[string]any, status bool) { + /* A templating function that inserts all of the arguments that would are needed to do an http request */ + + req, err := http.NewRequest(request.method, request.url, bytes.NewBuffer(request.body)) if err != nil { t.Error("Error creating request:", err) return } - - req.Header.Set("Content-Type", "application/json") + for key, val := range request.headers { + req.Header.Set(key, val.(string)) + } client := &http.Client{} resp, err := client.Do(req) @@ -44,251 +86,268 @@ func HTTP_REQUEST(graph_name string, url string, payload []byte, t *testing.T) ( var data map[string]interface{} errors := json.Unmarshal([]byte(buf.String()), &data) - t.Log("DATA: ", data) if errors != nil { - t.Error("Error:", errors) + t.Error("Error: ", errors) return nil, false } return data, true } -func Test_Filters(t *testing.T) { - tests := []struct { - name string - }{ - {name: "Slider and CheckBox"}, - {name: "Aggregation and Filter"}, - {name: "Combo_test"}, - {name: "NullOps"}, - {name: "GraphQL_NullOps"}, - {name: "NullOP_Results"}, - } - for _, tt := range tests { - if tt.name == "Slider and CheckBox" { - t.Run(tt.name, func(t *testing.T) { - payload := []byte(`{ - "query": "query ($filter: JSON) {\n patient(filter: $filter) {\n quality_adjusted_life_years_valueDecimal\n maritalStatus\n }\n}\n", - "variables": { - "filter": { - "AND": [ - { - "AND": [ - { - ">=": { - "quality_adjusted_life_years_valueDecimal": 66 - } - }, - { - "<=": { - "quality_adjusted_life_years_valueDecimal": 70 - } - } - ] - }, - { - "IN": { - "maritalStatus": [ - "M" - ] - } - } - ] - } - } - }`) - data, status := HTTP_REQUEST("synthea", "http://localhost:8201/api/graphql/", payload, t) - if status == false { - t.Error("HTTP Request failed") - } - if data, ok := data["data"].(map[string]any); ok { - if data, ok := data["patient"]; ok { - if data, ok := data.([]any); ok { - for _, value := range data { - if data, ok := value.(map[string]any); ok { - if upper_bound, ok := data["quality_adjusted_life_years_valueDecimal"].(float64); ok { - if data["maritalStatus"] == "M" && upper_bound >= 66 && upper_bound <= 70 { - continue - } else { - t.Error("Row Has failed filter") - } - } else { - t.Error("Row has failed type check") - } - } - } - } - } - } - - }) - } - if tt.name == "Aggregation and Filter" { - payload := []byte(`{ - "query": "query ($filter: JSON) {\n _aggregation {\n observation (filter: $filter) {\n code {\n histogram {\n count\n key\n }\n }\n\n }}\n}\n", - "variables": { - "filter": { - "AND": [ - { - "IN": { - "code": [ - "Creatinine" - ] - } - } - ] - } - } - }`) - data, status := HTTP_REQUEST("synthea", "http://localhost:8201/api/graphql/", payload, t) - if status == false { - t.Error("test failed") - } - if data, ok := data["data"].(map[string]any)["_aggregation"].(map[string]any)["observation"].(map[string]any)["code"].(map[string]any)["histogram"].([]any); ok { - for _, values := range data { - key := values.(map[string]any)["key"].(string) - count := values.(map[string]any)["count"].(float64) - if key != `string_value:"Creatinine"` || count != 5377 { - t.Error("Aggregation test failed. Did data change?") - } - } - } else { - t.Error("indexing failed. Did query change?") - } - } - if tt.name == "Combo_test" { - payload := []byte(`{ - "query": "query ($filter: JSON) {\n _aggregation {\n documentReference {\n category {\n histogram {\n count\n key\n }\n }\n }\n }\n observation(filter: $filter) {\n subject\n }\n}\n", - "variables": { - "filter": { - "AND": [ - { - "IN": { - "subject": [ - "Patient/5b13b8fc-f387-4a95-bb80-5c22eeed7697" - ] - } - } - ] - } - } - }`) - data, status := HTTP_REQUEST("synthea", "http://localhost:8201/api/graphql/", payload, t) - if status == false { - t.Error("test failed on HTTP Request") - } - if data, ok := data["data"].(map[string]any); ok { - if aggregation, ok := data["_aggregation"].(map[string]any)["documentReference"].(map[string]any)["category"].(map[string]any)["histogram"].([]any); ok { - for i, values := range aggregation { - if map_values, ok := values.(map[string]any); ok { - t.Log("MAP VALUES: ", map_values) - switch i { - case 0: - if map_values["key"].(string) == `string_value:"Clinical Note"` && map_values["count"].(float64) == 37378 { - continue - } else { - t.Error("test failed, values don't match") - } - case 1: - if map_values["key"].(string) == `string_value:"Image"` && map_values["count"].(float64) == 125 { - continue - } else { - t.Error("test failed, values don't match") - } - case 2: - if map_values["key"].(string) == `string_value:"Cancer related multigene analysis Molgen Doc (cfDNA)"` && map_values["count"].(float64) == 9 { - continue - } else { - t.Error("test failed, values don't match") - } - } - } - } - if res, ok := data["observation"].([]any); ok { - for _, val := range res { - t.Log("INFO: ", val) - if val.(map[string]any)["subject"] != "Patient/5b13b8fc-f387-4a95-bb80-5c22eeed7697" { - t.Error("filter test failed, values don't match") - } - } - } - } +func bulkLoad(url, directoryPath, accessToken string) (error, []map[string]any) { + files, _ := os.ReadDir(directoryPath) + var allData []map[string]any + for _, file := range files { + if !file.IsDir() && (filepath.Ext(file.Name()) == ".json" || filepath.Ext(file.Name()) == ".gz" || filepath.Ext(file.Name()) == ".ndjson") { + filePath := filepath.Join(directoryPath, file.Name()) + graphComponent := "vertex" + if strings.Contains(filePath, "edge") { + graphComponent = "edge" } - } - if tt.name == "NullOps" { - cmd := exec.Command("grip", "query", "outNull", `V(["875e3325-c2ad-4d63-82ad-be8432bd415b","1842609e-7a40-4ba3-8a82-2fa061fcf30f"]).outNull("subject_Patient")`) - output, err := cmd.Output() + file, _ := os.Open(filePath) + defer file.Close() + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, _ := writer.CreateFormFile("file", filepath.Base(filePath)) + io.Copy(part, file) + writer.WriteField("types", graphComponent) + writer.Close() + + req, _ := http.NewRequest("POST", url, body) + req.Header.Set("Authorization", accessToken) + req.Header.Set("Content-Type", writer.FormDataContentType()) + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + var err error + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(resp.Body) if err != nil { - t.Error("Error:", err) + return err, nil } - json_string := strings.Split(string(output), "\n") - var jsonMap map[string]interface{} - var NullOp map[string]interface{} + var data map[string]interface{} + errors := json.Unmarshal([]byte(buf.String()), &data) + if errors != nil { + return errors, nil + } + allData = append(allData, data) + } + } + return nil, allData +} - json.Unmarshal([]byte(json_string[0]), &jsonMap) - json.Unmarshal([]byte(json_string[1]), &NullOp) +func Test_Load_Ok(t *testing.T) { + req := &Request{ + url: "http://localhost:8201/graphql/add-graph/TEST/ohsu-test", + method: "POST", + headers: map[string]any{"Authorization": createToken(false, true, true)}, + } + t.Run("create_load_graph", func(t *testing.T) { + response_json, pass := TemplateRequest(req, t) + if !pass { + t.Error("status is not 200") + } + t.Log("RESPONSE JSON: ", response_json, "STATUS: ", pass) + }) - if val, ok := NullOp["vertex"]; ok { - if val == nil || reflect.DeepEqual(val, reflect.Zero(reflect.TypeOf(val)).Interface()) { - t.Error("Null Op Test Failed", val) - } - } + err, responses := bulkLoad("http://localhost:8201/graphql/TEST/bulk-load/ohsu-test", + "fixtures/combio-examples-grip", + createToken(false, true, true), + ) + if err != nil { + t.Error(err) + } + for _, resp := range responses { + if resp["status"].(float64) != 200 { + t.Error(resp) + } else { + t.Log(resp) + } + } +} - //t.Log("NULL OP", NullOp) +func Test_Load_Malformed_Token(t *testing.T) { + /* Server returns a 400 given an unparsable token */ + err, responses := bulkLoad("http://localhost:8201/graphql/TEST/bulk-load/ohsu-test", + "fixtures/combio-examples-grip", + createToken(false, true, true)[2:50], + ) + if err != nil { + t.Error(err) + } + for _, resp := range responses { + if resp["status"].(float64) != 400 { + t.Error(resp) + } else { + t.Log(resp) } - if tt.name == "GraphQL_NullOps" { - payload := []byte(`{ - "query": "query ($filter: JSON) {\n documentReference (filter:$filter, first: 1) {\n file_name\n subject {\n id\n birthDate\n subject_observation {\n code\n }\n } \n }\n}\n", - "variables": {} - }`) - data, status := HTTP_REQUEST("synthea", "http://localhost:8201/api/graphql/", payload, t) - if status == false { - t.Error("test failed on HTTP Request") - } - if data, ok := data["data"].(map[string]any)["documentReference"].([]any); ok { - if data, ok := data[0].(map[string]any); ok { - t.Log("CHECK: ", data["file_name"] == "output/clinical_reports/53fefa32-fcbb-4ff8-8a92-55ee120877b7") - if data["file_name"] != "output/clinical_reports/53fefa32-fcbb-4ff8-8a92-55ee120877b7" { - t.Error() - } - if data, ok := data["subject"].([]any); ok { - if data, ok := data[0].(map[string]any); ok { - t.Log("CHECK: ", data["birthDate"] == "1913-10-29" || data["id"] != "fb60e763-e799-4d59-82a3-66977cc6696c") - if data["birthDate"] != "1913-10-29" || data["id"] != "fb60e763-e799-4d59-82a3-66977cc6696c" { - t.Error() - } - if data, ok := data["subject_observation"].([]any); ok { - if data, ok := data[0].(map[string]any); ok { - t.Log("CHECK: ", data["code"] == "Bilirubin.total [Mass/volume] in Serum or Plasma") - if data["code"] != "Bilirubin.total [Mass/volume] in Serum or Plasma" { - t.Error() - } - - } - } - } - } - } - } + } +} + +func Test_Load_Expired_Token(t *testing.T) { + /* Server returns a 401 given an expired token */ + err, responses := bulkLoad("http://localhost:8201/graphql/TEST/bulk-load/ohsu-test", + "fixtures/combio-examples-grip", + createToken(true, true, true), + ) + if err != nil { + t.Error(err) + } + for _, resp := range responses { + if resp["status"].(float64) != 401 { + t.Error(resp) + } else { + t.Log(resp) } - if tt.name == "NullOP_Results" { - payload := []byte(`{ - "query": "query ($filter: JSON) {\n documentReference (filter:$filter, first: 7) {\n file_name\n subject {\n id\n birthDate\n subject_observation {\n code\n }\n } \n }\n}\n", - "variables": {} - }`) - data, status := HTTP_REQUEST("synthea", "http://localhost:8201/api/graphql/", payload, t) - if status == false { - t.Error("test failed on HTTP Request") - } - if data, ok := data["data"].(map[string]any)["documentReference"].([]any); ok { - if len(data) != 7 { - t.Error("Unexpected output length") - } - } else { - t.Error("Unexpected output structure") - } + } +} + +func Test_Load_No_Writer_Perms(t *testing.T) { + /* Server returns a 403 given a token that respresents a user with no writer perms on the specified project */ + err, responses := bulkLoad("http://localhost:8201/graphql/TEST/bulk-load/ohsu-test", + "fixtures/combio-examples-grip", + createToken(false, false, true), + ) + if err != nil { + t.Error(err) + } + for _, resp := range responses { + if resp["status"].(float64) != 403 { + t.Error(resp) + } else { + t.Log(resp) + } + } +} + +func Test_Get_Vertex_No_Reader_Perms(t *testing.T) { + /* Server returns a 403 given a token that respresents a user with no writer perms on the specified project */ + req := &Request{ + url: "http://localhost:8201/graphql/TEST/get-vertex/cec32723-9ede-5f24-ba63-63cb8c6a02cf/ohsu-test", + method: "GET", + headers: map[string]any{"Authorization": createToken(false, true, false)}, + } + + response, status := TemplateRequest(req, t) + if !(response["status"].(float64) == 403) || !status { + t.Error(response) + } + t.Log(status, response) +} + +func Test_Get_Vertex_Ok(t *testing.T) { + /* given appropriate perms, edge should be retrieved */ + req := &Request{ + url: "http://localhost:8201/graphql/TEST/get-vertex/cec32723-9ede-5f24-ba63-63cb8c6a02cf/ohsu-test", + method: "GET", + headers: map[string]any{"Authorization": createToken(false, false, true)}, + } + response, status := TemplateRequest(req, t) + if !(response["status"].(float64) == 200) || !status { + t.Error(response) + } + t.Log(status, response) +} + +func Test_Delete_Edge_Ok(t *testing.T) { + /* Regular delete should return 200 */ + req := &Request{ + url: "http://localhost:8201/graphql/TEST/del-edge/e2867c6d-db7e-5d6e-87d8-b1c293f5b47e/ohsu-test", + method: "DELETE", + headers: map[string]any{"Authorization": createToken(false, true, true)}, + } + response, status := TemplateRequest(req, t) + if !(response["status"].(float64) == 200) || !status { + t.Error(response) + } + t.Log(status, response) + +} + +func Test_Graphql_Query_Forbidden_Perms(t *testing.T) { + /* Testing do a query, but with a token that doesn't have read perms */ + payload := []byte(`{ + "query": "query PatientIdsWithSpecimenEdge { PatientIdsWithSpecimenEdge { id }}", + "variables": { + "limit": 1000 + } + }`) + + req := &Request{ + url: "http://localhost:8201/reader/api", + method: "POST", + headers: map[string]any{ + "Authorization": createToken(false, false, false), + "Content-Type": "application/json"}, + body: payload, + } + response, status := TemplateRequest(req, t) + if !(response["StatusCode"].(float64) == 403) { + t.Error(response) + } + t.Log(status, response) +} +func Test_Graphql_Query_Proj(t *testing.T) { + /* A basic test for Graphql style query with mock auth */ + payload := []byte(`{ + "query": "query PatientIdsWithSpecimenEdge { PatientIdsWithSpecimenEdge { id }}", + "variables": { + "limit": 1000 + } + }`) + + req := &Request{ + url: "http://localhost:8201/reader/api", + method: "POST", + headers: map[string]any{ + "Authorization": createToken(false, false, true), + "Content-Type": "application/json"}, + body: payload, + } + + response, status := TemplateRequest(req, t) + if !status { + t.Error("Status returned false: ", response) + } + correct_response, ok := response["data"].(map[string]any)["PatientIdsWithSpecimenEdge"].([]any) + if !ok { + t.Error("Response not indexable for 'data' and/or 'PatientIdsWithSpecimenEdge' keys") + } + for _, resp := range correct_response { + val, ok := resp.(map[string]any)["id"].(string) + if !ok { + t.Error("Response not indexable on 'id': ", resp) } + t.Log("Return VAL: ", val) + } +} + +func Test_Delete_Proj(t *testing.T) { + /* Delete Everything from test graph project ohsu-test. Should return 200 */ + req := &Request{ + url: "http://localhost:8201/graphql/TEST/proj-delete/ohsu-test", + method: "DELETE", + headers: map[string]any{"Authorization": createToken(false, true, true)}, + } + response, status := TemplateRequest(req, t) + if !(response["status"].(float64) == 200) { + t.Error(response) + } + t.Log(status, response) + + // Look for the vertex that was retrieved earlier. It should be gone + req = &Request{ + url: "http://localhost:8201/graphql/TEST/get-vertex/cec32723-9ede-5f24-ba63-63cb8c6a02cf/ohsu-test", + method: "GET", + headers: map[string]any{"Authorization": createToken(false, false, true)}, + } + response, status = TemplateRequest(req, t) + if !(response["status"].(float64) == 404) { + t.Error(response) } + t.Log("RESPONSE: ", response) } diff --git a/gen3_writer/handler.go b/gen3_writer/handler.go index a84698d..ba855bf 100644 --- a/gen3_writer/handler.go +++ b/gen3_writer/handler.go @@ -58,17 +58,17 @@ func ParseAccess(c *gin.Context, resourceList []string, resource string, method resource matches the allowable list of resource types for the provided method */ if len(resourceList) == 0 { - return &middleware.ServerError{StatusCode: 401, Message: fmt.Sprintf("User is not allowed to %s on any resource path", method)} + return &middleware.ServerError{StatusCode: 403, Message: fmt.Sprintf("User is not allowed to %s on any resource path", method)} } for _, v := range resourceList { if resource == v { return nil } } - return &middleware.ServerError{StatusCode: 401, Message: fmt.Sprintf("User is not allowed to %s on resource path: %s", method, resource)} + return &middleware.ServerError{StatusCode: 403, Message: fmt.Sprintf("User is not allowed to %s on resource path: %s", method, resource)} } -func TokenAuthMiddleware() gin.HandlerFunc { +func TokenAuthMiddleware(jwtHandler middleware.JWTHandler) gin.HandlerFunc { /* Authentication middleware function. Maps HTTP method to expected permssions. If user permissions don't match, abort command and return 401 */ @@ -95,7 +95,7 @@ func TokenAuthMiddleware() gin.HandlerFunc { return } - anyList, err := middleware.HandleJWTToken(Token, method) + anyList, err := jwtHandler.HandleJWTToken(Token, method) if err != nil { RegError(c, c.Writer, c.Param("graph"), err) return @@ -122,8 +122,14 @@ func TokenAuthMiddleware() gin.HandlerFunc { } func NewHTTPHandler(client gripql.Client, config map[string]string) (http.Handler, error) { - // Including below line to run in prod mode + // Need a way to toggle on mock auth + var mware middleware.JWTHandler = &middleware.ProdJWTHandler{} + if config["test"] == "true" { + mware = &middleware.MockJWTHandler{} + } + + // Including below line to run in prod mode gin.SetMode(gin.ReleaseMode) r := gin.New() @@ -163,18 +169,18 @@ func NewHTTPHandler(client gripql.Client, config map[string]string) (http.Handle r.GET("_status", func(c *gin.Context) { Response(c, c.Writer, "", 200, 200, fmt.Sprintf("[200] healthy _status")) }) + r.POST("/add-graph/:graph/:project-id", TokenAuthMiddleware(mware), func(c *gin.Context) { + h.AddGraph(c) + }) g := r.Group(":graph") - g.Use(TokenAuthMiddleware()) + g.Use(TokenAuthMiddleware(mware)) g.POST("/add-vertex/:project-id", func(c *gin.Context) { h.WriteVertex(c) }) g.POST("/add-edge/:project-id", func(c *gin.Context) { h.WriteEdge(c) }) - g.POST("/add-graph/:project-id", func(c *gin.Context) { - h.AddGraph(c) - }) g.POST("/mongo-load/:project-id", func(c *gin.Context) { h.MongoBulk(c) }) @@ -365,6 +371,10 @@ func (gh *Handler) GetVertex(c *gin.Context, vertex string) { writer, _, graph := getFields(c) gql_vertex, err := gh.client.GetVertex(graph, vertex) if err != nil { + if strings.Contains(err.Error(), "rpc error: code = NotFound") { + RegError(c, writer, graph, &middleware.ServerError{StatusCode: 404, Message: fmt.Sprintf("%s", err)}) + return + } RegError(c, writer, graph, GetInternalServerErr(err)) return } @@ -375,6 +385,10 @@ func (gh *Handler) GetEdge(c *gin.Context, edge string) { writer, _, graph := getFields(c) gql_edge, err := gh.client.GetEdge(graph, edge) if err != nil { + if strings.Contains(err.Error(), "rpc error: code = NotFound") { + RegError(c, writer, graph, &middleware.ServerError{StatusCode: 404, Message: fmt.Sprintf("%s", err)}) + return + } RegError(c, writer, graph, GetInternalServerErr(err)) return } diff --git a/graphql_gen3/README.md b/graphql_gen3/README.md index d5187a1..626635a 100644 --- a/graphql_gen3/README.md +++ b/graphql_gen3/README.md @@ -1,29 +1,38 @@ -# Graphql Grip Endpoint Gen3 Installation instructions +This directory is a legacy graphql reader that Isn't currently operational. Below are some archive docs. + +# Graphql Grip Endpoint Legacy Deveveloment Setup Instructions + +These instructions show how to load data into grip before there was an ETL image that could be run with g3t. + In addition to cloning this repo you will also need to have a running [gen3 helm deployment](https://github.com/ACED-IDP/gen3-helm/tree/feature/grip). Once you have followed the gen3helm deployment instructions, and have running grip and mongodb pods you will need to exec into the grip pod to load the data into mongo and start the server: Get a list of all running pods to make sure grip pod is running + ``` kubectl get pods ``` copy the config, data, and files into the grip pod with: + ``` kubectl cp graphql_gen3.so local-grip-your_unique_hash:/data kubectl cp mongo.yml local-grip-your_unique_hash:/data ``` + The shared object file should have been built with the image and should already be in /data -Exec into grip pod with: +Exec into grip pod with: + ``` kubectl exec --stdin --tty deployment/local-grip -- /bin/bash cd data grip server -w api/graphql=graphql_gen3.so -c mongo.yml ``` -Create a new tab and exec into the same pod with the same command above, then run the below commands to +Create a new tab and exec into the same pod with the same command above, then run the below commands to import data into mongo, generate a schema from the populated data in mongo and post it to the graphql endpoint: ``` @@ -34,4 +43,5 @@ grip server load --vertex output/DocumentReference_new.ndjson grip schema sample synthea2 > synthea2.schema.json grip schema post --json synthea2.schema.json ``` -Note: output/ is the directory that contains the bare minimum 3 vertex data files that are needed to display data on the exploration page. \ No newline at end of file + +Note: output/ is the directory that contains the bare minimum 3 vertex data files that are needed to display data on the exploration page. diff --git a/gripgraphql/handler.go b/gripgraphql/handler.go index a7a5260..959c9a5 100644 --- a/gripgraphql/handler.go +++ b/gripgraphql/handler.go @@ -43,6 +43,7 @@ type GraphQLJS struct { client gripql.Client gjHandler *handler.Handler Pool sync.Pool + cw *JSClientWrapper //Once sync.Once } @@ -297,7 +298,7 @@ func NewHTTPHandler(client gripql.Client, config map[string]string) (http.Handle hnd = handler.New(&handler.Config{ Schema: schema, }) - gh := &GraphQLJS{client: client, gjHandler: hnd} + gh := &GraphQLJS{client: client, gjHandler: hnd, cw: jsClient} return gh }, } @@ -322,10 +323,14 @@ func (gh *GraphQLJS) ServeHTTP(writer http.ResponseWriter, request *http.Request requestHeaders := request.Header ctx := context.WithValue(context.Background(), "Header", requestHeaders) + var jwtHandler middleware.JWTHandler = &middleware.ProdJWTHandler{} + if gh.cw.graph == "TEST" { + jwtHandler = &middleware.MockJWTHandler{} + } //fmt.Println("REQUEST HEADERS:::: +++++++++++++++++++", requestHeaders) if val, ok := requestHeaders["Authorization"]; ok { Token := val[0] - resourceList, err := middleware.HandleJWTToken(Token, "read") + resourceList, err := jwtHandler.HandleJWTToken(Token, "read") //resourceList := []any{"/programs/cbds/projects/demo", "/programs/cbds/projects/welcome", "/programs/synthea/projects/test"} if err != nil { middleware.HandleError(err, writer) @@ -333,11 +338,9 @@ func (gh *GraphQLJS) ServeHTTP(writer http.ResponseWriter, request *http.Request } if len(resourceList) == 0 || err != nil { - //fmt.Println("_+_+_+__+_+_+__+_+_+_+_+_+_+_+_+_+_+_+_+_+_", err) if len(resourceList) == 0 { - err = &middleware.ServerError{StatusCode: http.StatusUnauthorized, Message: "resource list is len 0 or error has occured"} + err = &middleware.ServerError{StatusCode: http.StatusForbidden, Message: "User does not have access to any projects"} } - middleware.HandleError(err, writer) return err } diff --git a/gripgraphql/js_client.go b/gripgraphql/js_client.go index e924e1a..5567c1e 100644 --- a/gripgraphql/js_client.go +++ b/gripgraphql/js_client.go @@ -145,7 +145,6 @@ func (cw *JSClientWrapper) ToList(args goja.Value) goja.Value { out = append(out, cw.vm.ToValue(toInterface(row))) } - //fmt.Printf("EXIT TOLIST FUNCTION", out) return cw.vm.ToValue(out) } diff --git a/middleware/gen3_caching.go b/middleware/gen3_caching.go index 24c78da..8491c20 100644 --- a/middleware/gen3_caching.go +++ b/middleware/gen3_caching.go @@ -40,7 +40,56 @@ func GetExpiration(tokenString string) (time.Time, error) { return time.Time{}, fmt.Errorf("Expiration field 'exp' type float64 not found in token %s", token) } -func HandleJWTToken(token string, perm_method string) ([]any, error) { +type ProdJWTHandler struct{} +type JWTHandler interface { + HandleJWTToken(token, method string) ([]interface{}, error) +} +type MockJWTHandler struct{} + +func (m *MockJWTHandler) decodeToken(tokenString string) (jwt.MapClaims, error) { + /* A Mock token decoder designed to decode mock tokens for testing purposes only */ + token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return []byte("foo-bar-signature"), nil + }) + if err != nil { + return nil, err + } + if claims, ok := token.Claims.(jwt.MapClaims); ok && token.Valid { + return claims, nil + } + return nil, fmt.Errorf("invalid token") +} + +func (m *MockJWTHandler) HandleJWTToken(token string, perm_method string) ([]any, error) { + expiration, err := GetExpiration(token) + if err != nil { + return nil, &ServerError{StatusCode: 400, Message: fmt.Sprintf("%s Failed to get expiration from token %s", err, token)} + } + + if expiration.After(time.Now()) { + claims, err := m.decodeToken(token) + if err != nil { + return nil, &ServerError{StatusCode: 400, Message: fmt.Sprintf("failed to parse token data %#v", token)} + } + subject, err := claims.GetSubject() + if err != nil { + return nil, &ServerError{StatusCode: 400, Message: fmt.Sprintf("failed to parse token claims data %#v", claims)} + } + fmt.Println("SUBJECT:", subject, "") + if ((subject == "create-reader" || subject == "create") && perm_method == "create") || + ((subject == "reader" || subject == "create-reader") && perm_method == "read") { + return []any{"/programs/ohsu/projects/test"}, nil + } + + return []any{}, nil + } + return nil, &ServerError{StatusCode: 401, Message: fmt.Sprintf("token %s has expired %s", token, expiration)} +} + +func (j *ProdJWTHandler) HandleJWTToken(token string, perm_method string) ([]any, error) { cachedData, found := jwtCache.Get(token) // If cache hit check expiration and return resourceList @@ -50,8 +99,6 @@ func HandleJWTToken(token string, perm_method string) ([]any, error) { return nil, &ServerError{StatusCode: 400, Message: fmt.Sprintf("failed to parse token data %#v", cachedData)} } - fmt.Println("expiration:", tokenData.Expiration) - if tokenData.Expiration.After(time.Now()) { log.Infoln("Retrieved Cached token") return tokenData.ResourceList, nil @@ -63,7 +110,7 @@ func HandleJWTToken(token string, perm_method string) ([]any, error) { // Otherise check expiration, add token to cache and return resourceList expiration, err := GetExpiration(token) if err != nil { - return nil, &ServerError{StatusCode: 400, Message: fmt.Sprintf("Failed to get expiration from token %s", token)} + return nil, &ServerError{StatusCode: 400, Message: fmt.Sprintf("%s Failed to get expiration from token %s", err, token)} } if expiration.After(time.Now()) { From 53cd084967dc24626aadc0f9d5e80b43a28378c4 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 12 Sep 2024 08:55:21 -0700 Subject: [PATCH 2/3] Adds bulk data fetcher for projects --- gen3_writer/gen3_test.go | 58 ++++++++++++++++++++++++++++++++++++++++ gen3_writer/handler.go | 36 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/gen3_writer/gen3_test.go b/gen3_writer/gen3_test.go index 1d0b6cb..67fc857 100644 --- a/gen3_writer/gen3_test.go +++ b/gen3_writer/gen3_test.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "bytes" "encoding/json" "fmt" @@ -13,7 +14,9 @@ import ( "testing" "time" + "github.com/bmeg/grip/gripql" "github.com/golang-jwt/jwt/v5" + "google.golang.org/protobuf/encoding/protojson" ) type Request struct { @@ -253,6 +256,61 @@ func Test_Get_Vertex_Ok(t *testing.T) { t.Log(status, response) } +func Test_Get_Project_Vertices_Ok(t *testing.T) { + req := &Request{ + url: "http://localhost:8201/graphql/TEST/get-vertices/ohsu-test", + method: "GET", + headers: map[string]any{"Authorization": createToken(false, false, true)}, + } + + request, err := http.NewRequest(req.method, req.url, bytes.NewBuffer(req.body)) + if err != nil { + t.Error("Error creating request:", err) + return + } + for key, val := range req.headers { + request.Header.Set(key, val.(string)) + } + + client := &http.Client{} + resp, err := client.Do(request) + if err != nil { + t.Error("Error sending request:", err) + } + t.Log("RESP: ") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Logf("server responded with status: %s", resp.Status) + } + + reader := bufio.NewReader(resp.Body) + jum := protojson.UnmarshalOptions{DiscardUnknown: true} + for { + line, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + t.Errorf("error reading response: %s", err) + } + v := &gripql.Vertex{} + err = jum.Unmarshal([]byte(line), v) + if err != nil { + t.Error(err) + } + if v.Gid == "" { + t.Error("Gid should be populated if unmarshal was successful") + } + mappedData := v.Data.AsMap() + t.Logf("Received Vertex: %s\n", mappedData["resourceType"]) + + if mappedData["auth_resource_path"] != "/programs/ohsu/projects/test" { + t.Error("returned data should have resource path: /programs/ohsu/projects/test") + } + } +} + func Test_Delete_Edge_Ok(t *testing.T) { /* Regular delete should return 200 */ req := &Request{ diff --git a/gen3_writer/handler.go b/gen3_writer/handler.go index ba855bf..52bbee6 100644 --- a/gen3_writer/handler.go +++ b/gen3_writer/handler.go @@ -217,6 +217,9 @@ func NewHTTPHandler(client gripql.Client, config map[string]string) (http.Handle g.GET("/get-vertex/:vertex-id/:project-id", func(c *gin.Context) { h.GetVertex(c, c.Param("vertex-id")) }) + g.GET("/get-vertices/:project-id", func(c *gin.Context) { + h.GetProjectVertices(c) + }) return h, nil } @@ -451,6 +454,39 @@ func (gh *Handler) BulkDelete(c *gin.Context) { Response(c, writer, graph, nil, 200, fmt.Sprintf("[200] bulk-delete on graph %s", graph)) } +func (gh *Handler) GetProjectVertices(c *gin.Context) { + ctx := context.Background() + + writer, _, graph := getFields(c) + project_id := c.Param("project-id") + str_split := strings.Split(project_id, "-") + project := "/programs/" + str_split[0] + "/projects/" + str_split[1] + + Vquery := gripql.V().Has(gripql.Eq("auth_resource_path", project)) + query := &gripql.GraphQuery{Graph: c.Param("graph"), Query: Vquery.Statements} + result, err := gh.client.Traversal(ctx, query) + if err != nil { + RegError(c, writer, graph, GetInternalServerErr(err)) + return + } + + flusher, ok := writer.(http.Flusher) + if !ok { + RegError(c, writer, graph, GetInternalServerErr(err)) + return + } + + for i := range result { + rowString, _ := protojson.Marshal(i.GetVertex()) + _, err := writer.Write(append(rowString, '\n')) + if err != nil { + RegError(c, writer, graph, GetInternalServerErr(err)) + return + } + flusher.Flush() + } +} + func (gh *Handler) ProjectDelete(c *gin.Context) { ctx := context.Background() var delVs []string From 5f143397016c0bd69dbb3d8d01584a04741256b4 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 17 Oct 2024 16:28:15 -0700 Subject: [PATCH 3/3] Add github action for auto build image to quay --- .github/workflows/build.yaml | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..ba7be95 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,38 @@ +name: Build and publish grip graphql plugin server image + +on: + push: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Login to Quay.io + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_ROBOT_TOKEN }} + + - name: Build and push image + run: | + # Set Image tag to the branch name + BRANCH=$(echo ${GITHUB_REF#refs/*/} | tr / _) + REPO=quay.io/ohsu-comp-bio/grip-caliper + echo "Setting image tag to $REPO:$BRANCH" + + # Login to Quay.io and build image + docker login quay.io + docker build -t $REPO:$BRANCH . + + # Add 'latest' tag to 'main' image + if [[ $BRANCH == 'main' ]]; then + docker image tag $REPO:main $REPO:latest + fi + + # Push the tagged image to Quay.io + docker push --all-tags $REPO \ No newline at end of file