From 9a9040264f136514f1ec7b1234ae37d56b2bd883 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 20 Feb 2024 14:48:58 -0500 Subject: [PATCH 01/21] Updates and changes to worker and load process from debugging --- deploy/k8s/base/deployment.yaml | 22 ++++++------ deploy/k8s/base/temporal_worker.yaml | 8 ++--- src/geneweaver/aon/cli/load.py | 5 +-- src/geneweaver/aon/cli/temporal.py | 35 +++++++++++++++++-- .../aon/temporal/load_data_workflow.py | 14 ++++++-- src/geneweaver/aon/temporal/worker.py | 5 +-- 6 files changed, 63 insertions(+), 26 deletions(-) diff --git a/deploy/k8s/base/deployment.yaml b/deploy/k8s/base/deployment.yaml index c2a2638..2113a3e 100644 --- a/deploy/k8s/base/deployment.yaml +++ b/deploy/k8s/base/deployment.yaml @@ -15,17 +15,17 @@ spec: app: geneweaver-aon-api spec: serviceAccountName: workload-identity-geneweaver - initContainers: - - name: geneweaver-aon-api-init - image: geneweaver-aon-api - imagePullPolicy: Always - command: ["/bin/bash", "-c", "--" ] - args: ["poetry run gwaon setup version-table && poetry run gwaon temporal schedule-load --hour-frequency=1 && poetry run gwaon temporal start-load"] - envFrom: - - configMapRef: - name: geneweaver-aon-config - - secretRef: - name: geneweaver-aon-db +# initContainers: +# - name: geneweaver-aon-api-init +# image: geneweaver-aon-api +# imagePullPolicy: Always +# command: ["/bin/bash", "-c", "--" ] +# args: ["poetry run gwaon setup version-table && poetry run gwaon temporal start-load"] +# envFrom: +# - configMapRef: +# name: geneweaver-aon-config +# - secretRef: +# name: geneweaver-aon-db containers: - name: geneweaver-aon-api image: geneweaver-aon-api diff --git a/deploy/k8s/base/temporal_worker.yaml b/deploy/k8s/base/temporal_worker.yaml index 958dc42..a290b6b 100644 --- a/deploy/k8s/base/temporal_worker.yaml +++ b/deploy/k8s/base/temporal_worker.yaml @@ -49,8 +49,8 @@ spec: key: DB_NAME resources: requests: - cpu: 100m - memory: 128Mi + cpu: 500m + memory: 256Mi limits: - cpu: 1000m - memory: 512Mi \ No newline at end of file + cpu: 2000m + memory: 1024Mi \ No newline at end of file diff --git a/src/geneweaver/aon/cli/load.py b/src/geneweaver/aon/cli/load.py index 7ee99c1..f2877af 100644 --- a/src/geneweaver/aon/cli/load.py +++ b/src/geneweaver/aon/cli/load.py @@ -174,11 +174,8 @@ def complete( @cli.command() -def gw(schema_id: Optional[int] = None) -> bool: +def gw(schema_id: int) -> bool: """Load data from Geneweaver into the AON database.""" - if not schema_id: - schema_name, schema_id = create_schema(release) - version = get_schema_version(schema_id) schema_name = version.schema_name session, _ = set_up_sessionmanager(version) diff --git a/src/geneweaver/aon/cli/temporal.py b/src/geneweaver/aon/cli/temporal.py index c9e91b6..ecc6154 100644 --- a/src/geneweaver/aon/cli/temporal.py +++ b/src/geneweaver/aon/cli/temporal.py @@ -20,7 +20,8 @@ async def _clear_schedules(): """Clear all schedules.""" try: - client = await Client.connect(config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect(config.TEMPORAL_URI, + namespace=config.TEMPORAL_NAMESPACE) handle = client.get_schedule_handle( "geneweaver-aon-agr-load-schedule", ) @@ -40,7 +41,8 @@ def clear_schedules(): async def _schedule_load(hour_frequency: int = 24): """Schedule a load job.""" await _clear_schedules() - client = await Client.connect(config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect(config.TEMPORAL_URI, + namespace=config.TEMPORAL_NAMESPACE) await client.create_schedule( "geneweaver-aon-agr-load-schedule", Schedule( @@ -64,7 +66,8 @@ def schedule_load(hour_frequency: int = 24): async def _start_load(): """Start a load job.""" - client = await Client.connect(config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect(config.TEMPORAL_URI, + namespace=config.TEMPORAL_NAMESPACE) await client.start_workflow( GeneWeaverAonDataLoad.run, id="geneweaver-aon-agr-load-workflow", @@ -78,6 +81,32 @@ def start_load(): aiorun(_start_load()) +async def _cancel_load(): + """Cancel a load job.""" + client = await Client.connect(config.TEMPORAL_URI, + namespace=config.TEMPORAL_NAMESPACE) + await client.get_workflow_handle("geneweaver-aon-agr-load-workflow").cancel() + + +@cli.command() +def cancel_load(): + """Cancel a load job.""" + aiorun(_cancel_load()) + + +async def _terminate_load(): + """Cancel a load job.""" + client = await Client.connect(config.TEMPORAL_URI, + namespace=config.TEMPORAL_NAMESPACE) + await client.get_workflow_handle("geneweaver-aon-agr-load-workflow").terminate() + + +@cli.command() +def terminate_load(): + """Cancel a load job.""" + aiorun(_terminate_load()) + + @cli.command() def start_worker(): aiorun(worker.main()) diff --git a/src/geneweaver/aon/temporal/load_data_workflow.py b/src/geneweaver/aon/temporal/load_data_workflow.py index faa1025..f496d7f 100644 --- a/src/geneweaver/aon/temporal/load_data_workflow.py +++ b/src/geneweaver/aon/temporal/load_data_workflow.py @@ -2,6 +2,7 @@ from typing import Optional, Tuple from temporalio import workflow +from temporalio.common import RetryPolicy with workflow.unsafe.imports_passed_through(): from geneweaver.aon.temporal.activities.download_source import ( @@ -45,25 +46,34 @@ async def run(self, release: Optional[str] = None) -> bool: load_agr_activity, args=(orthology_file, schema_id), schedule_to_close_timeout=timedelta(seconds=3600), + retry_policy=RetryPolicy( + maximum_attempts=1, + ) ) gw_load_success = await workflow.execute_activity( load_gw_activity, schema_id, schedule_to_close_timeout=timedelta(seconds=36000), + retry_policy=RetryPolicy( + maximum_attempts=1, + ) ) homology_load_success = await workflow.execute_activity( load_homology_activity, - args=schema_id, + schema_id, schedule_to_close_timeout=timedelta(seconds=6000), + retry_policy=RetryPolicy( + maximum_attempts=1, + ) ) if agr_load_success and gw_load_success and homology_load_success: await workflow.execute_activity( mark_load_complete_activity, schema_id, - schedule_to_close_timeout=timedelta(seconds=15), + schedule_to_close_timeout=timedelta(seconds=60), ) return agr_load_success and gw_load_success and homology_load_success diff --git a/src/geneweaver/aon/temporal/worker.py b/src/geneweaver/aon/temporal/worker.py index eead965..cee7ab4 100644 --- a/src/geneweaver/aon/temporal/worker.py +++ b/src/geneweaver/aon/temporal/worker.py @@ -18,7 +18,8 @@ async def main(): - client = await Client.connect(config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect(config.TEMPORAL_URI, + namespace=config.TEMPORAL_NAMESPACE) worker = Worker( client, @@ -26,12 +27,12 @@ async def main(): workflows=[GeneWeaverAonDataLoad], activities=[ get_data_activity, + release_exists_activity, create_schema_activity, load_agr_activity, load_gw_activity, load_homology_activity, mark_load_complete_activity, - release_exists_activity ], ) From 44b57a0751c63b36d0e2aa05a0b8f7ca186f7ad8 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 20 Feb 2024 14:58:15 -0500 Subject: [PATCH 02/21] Removing older flask files --- src/geneweaver/aon/app.py | 7 - src/geneweaver/aon/factory.py | 51 ------ src/geneweaver/aon/wsgi.py | 26 --- tests/test_endpoints.py | 328 ---------------------------------- 4 files changed, 412 deletions(-) delete mode 100644 src/geneweaver/aon/app.py delete mode 100755 src/geneweaver/aon/factory.py delete mode 100644 src/geneweaver/aon/wsgi.py delete mode 100644 tests/test_endpoints.py diff --git a/src/geneweaver/aon/app.py b/src/geneweaver/aon/app.py deleted file mode 100644 index a158081..0000000 --- a/src/geneweaver/aon/app.py +++ /dev/null @@ -1,7 +0,0 @@ -"""An entrypoint to application startup.""" - -from geneweaver.aon.factory import create_app - -if __name__ == "__main__": - app = create_app() - app.run() diff --git a/src/geneweaver/aon/factory.py b/src/geneweaver/aon/factory.py deleted file mode 100755 index 1f4c846..0000000 --- a/src/geneweaver/aon/factory.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Factory for the flask app.""" - -import logging - -from flask import Flask -from flask_cors import CORS -from flask_restx import Api -from geneweaver.aon import __version__ -from geneweaver.aon.controller.flask.controller import NS as AGR -from geneweaver.aon.controller.flask.healthcheck import NS as HEALTH_CHECK -from geneweaver.aon.core.config import config -from geneweaver.aon.core.database import SessionLocal -from sqlalchemy.orm import scoped_session - - -def create_app(app=None): - """Create an instance of a flask app - :param app: existing app if initialized - :return: - """ - app = app or Flask(__name__, static_url_path="/static", static_folder="static") - - app.config.from_object(config) - - app.app_context().push() - - logging.basicConfig(level=app.config["LOG_LEVEL"]) - - api = Api( - title=app.config["TITLE"], - version=__version__, - description=app.config["DESCRIPTION"], - ) - - # Add our service and healthcheck endpoints - api.add_namespace(HEALTH_CHECK) - api.add_namespace(AGR) - - api.init_app(app) - - with app.app_context(): - CORS(app) - - app.session = scoped_session(SessionLocal) - - @app.teardown_request(Exception) - def close_session(): - app.session.close() - return {"message": "END"} - - return app diff --git a/src/geneweaver/aon/wsgi.py b/src/geneweaver/aon/wsgi.py deleted file mode 100644 index 0e9dbbf..0000000 --- a/src/geneweaver/aon/wsgi.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -"""wsgi -~~~~. - -pfs_api wsgi module -""" -from geneweaver.aon.factory import create_app -from werkzeug.middleware.dispatcher import DispatcherMiddleware -from werkzeug.serving import run_simple - -app = create_app() -application = DispatcherMiddleware(app) - -try: - import uwsgi - - def postfork(): - app.config["db_engine"].dispose() - - uwsgi.post_fork_hook = postfork - -except ImportError: - pass - -if __name__ == "__main__": - run_simple("localhost", 8000, application, use_reloader=True, use_debugger=True) diff --git a/tests/test_endpoints.py b/tests/test_endpoints.py deleted file mode 100644 index 932b11c..0000000 --- a/tests/test_endpoints.py +++ /dev/null @@ -1,328 +0,0 @@ -""" -Tests every endpoint - -Endpoints that return many items are compared to json file in the results directory to -confirm correct output. This can be done by directly comparing the output and the file, -or comparing the IDs of both. - -Endpoints that return a couple items or less information are directly compared in a single -assertion. - -All endpoints are checked for a '200 OK' status. If the query was unsuccessful, this assertion -will not pass in the first place. -""" -import unittest -import xmlrunner - -import sys, os -from unittest import mock - -from geneweaver.aon.wsgi import application - - -@mock.patch.dict( - os.environ, - { - "DB_HOST": "localhost", - "DB_NAME": "agr", - "GW_DB_HOST": "localhost", - "GW_DB_NAME": "geneweaver", - }, -) -class testEndpoints(unittest.TestCase): - def setUp(self): - # initialize logic for test method, is run before each test - application.app.config["DEBUG"] = True - application.app.config["TESTING"] = True - self.app = application.app.test_client() - - def tearDown(self): - # take down anything we've specifically created for each test method - self.app = None - - def test_app_root(self): - rv = self.app.get("/") - # then expect a 200... - assert "200 OK" in rv.status - - def test_get_algorithm_by_name(self): - rv = self.app.get("/agr-service/get_algorithm_by_name/ZFIN") - assert b"12" in rv.data and b"ZFIN" in rv.data - assert "200 OK" in rv.status - - def test_all_algorithms(self): - rv = self.app.get("/agr-service/all_algorithms") - assert len(rv.json) == 12 - - def test_all_orthologs(self): - rv = self.app.get("/agr-service/all_orthologs") - assert len(rv.json) == 560722 - - def test_get_orthologs_by_from_gene(self): - rv = self.app.get( - "/agr-service/get_orthologs_by_from_gene/ZDB-GENE-040426-960/181874" - ) - data = rv.json - ids = [element["from_gene"] for element in data] - assert len(data) == 6 - assert 57810 in ids and 57810 in ids - - def test_get_orthologs_by_to_gene(self): - rv = self.app.get("/agr-service/get_orthologs_by_to_gene/S000000004/366222") - assert len(rv.json) == 32 - assert (rv.json)[0]["ort_id"] == 3546 - - def test_get_ortholog_by_id(self): - rv = self.app.get("/agr-service/get_ortholog_by_id/1") - ortholog = rv.json - assert ortholog[0]["ort_id"] == 1 - assert "200 OK" in rv.status - - def test_get_orthologs_by_to_and_from_gene(self): - rv = self.app.get( - "/agr-service/get_orthologs_by_to_and_from_gene/WBGene00019900/336853/FBgn0260453/254551" - ) - assert len(rv.json) == 1 - assert (rv.json)[0]["from_gene"] == 12620 and (rv.json)[0]["to_gene"] == 12635 - - def test_get_orthologs_by_from_gene_and_best(self): - rv = self.app.get( - "/agr-service/get_orthologs_by_from_gene_and_best/HGNC:3211/78104/T" - ) - assert len(rv.json) == 7 - - def test_get_orthologs_by_from_to_gene_and_best(self): - rv = self.app.get( - "/agr-service/get_orthologs_by_from_to_gene_and_best/RGD620664/91714/S000000004/366222/true" - ) - assert ( - (rv.json)[0]["from_gene"] == 3793 - and (rv.json)[0]["to_gene"] == 3786 - and (rv.json)[0]["ort_is_best"] == True - ) - - def test_get_orthologs_by_from_to_gene_and_revised(self): - rv = self.app.get( - "/agr-service/get_orthologs_by_from_to_gene_and_revised/RGD620664/91714/S000000004/366222/false" - ) - assert len(rv.json) == 1 - assert (rv.json)[0]["ort_id"] == 3553 - - def test_get_from_gene_of_ortholog_by_id(self): - rv = self.app.get("/agr-service/get_from_gene_of_ortholog_by_id/1") - assert (rv.json)["gn_id"] == 1 and (rv.json)["gn_ref_id"] == "WB:WBGene00011502" - - def test_get_to_gene_of_ortholog_by_id(self): - rv = self.app.get("/agr-service/get_to_gene_of_ortholog_by_id/112199") - assert ( - rv.json["gn_id"] == 65603 - and rv.json["gn_ref_id"] == "ZFIN:ZDB-GENE-010309-3" - ) - - def test_all_genes(self): - rv = self.app.get("/agr-service/all_genes") - assert len(rv.json) == 245167 - - def test_get_genes_by_prefix(self): - rv = self.app.get("/agr-service/get_genes_by_prefix/MGI") - assert len(rv.json) == 20707 - - def test_get_genes_by_ode_gene_id(self): - rv = self.app.get("/agr-service/get_genes_by_ode_gene_id/HGNC%3A3211/78104") - assert rv.json["gn_id"] == 3781 and rv.json["gn_ref_id"] == "HGNC:3211" - - def test_get_genes_by_species(self): - rv = self.app.get("/agr-service/get_genes_by_species/Danio%20rerio") - assert len(rv.json) == 18688 - - def test_get_gene_species_name(self): - rv = self.app.get("/agr-service/get_gene_species_name/FBgn0013275/267529") - assert b"Drosophila melanogaster" in rv.data - assert "200 OK" in rv.status - - def test_all_species(self): - rv = self.app.get("/agr-service/all_species") - assert len(rv.json) == 10 - - def test_get_species_by_id(self): - rv = self.app.get("/agr-service/get_species_by_id/3") - assert b"Saccharomyces cerevisiae" in rv.data and b"559292" in rv.data - assert "200 OK" in rv.status - - def test_get_sp_id_by_hom_id(self): - rv = self.app.get("/agr-service/get_sp_id_by_hom_id/1") - assert len(rv.json) == 8 - - def test_get_species_homologs_list(self): - rv = self.app.get( - "/agr-service/get_species_homologs_list?hom_ids=75338&hom_ids=75341&hom_ids=75340" - ) - assert len(rv.json) == 9 - - def test_get_orthologs_by_num_algoritms(self): - rv = self.app.get("/agr-service/get_orthologs_by_num_algoritms/11") - assert len(rv.json) == 161024 - - def test_get_ortholog_by_algorithm(self): - rv = self.app.get("/agr-service/get_ortholog_by_algorithm/HGNC") - data = rv.json - ids = [element["ort_id"] for element in data] - assert all( - i in ids - for i in [199869, 199959, 204513, 215632, 209234, 240617, 527434, 558257] - ) - assert len(data) == 70490 - - def test_all_homology(self): - rv = self.app.get("/agr-service/all_homology") - assert len(rv.json) == 2619198 - - def test_get_homology_by_id(self): - rv = self.app.get("/agr-service/get_homolgy_by_id/14") - assert len(rv.json) == 22 - assert (rv.json)[0]["gn_id"] == 128 - - def test_get_homology_by_gene(self): - rv = self.app.get("/agr-service/get_homology_by_gene/147") - assert len(rv.json) == 6 - assert (rv.json)[0]["gn_id"] == 147 and (rv.json)[0]["hom_id"] == 14 - - def test_get_homology_by_species(self): - rv = self.app.get("/agr-service/get_homology_by_species/2") - data = rv.json - assert len(data) == 388749 - - def test_get_homology_by_id_and_species(self): - rv = self.app.get("/agr-service/get_homology_by_id_and_species/12/2") - data = rv.json - assert len(data) == 1 - assert data[0]["hom_id"] == 12 and data[0]["sp_id"] == 2 - - def test_get_homology_by_id_and_source(self): - rv = self.app.get("/agr-service/get_homology_by_id_and_source/6/AGR") - data = rv.json - assert len(data) == 12 - assert data[2]["hom_id"] == 6 and data[0]["hom_source_name"] == "AGR" - - def test_get_homology_by_gene_and_source(self): - rv = self.app.get("/agr-service/get_homology_by_gene_and_source/28220/AGR") - data = rv.json - assert len(data) == 10 - assert data[2]["gn_id"] == 28220 and data[0]["hom_source_name"] == "AGR" - - def test_get_ortholog_by_from_species(self): - rv = self.app.get("/agr-service/get_ortholog_by_from_species/Homo%20sapiens") - assert len(rv.json) == 101646 - - def test_get_ortholog_by_to_species(self): - rv = self.app.get( - "/agr-service/get_ortholog_by_to_species/Saccharomyces%20cerevisiae" - ) - assert len(rv.json) == 31928 - - def test_get_orthologous_species(self): - rv = self.app.get("/agr-service/get_orthologous_species/366222/S000000004") - assert len(rv.json) == 6 - - def test_get_ortholog_by_to_and_from_species(self): - rv = self.app.get( - "/agr-service/get_ortholog_by_to_and_from_species/Danio%20rerio/Mus%20musculus" - ) - data = list(rv.json) - assert len(rv.json) == 22734 - - def test_get_ortholog_by_to_from_species_and_algorithm(self): - rv = self.app.get( - "/agr-service/get_ortholog_by_to_from_species_and_algorithm/Saccharomyces%20cerevisiae/Drosophila" - "%20melanogaster/PANTHER" - ) - assert len(rv.json) == 3681 - - def test_agr_to_geneweaver_species(self): - rv = self.app.get("/agr-service/agr_to_geneweaver_species/2") - assert b"3" in rv.data - assert "200 OK" in rv.status - - def test_id_convert_agr_to_ode(self): - rv = self.app.get("/agr-service/id_convert_agr_to_ode/34387") - assert rv.json == 270952 - - def test_id_convert_ode_to_agr(self): - rv = self.app.get("/agr-service/id_convert_ode_to_agr/366222/S000000004") - assert rv.json == 3786 - - def test_get_ode_gene_by_gdb_id(self): - rv = self.app.get("/agr-service/get_ode_gene_by_gdb_id/13") - data = list(rv.json) - ode_gene_ids = [element["ode_gene_id"] for element in data] - assert len(rv.json) == 111648 - assert all( - ids in ode_gene_ids - for ids in [163896, 165108, 165168, 139768, 141616, 143806] - ) - - def test_get_ode_gene_by_gene_id(self): - rv = self.app.get("/agr-service/get_ode_gene_by_gene_id/96483") - assert b"LOC100363666" in rv.data and b"RGD2322584" in rv.data - assert "200 OK" in rv.status - - def test_get_ode_gene_by_species(self): - rv = self.app.get( - "/agr-service/get_ode_gene_by_species/96602/Rattus%20norvegicus" - ) - assert b"LOC100363817" in rv.data and b"RGD2322238" in rv.data - assert "200 OK" in rv.status - - def test_get_ort_id_if_gene_is_ortholog(self): - rv = self.app.get( - "/agr-service/get_ort_id_if_gene_is_ortholog/191876/ZDB-GENE-070112-1002" - ) - data = list(rv.json) - assert len(data) == 14 - assert all( - ids in data - for ids in [[24382], [139827], [139830], [139832], [255497], [7]] - ) - - def test_get_homology_by_ode_gene_id(self): - rv = self.app.get("/agr-service/get_homology_by_ode_gene_id/188364") - assert len(rv.json) == 2 - - def test_get_ode_genes_from_hom_id(self): - rv = self.app.get("/agr-service/get_ode_genes_from_hom_id/75341/10") - assert len(rv.json) == 3 - - def test_get_orthologs_by_symbol(): - rv = self.app.get( - "/agr-service/get_orthologs_by_symbol/1110017D15Rik%2C1110032A03Rik%2C1700001L19Rik/Mus%20musculus/Homo%20sapiens" - ) - assert len(rv.json) == 3 - - def test_get_ortholog_by_from_gene_and_gdb(self): - rv = self.app.get("/agr-service/get_ortholog_by_from_gene_and_gdb/12/11") - data = list(rv.json) - assert len(data) == 1 - assert data[0][0] == "HGNC:88" - - def test_get_intersect_by_homology(self): - rv = self.app.get( - "/agr-service/get_intersect_by_homology?gs1=329155&gs1=WBGene00011502&gs1" - "=vps-53&gs1=6264&gs1=366400&gs1=S000003566&gs1=VPS53&gs1=68774&gs2=366400&" - "gs2=S000003566&gs2=VPS53&gs2=68774&gs2=329155&gs2=WBGene00011502&gs2=vps-53" - "&gs2=6264" - ) - assert len(rv.json) == 9 - assert (rv.json)[0][0] == "366400" and (rv.json)[1][0] == "329155" - - def test_transpose_genes_by_species(self): - rv = self.app.get( - "/agr-service/transpose_genes_by_species?genes=AAG12&genes=PEMP&genes=DXS552" - "E&genes=EMP55&genes=4354&genes=HGNC%3A7219&genes=Hs.496984&genes=Hs.1861&ge" - "nes=Hs.372714&genes=Hs.422215&genes=Hs.322719&genes=Hs.376448&genes=Hs.3476" - "0&genes=Hs.75304&genes=ENSG00000130830&genes=MPP1&species=1" - ) - assert (rv.json)[0] == "Cask" - - -if __name__ == "__main__": - unittest.main(testRunner=xmlrunner.XMLTestRunner(output="reports/test-application")) From da747d26a5425712bdcc103a781394e7a4648416 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 20 Feb 2024 14:58:55 -0500 Subject: [PATCH 03/21] Adding common tests and upgrading geneweaver-testing dep --- poetry.lock | 310 ++++++++++++++++++++++++------------------- pyproject.toml | 4 +- tests/__init__.py | 0 tests/conftest.py | 6 + tests/test_common.py | 6 + 5 files changed, 187 insertions(+), 139 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_common.py diff --git a/poetry.lock b/poetry.lock index 09ac2a8..f7834ef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,13 +34,13 @@ dev = ["black", "coverage", "isort", "pre-commit", "pyenchant", "pylint"] [[package]] name = "anyio" -version = "4.2.0" +version = "4.3.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.8" files = [ - {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"}, - {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, + {file = "anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8"}, + {file = "anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"}, ] [package.dependencies] @@ -229,33 +229,33 @@ lxml = ["lxml"] [[package]] name = "black" -version = "23.12.1" +version = "24.2.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-23.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2"}, - {file = "black-23.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba"}, - {file = "black-23.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a920b569dc6b3472513ba6ddea21f440d4b4c699494d2e972a1753cdc25df7b0"}, - {file = "black-23.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:3fa4be75ef2a6b96ea8d92b1587dd8cb3a35c7e3d51f0738ced0781c3aa3a5a3"}, - {file = "black-23.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d4df77958a622f9b5a4c96edb4b8c0034f8434032ab11077ec6c56ae9f384ba"}, - {file = "black-23.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:602cfb1196dc692424c70b6507593a2b29aac0547c1be9a1d1365f0d964c353b"}, - {file = "black-23.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c4352800f14be5b4864016882cdba10755bd50805c95f728011bcb47a4afd59"}, - {file = "black-23.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:0808494f2b2df923ffc5723ed3c7b096bd76341f6213989759287611e9837d50"}, - {file = "black-23.12.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:25e57fd232a6d6ff3f4478a6fd0580838e47c93c83eaf1ccc92d4faf27112c4e"}, - {file = "black-23.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d9e13db441c509a3763a7a3d9a49ccc1b4e974a47be4e08ade2a228876500ec"}, - {file = "black-23.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1bd9c210f8b109b1762ec9fd36592fdd528485aadb3f5849b2740ef17e674e"}, - {file = "black-23.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:ae76c22bde5cbb6bfd211ec343ded2163bba7883c7bc77f6b756a1049436fbb9"}, - {file = "black-23.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1fa88a0f74e50e4487477bc0bb900c6781dbddfdfa32691e780bf854c3b4a47f"}, - {file = "black-23.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a4d6a9668e45ad99d2f8ec70d5c8c04ef4f32f648ef39048d010b0689832ec6d"}, - {file = "black-23.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b18fb2ae6c4bb63eebe5be6bd869ba2f14fd0259bda7d18a46b764d8fb86298a"}, - {file = "black-23.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:c04b6d9d20e9c13f43eee8ea87d44156b8505ca8a3c878773f68b4e4812a421e"}, - {file = "black-23.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e1b38b3135fd4c025c28c55ddfc236b05af657828a8a6abe5deec419a0b7055"}, - {file = "black-23.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4f0031eaa7b921db76decd73636ef3a12c942ed367d8c3841a0739412b260a54"}, - {file = "black-23.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97e56155c6b737854e60a9ab1c598ff2533d57e7506d97af5481141671abf3ea"}, - {file = "black-23.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:dd15245c8b68fe2b6bd0f32c1556509d11bb33aec9b5d0866dd8e2ed3dba09c2"}, - {file = "black-23.12.1-py3-none-any.whl", hash = "sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e"}, - {file = "black-23.12.1.tar.gz", hash = "sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5"}, + {file = "black-24.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29"}, + {file = "black-24.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430"}, + {file = "black-24.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f"}, + {file = "black-24.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a"}, + {file = "black-24.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd"}, + {file = "black-24.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2"}, + {file = "black-24.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92"}, + {file = "black-24.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23"}, + {file = "black-24.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b"}, + {file = "black-24.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9"}, + {file = "black-24.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693"}, + {file = "black-24.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982"}, + {file = "black-24.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4"}, + {file = "black-24.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218"}, + {file = "black-24.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0"}, + {file = "black-24.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d"}, + {file = "black-24.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8"}, + {file = "black-24.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8"}, + {file = "black-24.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540"}, + {file = "black-24.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31"}, + {file = "black-24.2.0-py3-none-any.whl", hash = "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6"}, + {file = "black-24.2.0.tar.gz", hash = "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894"}, ] [package.dependencies] @@ -509,63 +509,63 @@ test = ["pytest"] [[package]] name = "coverage" -version = "7.4.1" +version = "7.4.2" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:077d366e724f24fc02dbfe9d946534357fda71af9764ff99d73c3c596001bbd7"}, - {file = "coverage-7.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0193657651f5399d433c92f8ae264aff31fc1d066deee4b831549526433f3f61"}, - {file = "coverage-7.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d17bbc946f52ca67adf72a5ee783cd7cd3477f8f8796f59b4974a9b59cacc9ee"}, - {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3277f5fa7483c927fe3a7b017b39351610265308f5267ac6d4c2b64cc1d8d25"}, - {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dceb61d40cbfcf45f51e59933c784a50846dc03211054bd76b421a713dcdf19"}, - {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6008adeca04a445ea6ef31b2cbaf1d01d02986047606f7da266629afee982630"}, - {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c61f66d93d712f6e03369b6a7769233bfda880b12f417eefdd4f16d1deb2fc4c"}, - {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9bb62fac84d5f2ff523304e59e5c439955fb3b7f44e3d7b2085184db74d733b"}, - {file = "coverage-7.4.1-cp310-cp310-win32.whl", hash = "sha256:f86f368e1c7ce897bf2457b9eb61169a44e2ef797099fb5728482b8d69f3f016"}, - {file = "coverage-7.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:869b5046d41abfea3e381dd143407b0d29b8282a904a19cb908fa24d090cc018"}, - {file = "coverage-7.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8ffb498a83d7e0305968289441914154fb0ef5d8b3157df02a90c6695978295"}, - {file = "coverage-7.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3cacfaefe6089d477264001f90f55b7881ba615953414999c46cc9713ff93c8c"}, - {file = "coverage-7.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d6850e6e36e332d5511a48a251790ddc545e16e8beaf046c03985c69ccb2676"}, - {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e961aa13b6d47f758cc5879383d27b5b3f3dcd9ce8cdbfdc2571fe86feb4dd"}, - {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfd1e1b9f0898817babf840b77ce9fe655ecbe8b1b327983df485b30df8cc011"}, - {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6b00e21f86598b6330f0019b40fb397e705135040dbedc2ca9a93c7441178e74"}, - {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:536d609c6963c50055bab766d9951b6c394759190d03311f3e9fcf194ca909e1"}, - {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7ac8f8eb153724f84885a1374999b7e45734bf93a87d8df1e7ce2146860edef6"}, - {file = "coverage-7.4.1-cp311-cp311-win32.whl", hash = "sha256:f3771b23bb3675a06f5d885c3630b1d01ea6cac9e84a01aaf5508706dba546c5"}, - {file = "coverage-7.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:9d2f9d4cc2a53b38cabc2d6d80f7f9b7e3da26b2f53d48f05876fef7956b6968"}, - {file = "coverage-7.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f68ef3660677e6624c8cace943e4765545f8191313a07288a53d3da188bd8581"}, - {file = "coverage-7.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23b27b8a698e749b61809fb637eb98ebf0e505710ec46a8aa6f1be7dc0dc43a6"}, - {file = "coverage-7.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3424c554391dc9ef4a92ad28665756566a28fecf47308f91841f6c49288e66"}, - {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0860a348bf7004c812c8368d1fc7f77fe8e4c095d661a579196a9533778e156"}, - {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe558371c1bdf3b8fa03e097c523fb9645b8730399c14fe7721ee9c9e2a545d3"}, - {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3468cc8720402af37b6c6e7e2a9cdb9f6c16c728638a2ebc768ba1ef6f26c3a1"}, - {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:02f2edb575d62172aa28fe00efe821ae31f25dc3d589055b3fb64d51e52e4ab1"}, - {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ca6e61dc52f601d1d224526360cdeab0d0712ec104a2ce6cc5ccef6ed9a233bc"}, - {file = "coverage-7.4.1-cp312-cp312-win32.whl", hash = "sha256:ca7b26a5e456a843b9b6683eada193fc1f65c761b3a473941efe5a291f604c74"}, - {file = "coverage-7.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:85ccc5fa54c2ed64bd91ed3b4a627b9cce04646a659512a051fa82a92c04a448"}, - {file = "coverage-7.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8bdb0285a0202888d19ec6b6d23d5990410decb932b709f2b0dfe216d031d218"}, - {file = "coverage-7.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:918440dea04521f499721c039863ef95433314b1db00ff826a02580c1f503e45"}, - {file = "coverage-7.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379d4c7abad5afbe9d88cc31ea8ca262296480a86af945b08214eb1a556a3e4d"}, - {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b094116f0b6155e36a304ff912f89bbb5067157aff5f94060ff20bbabdc8da06"}, - {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2f5968608b1fe2a1d00d01ad1017ee27efd99b3437e08b83ded9b7af3f6f766"}, - {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:10e88e7f41e6197ea0429ae18f21ff521d4f4490aa33048f6c6f94c6045a6a75"}, - {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a4a3907011d39dbc3e37bdc5df0a8c93853c369039b59efa33a7b6669de04c60"}, - {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d224f0c4c9c98290a6990259073f496fcec1b5cc613eecbd22786d398ded3ad"}, - {file = "coverage-7.4.1-cp38-cp38-win32.whl", hash = "sha256:23f5881362dcb0e1a92b84b3c2809bdc90db892332daab81ad8f642d8ed55042"}, - {file = "coverage-7.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:a07f61fc452c43cd5328b392e52555f7d1952400a1ad09086c4a8addccbd138d"}, - {file = "coverage-7.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8e738a492b6221f8dcf281b67129510835461132b03024830ac0e554311a5c54"}, - {file = "coverage-7.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46342fed0fff72efcda77040b14728049200cbba1279e0bf1188f1f2078c1d70"}, - {file = "coverage-7.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9641e21670c68c7e57d2053ddf6c443e4f0a6e18e547e86af3fad0795414a628"}, - {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeb2c2688ed93b027eb0d26aa188ada34acb22dceea256d76390eea135083950"}, - {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d12c923757de24e4e2110cf8832d83a886a4cf215c6e61ed506006872b43a6d1"}, - {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0491275c3b9971cdbd28a4595c2cb5838f08036bca31765bad5e17edf900b2c7"}, - {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8dfc5e195bbef80aabd81596ef52a1277ee7143fe419efc3c4d8ba2754671756"}, - {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a78b656a4d12b0490ca72651fe4d9f5e07e3c6461063a9b6265ee45eb2bdd35"}, - {file = "coverage-7.4.1-cp39-cp39-win32.whl", hash = "sha256:f90515974b39f4dea2f27c0959688621b46d96d5a626cf9c53dbc653a895c05c"}, - {file = "coverage-7.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:64e723ca82a84053dd7bfcc986bdb34af8d9da83c521c19d6b472bc6880e191a"}, - {file = "coverage-7.4.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:32a8d985462e37cfdab611a6f95b09d7c091d07668fdc26e47a725ee575fe166"}, - {file = "coverage-7.4.1.tar.gz", hash = "sha256:1ed4b95480952b1a26d863e546fa5094564aa0065e1e5f0d4d0041f293251d04"}, + {file = "coverage-7.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf54c3e089179d9d23900e3efc86d46e4431188d9a657f345410eecdd0151f50"}, + {file = "coverage-7.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fe6e43c8b510719b48af7db9631b5fbac910ade4bd90e6378c85ac5ac706382c"}, + {file = "coverage-7.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b98c89db1b150d851a7840142d60d01d07677a18f0f46836e691c38134ed18b"}, + {file = "coverage-7.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5f9683be6a5b19cd776ee4e2f2ffb411424819c69afab6b2db3a0a364ec6642"}, + {file = "coverage-7.4.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78cdcbf7b9cb83fe047ee09298e25b1cd1636824067166dc97ad0543b079d22f"}, + {file = "coverage-7.4.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2599972b21911111114100d362aea9e70a88b258400672626efa2b9e2179609c"}, + {file = "coverage-7.4.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ef00d31b7569ed3cb2036f26565f1984b9fc08541731ce01012b02a4c238bf03"}, + {file = "coverage-7.4.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:20a875bfd8c282985c4720c32aa05056f77a68e6d8bbc5fe8632c5860ee0b49b"}, + {file = "coverage-7.4.2-cp310-cp310-win32.whl", hash = "sha256:b3f2b1eb229f23c82898eedfc3296137cf1f16bb145ceab3edfd17cbde273fb7"}, + {file = "coverage-7.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:7df95fdd1432a5d2675ce630fef5f239939e2b3610fe2f2b5bf21fa505256fa3"}, + {file = "coverage-7.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8ddbd158e069dded57738ea69b9744525181e99974c899b39f75b2b29a624e2"}, + {file = "coverage-7.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81a5fb41b0d24447a47543b749adc34d45a2cf77b48ca74e5bf3de60a7bd9edc"}, + {file = "coverage-7.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2412e98e70f16243be41d20836abd5f3f32edef07cbf8f407f1b6e1ceae783ac"}, + {file = "coverage-7.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb79414c15c6f03f56cc68fa06994f047cf20207c31b5dad3f6bab54a0f66ef"}, + {file = "coverage-7.4.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf89ab85027427d351f1de918aff4b43f4eb5f33aff6835ed30322a86ac29c9e"}, + {file = "coverage-7.4.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a178b7b1ac0f1530bb28d2e51f88c0bab3e5949835851a60dda80bff6052510c"}, + {file = "coverage-7.4.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:06fe398145a2e91edaf1ab4eee66149c6776c6b25b136f4a86fcbbb09512fd10"}, + {file = "coverage-7.4.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:18cac867950943fe93d6cd56a67eb7dcd2d4a781a40f4c1e25d6f1ed98721a55"}, + {file = "coverage-7.4.2-cp311-cp311-win32.whl", hash = "sha256:f72cdd2586f9a769570d4b5714a3837b3a59a53b096bb954f1811f6a0afad305"}, + {file = "coverage-7.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:d779a48fac416387dd5673fc5b2d6bd903ed903faaa3247dc1865c65eaa5a93e"}, + {file = "coverage-7.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:adbdfcda2469d188d79771d5696dc54fab98a16d2ef7e0875013b5f56a251047"}, + {file = "coverage-7.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ac4bab32f396b03ebecfcf2971668da9275b3bb5f81b3b6ba96622f4ef3f6e17"}, + {file = "coverage-7.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:006d220ba2e1a45f1de083d5022d4955abb0aedd78904cd5a779b955b019ec73"}, + {file = "coverage-7.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3733545eb294e5ad274abe131d1e7e7de4ba17a144505c12feca48803fea5f64"}, + {file = "coverage-7.4.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a9e754aa250fe61f0f99986399cec086d7e7a01dd82fd863a20af34cbce962"}, + {file = "coverage-7.4.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2ed37e16cf35c8d6e0b430254574b8edd242a367a1b1531bd1adc99c6a5e00fe"}, + {file = "coverage-7.4.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b953275d4edfab6cc0ed7139fa773dfb89e81fee1569a932f6020ce7c6da0e8f"}, + {file = "coverage-7.4.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32b4ab7e6c924f945cbae5392832e93e4ceb81483fd6dc4aa8fb1a97b9d3e0e1"}, + {file = "coverage-7.4.2-cp312-cp312-win32.whl", hash = "sha256:f5df76c58977bc35a49515b2fbba84a1d952ff0ec784a4070334dfbec28a2def"}, + {file = "coverage-7.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:34423abbaad70fea9d0164add189eabaea679068ebdf693baa5c02d03e7db244"}, + {file = "coverage-7.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5b11f9c6587668e495cc7365f85c93bed34c3a81f9f08b0920b87a89acc13469"}, + {file = "coverage-7.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:51593a1f05c39332f623d64d910445fdec3d2ac2d96b37ce7f331882d5678ddf"}, + {file = "coverage-7.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69f1665165ba2fe7614e2f0c1aed71e14d83510bf67e2ee13df467d1c08bf1e8"}, + {file = "coverage-7.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3c8bbb95a699c80a167478478efe5e09ad31680931ec280bf2087905e3b95ec"}, + {file = "coverage-7.4.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:175f56572f25e1e1201d2b3e07b71ca4d201bf0b9cb8fad3f1dfae6a4188de86"}, + {file = "coverage-7.4.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8562ca91e8c40864942615b1d0b12289d3e745e6b2da901d133f52f2d510a1e3"}, + {file = "coverage-7.4.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d9a1ef0f173e1a19738f154fb3644f90d0ada56fe6c9b422f992b04266c55d5a"}, + {file = "coverage-7.4.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f40ac873045db4fd98a6f40387d242bde2708a3f8167bd967ccd43ad46394ba2"}, + {file = "coverage-7.4.2-cp38-cp38-win32.whl", hash = "sha256:d1b750a8409bec61caa7824bfd64a8074b6d2d420433f64c161a8335796c7c6b"}, + {file = "coverage-7.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:b4ae777bebaed89e3a7e80c4a03fac434a98a8abb5251b2a957d38fe3fd30088"}, + {file = "coverage-7.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ff7f92ae5a456101ca8f48387fd3c56eb96353588e686286f50633a611afc95"}, + {file = "coverage-7.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:861d75402269ffda0b33af94694b8e0703563116b04c681b1832903fac8fd647"}, + {file = "coverage-7.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3507427d83fa961cbd73f11140f4a5ce84208d31756f7238d6257b2d3d868405"}, + {file = "coverage-7.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf711d517e21fb5bc429f5c4308fbc430a8585ff2a43e88540264ae87871e36a"}, + {file = "coverage-7.4.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c00e54f0bd258ab25e7f731ca1d5144b0bf7bec0051abccd2bdcff65fa3262c9"}, + {file = "coverage-7.4.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f8e845d894e39fb53834da826078f6dc1a933b32b1478cf437007367efaf6f6a"}, + {file = "coverage-7.4.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:840456cb1067dc350af9080298c7c2cfdddcedc1cb1e0b30dceecdaf7be1a2d3"}, + {file = "coverage-7.4.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c11ca2df2206a4e3e4c4567f52594637392ed05d7c7fb73b4ea1c658ba560265"}, + {file = "coverage-7.4.2-cp39-cp39-win32.whl", hash = "sha256:3ff5bdb08d8938d336ce4088ca1a1e4b6c8cd3bef8bb3a4c0eb2f37406e49643"}, + {file = "coverage-7.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:ac9e95cefcf044c98d4e2c829cd0669918585755dd9a92e28a1a7012322d0a95"}, + {file = "coverage-7.4.2-pp38.pp39.pp310-none-any.whl", hash = "sha256:f593a4a90118d99014517c2679e04a4ef5aee2d81aa05c26c734d271065efcb6"}, + {file = "coverage-7.4.2.tar.gz", hash = "sha256:1a5ee18e3a8d766075ce9314ed1cb695414bae67df6a4b0805f5137d93d6f1cb"}, ] [package.dependencies] @@ -629,22 +629,22 @@ files = [ [[package]] name = "dnspython" -version = "2.5.0" +version = "2.6.1" description = "DNS toolkit" optional = false python-versions = ">=3.8" files = [ - {file = "dnspython-2.5.0-py3-none-any.whl", hash = "sha256:6facdf76b73c742ccf2d07add296f178e629da60be23ce4b0a9c927b1e02c3a6"}, - {file = "dnspython-2.5.0.tar.gz", hash = "sha256:a0034815a59ba9ae888946be7ccca8f7c157b286f8455b379c692efb51022a15"}, + {file = "dnspython-2.6.1-py3-none-any.whl", hash = "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50"}, + {file = "dnspython-2.6.1.tar.gz", hash = "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc"}, ] [package.extras] -dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=5.0.3)", "mypy (>=1.0.1)", "pylint (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0.0)", "sphinx (>=7.0.0)", "twine (>=4.0.0)", "wheel (>=0.41.0)"] +dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "sphinx (>=7.2.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] dnssec = ["cryptography (>=41)"] -doh = ["h2 (>=4.1.0)", "httpcore (>=0.17.3)", "httpx (>=0.25.1)"] -doq = ["aioquic (>=0.9.20)"] -idna = ["idna (>=2.1)"] -trio = ["trio (>=0.14)"] +doh = ["h2 (>=4.1.0)", "httpcore (>=1.0.0)", "httpx (>=0.26.0)"] +doq = ["aioquic (>=0.9.25)"] +idna = ["idna (>=3.6)"] +trio = ["trio (>=0.23)"] wmi = ["wmi (>=1.5.1)"] [[package]] @@ -855,13 +855,13 @@ files = [ [[package]] name = "geneweaver-core" -version = "0.9.0a2" +version = "0.9.0a4" description = "The core of the Jax-Geneweaver Python library" optional = false python-versions = ">=3.9,<4.0" files = [ - {file = "geneweaver_core-0.9.0a2-py3-none-any.whl", hash = "sha256:b9d44fe033b5b5a0e7bc4ae50d2fc0eefccd1a0a62131ee3ca1e1dbc576b96a6"}, - {file = "geneweaver_core-0.9.0a2.tar.gz", hash = "sha256:68c6343d1761b40548fdfaeb491297789841b0e3ac5cc7fb57d444c0e4c1af98"}, + {file = "geneweaver_core-0.9.0a4-py3-none-any.whl", hash = "sha256:4077a974ba62f1fc70c22d6ad2c196f405b54c44972f9393c604c48d4516a841"}, + {file = "geneweaver_core-0.9.0a4.tar.gz", hash = "sha256:ae35e65d5cc03ab885d455808a4e610de9840234e5217d33c08cd6d6e5800cef"}, ] [package.dependencies] @@ -887,22 +887,23 @@ psycopg = {version = ">=3.1.13,<4.0.0", extras = ["binary"]} [[package]] name = "geneweaver-testing" -version = "0.0.3" +version = "0.1.0" description = "A library to standardize testing of GeneWeaver pacakges." optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "geneweaver_testing-0.0.3-py3-none-any.whl", hash = "sha256:dd9d04cd9ff6366496a9f66ea3347121d0d810d9748936db736837d6f773d7f5"}, - {file = "geneweaver_testing-0.0.3.tar.gz", hash = "sha256:e268462ced2d44ad213cae7cb2e669d5c0065e6bf82965730b7d3eb0f0f0e9a8"}, + {file = "geneweaver_testing-0.1.0-py3-none-any.whl", hash = "sha256:c2082d342b82a5d65c6ec11364f7da2e596995c0fe4e8e8d1a0bfdff83e78251"}, + {file = "geneweaver_testing-0.1.0.tar.gz", hash = "sha256:1b62713b4c8c63e7a1a330f2719be803e8ed54cb113097cc90942bb66fd61b7c"}, ] [package.dependencies] -black = ">=23.3.0,<24.0.0" +black = ">=24.2.0,<25.0.0" isort = ">=5.12.0,<6.0.0" mypy = ">=1.4.1,<2.0.0" -pytest = ">=7.4.0,<8.0.0" +pytest = ">=7.4.0,<9.0.0" pytest-cov = ">=4.1.0,<5.0.0" -ruff = ">=0.0.277,<0.0.278" +radon = ">=6.0.1,<7.0.0" +ruff = ">=0.2.2,<0.3.0" tomli = ">=2.0.1,<3.0.0" [[package]] @@ -989,13 +990,13 @@ files = [ [[package]] name = "httpcore" -version = "1.0.2" +version = "1.0.3" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, - {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, + {file = "httpcore-1.0.3-py3-none-any.whl", hash = "sha256:9a6a501c3099307d9fd76ac244e08503427679b1e81ceb1d922485e2f2462ad2"}, + {file = "httpcore-1.0.3.tar.gz", hash = "sha256:5c0f9546ad17dac4d0772b0808856eb616eb8b48ce94f49ed819fd6982a8a544"}, ] [package.dependencies] @@ -1006,7 +1007,7 @@ h11 = ">=0.13,<0.15" asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<0.23.0)"] +trio = ["trio (>=0.22.0,<0.24.0)"] [[package]] name = "httptools" @@ -1268,13 +1269,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "json5" -version = "0.9.14" +version = "0.9.17" description = "A Python implementation of the JSON5 data format." optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "json5-0.9.14-py2.py3-none-any.whl", hash = "sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f"}, - {file = "json5-0.9.14.tar.gz", hash = "sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02"}, + {file = "json5-0.9.17-py2.py3-none-any.whl", hash = "sha256:f8ec1ecf985951d70f780f6f877c4baca6a47b6e61e02c4cd190138d10a7805a"}, + {file = "json5-0.9.17.tar.gz", hash = "sha256:717d99d657fa71b7094877b1d921b1cce40ab444389f6d770302563bb7dfd9ae"}, ] [package.extras] @@ -1474,13 +1475,13 @@ test = ["jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-jupyter[server] (> [[package]] name = "jupyterlab" -version = "4.1.1" +version = "4.1.2" description = "JupyterLab computational environment" optional = false python-versions = ">=3.8" files = [ - {file = "jupyterlab-4.1.1-py3-none-any.whl", hash = "sha256:fa3e8c18b804eac04e51ceebd9dd3dd396e08106816f0d09cc426799d7087632"}, - {file = "jupyterlab-4.1.1.tar.gz", hash = "sha256:8acc9f561729d8f32c14c294c397917cddfeeb13a5d46f811979b71b4911a9fd"}, + {file = "jupyterlab-4.1.2-py3-none-any.whl", hash = "sha256:aa88193f03cf4d3555f6712f04d74112b5eb85edd7d222c588c7603a26d33c5b"}, + {file = "jupyterlab-4.1.2.tar.gz", hash = "sha256:5d6348b3ed4085181499f621b7dfb6eb0b1f57f3586857aadfc8e3bf4c4885f9"}, ] [package.dependencies] @@ -1561,6 +1562,23 @@ babel = ["Babel"] lingua = ["lingua"] testing = ["pytest"] +[[package]] +name = "mando" +version = "0.7.1" +description = "Create Python CLI apps with little to no effort at all!" +optional = false +python-versions = "*" +files = [ + {file = "mando-0.7.1-py2.py3-none-any.whl", hash = "sha256:26ef1d70928b6057ee3ca12583d73c63e05c49de8972d620c278a7b206581a8a"}, + {file = "mando-0.7.1.tar.gz", hash = "sha256:18baa999b4b613faefb00eac4efadcf14f510b59b924b66e08289aa1de8c3500"}, +] + +[package.dependencies] +six = "*" + +[package.extras] +restructuredtext = ["rst2ansi"] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -1792,13 +1810,13 @@ test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>= [[package]] name = "nbconvert" -version = "7.16.0" -description = "Converting Jupyter Notebooks" +version = "7.16.1" +description = "Converting Jupyter Notebooks (.ipynb files) to other formats. Output formats include asciidoc, html, latex, markdown, pdf, py, rst, script. nbconvert can be used both as a Python library (`import nbconvert`) or as a command line tool (invoked as `jupyter nbconvert ...`)." optional = false python-versions = ">=3.8" files = [ - {file = "nbconvert-7.16.0-py3-none-any.whl", hash = "sha256:ad3dc865ea6e2768d31b7eb6c7ab3be014927216a5ece3ef276748dd809054c7"}, - {file = "nbconvert-7.16.0.tar.gz", hash = "sha256:813e6553796362489ae572e39ba1bff978536192fb518e10826b0e8cadf03ec8"}, + {file = "nbconvert-7.16.1-py3-none-any.whl", hash = "sha256:3188727dffadfdc9c6a1c7250729063d7bc78b355ad7aa023138afa030d1cd07"}, + {file = "nbconvert-7.16.1.tar.gz", hash = "sha256:e79e6a074f49ba3ed29428ed86487bf51509d9aab613bd8522ac08f6d28fd7fd"}, ] [package.dependencies] @@ -1885,13 +1903,13 @@ test = ["importlib-resources (>=5.0)", "ipykernel", "jupyter-server[test] (>=2.4 [[package]] name = "notebook-shim" -version = "0.2.3" +version = "0.2.4" description = "A shim layer for notebook traits and config" optional = false python-versions = ">=3.7" files = [ - {file = "notebook_shim-0.2.3-py3-none-any.whl", hash = "sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7"}, - {file = "notebook_shim-0.2.3.tar.gz", hash = "sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9"}, + {file = "notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef"}, + {file = "notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb"}, ] [package.dependencies] @@ -2182,13 +2200,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "prometheus-client" -version = "0.19.0" +version = "0.20.0" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.8" files = [ - {file = "prometheus_client-0.19.0-py3-none-any.whl", hash = "sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92"}, - {file = "prometheus_client-0.19.0.tar.gz", hash = "sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1"}, + {file = "prometheus_client-0.20.0-py3-none-any.whl", hash = "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7"}, + {file = "prometheus_client-0.20.0.tar.gz", hash = "sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89"}, ] [package.extras] @@ -2801,6 +2819,24 @@ files = [ [package.dependencies] cffi = {version = "*", markers = "implementation_name == \"pypy\""} +[[package]] +name = "radon" +version = "6.0.1" +description = "Code Metrics in Python" +optional = false +python-versions = "*" +files = [ + {file = "radon-6.0.1-py2.py3-none-any.whl", hash = "sha256:632cc032364a6f8bb1010a2f6a12d0f14bc7e5ede76585ef29dc0cecf4cd8859"}, + {file = "radon-6.0.1.tar.gz", hash = "sha256:d1ac0053943a893878940fedc8b19ace70386fc9c9bf0a09229a44125ebf45b5"}, +] + +[package.dependencies] +colorama = {version = ">=0.4.1", markers = "python_version > \"3.4\""} +mando = ">=0.6,<0.8" + +[package.extras] +toml = ["tomli (>=2.0.1)"] + [[package]] name = "referencing" version = "0.33.0" @@ -3004,28 +3040,28 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.0.277" -description = "An extremely fast Python linter, written in Rust." +version = "0.2.2" +description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.0.277-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:3250b24333ef419b7a232080d9724ccc4d2da1dbbe4ce85c4caa2290d83200f8"}, - {file = "ruff-0.0.277-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:3e60605e07482183ba1c1b7237eca827bd6cbd3535fe8a4ede28cbe2a323cb97"}, - {file = "ruff-0.0.277-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7baa97c3d7186e5ed4d5d4f6834d759a27e56cf7d5874b98c507335f0ad5aadb"}, - {file = "ruff-0.0.277-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:74e4b206cb24f2e98a615f87dbe0bde18105217cbcc8eb785bb05a644855ba50"}, - {file = "ruff-0.0.277-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:479864a3ccd8a6a20a37a6e7577bdc2406868ee80b1e65605478ad3b8eb2ba0b"}, - {file = "ruff-0.0.277-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:468bfb0a7567443cec3d03cf408d6f562b52f30c3c29df19927f1e0e13a40cd7"}, - {file = "ruff-0.0.277-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f32ec416c24542ca2f9cc8c8b65b84560530d338aaf247a4a78e74b99cd476b4"}, - {file = "ruff-0.0.277-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:14a7b2f00f149c5a295f188a643ac25226ff8a4d08f7a62b1d4b0a1dc9f9b85c"}, - {file = "ruff-0.0.277-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9879f59f763cc5628aa01c31ad256a0f4dc61a29355c7315b83c2a5aac932b5"}, - {file = "ruff-0.0.277-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f612e0a14b3d145d90eb6ead990064e22f6f27281d847237560b4e10bf2251f3"}, - {file = "ruff-0.0.277-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:323b674c98078be9aaded5b8b51c0d9c424486566fb6ec18439b496ce79e5998"}, - {file = "ruff-0.0.277-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3a43fbe026ca1a2a8c45aa0d600a0116bec4dfa6f8bf0c3b871ecda51ef2b5dd"}, - {file = "ruff-0.0.277-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:734165ea8feb81b0d53e3bf523adc2413fdb76f1264cde99555161dd5a725522"}, - {file = "ruff-0.0.277-py3-none-win32.whl", hash = "sha256:88d0f2afb2e0c26ac1120e7061ddda2a566196ec4007bd66d558f13b374b9efc"}, - {file = "ruff-0.0.277-py3-none-win_amd64.whl", hash = "sha256:6fe81732f788894a00f6ade1fe69e996cc9e485b7c35b0f53fb00284397284b2"}, - {file = "ruff-0.0.277-py3-none-win_arm64.whl", hash = "sha256:2d4444c60f2e705c14cd802b55cd2b561d25bf4311702c463a002392d3116b22"}, - {file = "ruff-0.0.277.tar.gz", hash = "sha256:2dab13cdedbf3af6d4427c07f47143746b6b95d9e4a254ac369a0edb9280a0d2"}, + {file = "ruff-0.2.2-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0a9efb032855ffb3c21f6405751d5e147b0c6b631e3ca3f6b20f917572b97eb6"}, + {file = "ruff-0.2.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d450b7fbff85913f866a5384d8912710936e2b96da74541c82c1b458472ddb39"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecd46e3106850a5c26aee114e562c329f9a1fbe9e4821b008c4404f64ff9ce73"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e22676a5b875bd72acd3d11d5fa9075d3a5f53b877fe7b4793e4673499318ba"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1695700d1e25a99d28f7a1636d85bafcc5030bba9d0578c0781ba1790dbcf51c"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:b0c232af3d0bd8f521806223723456ffebf8e323bd1e4e82b0befb20ba18388e"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f63d96494eeec2fc70d909393bcd76c69f35334cdbd9e20d089fb3f0640216ca"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a61ea0ff048e06de273b2e45bd72629f470f5da8f71daf09fe481278b175001"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1439c8f407e4f356470e54cdecdca1bd5439a0673792dbe34a2b0a551a2fe3"}, + {file = "ruff-0.2.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:940de32dc8853eba0f67f7198b3e79bc6ba95c2edbfdfac2144c8235114d6726"}, + {file = "ruff-0.2.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0c126da55c38dd917621552ab430213bdb3273bb10ddb67bc4b761989210eb6e"}, + {file = "ruff-0.2.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3b65494f7e4bed2e74110dac1f0d17dc8e1f42faaa784e7c58a98e335ec83d7e"}, + {file = "ruff-0.2.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1ec49be4fe6ddac0503833f3ed8930528e26d1e60ad35c2446da372d16651ce9"}, + {file = "ruff-0.2.2-py3-none-win32.whl", hash = "sha256:d920499b576f6c68295bc04e7b17b6544d9d05f196bb3aac4358792ef6f34325"}, + {file = "ruff-0.2.2-py3-none-win_amd64.whl", hash = "sha256:cc9a91ae137d687f43a44c900e5d95e9617cb37d4c989e462980ba27039d239d"}, + {file = "ruff-0.2.2-py3-none-win_arm64.whl", hash = "sha256:c9d15fc41e6054bfc7200478720570078f0b41c9ae4f010bcc16bd6f4d1aacdd"}, + {file = "ruff-0.2.2.tar.gz", hash = "sha256:e62ed7f36b3068a30ba39193a14274cd706bc486fad521276458022f7bccb31d"}, ] [[package]] @@ -3830,4 +3866,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "4f4b0294ceba464a7606a284969f840a58dafd8927ccb28cb890d767e551a39f" +content-hash = "60a8c0f81852e9088280552dfee0a147b1528aa7b04fe1512f00c79784e6d5b9" diff --git a/pyproject.toml b/pyproject.toml index da20d6a..8f6e98a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "geneweaver-aon" version = "0.1.0" -description = "" +description = "A service to do Homology and Orthology mapping." authors = ["Sophie Kearney ", "Alexander Berger "] readme = "README.md" license = "Apache-2.0" @@ -34,7 +34,7 @@ typer = {extras = ["all"], version = "^0.9.0"} temporalio = "^1.5.0" [tool.poetry.group.dev.dependencies] -geneweaver-testing = "^0.0.3" +geneweaver-testing = "^0.1.0" xmlrunner = "1.7.7" notebook = "^7.0.7" pytest = "^7.4.0" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0f5b888 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,6 @@ +"""Pytest fixtures for unit tests. + +These fixtures are available to be used by all tests in the unit test suite. +""" + +from geneweaver.testing.fixtures import * # noqa: F403 diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 0000000..7f16c6e --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,6 @@ +"""Test common Geneweaver package functionality.""" + +# Import all pre-defined tests from the geneweaver-testing pacakge so that PyTest +# automatically runs them. Additional tests can also be added here as needed, but +# should be kept to a minimum. +from geneweaver.testing import * # noqa: F403 From ce32d7a887e225f679b137e2a40f19f81881e9e4 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 20 Feb 2024 15:00:44 -0500 Subject: [PATCH 04/21] Adding github actions yaml --- .github/workflows/README.md | 72 ++++++++++ .github/workflows/_check-coverage-action.yml | 102 ++++++++++++++ .github/workflows/_format-lint-action.yml | 32 +++++ .github/workflows/_run-tests-action.yml | 44 ++++++ .github/workflows/_skaffold-build-k8s.yml | 34 +++++ .github/workflows/_skaffold-deploy-k8s.yml | 41 ++++++ .github/workflows/coverage.yml | 13 ++ .github/workflows/pull_requests.yml | 46 +++++++ .github/workflows/release.yml | 134 +++++++++++++++++++ .github/workflows/style.yml | 10 ++ .github/workflows/tests.yml | 16 +++ 11 files changed, 544 insertions(+) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/_check-coverage-action.yml create mode 100644 .github/workflows/_format-lint-action.yml create mode 100644 .github/workflows/_run-tests-action.yml create mode 100644 .github/workflows/_skaffold-build-k8s.yml create mode 100644 .github/workflows/_skaffold-deploy-k8s.yml create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/pull_requests.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/style.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..5b8de5e --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,72 @@ +# GitHub Actions + +The files in this directory are used to configure the Github Actions workflows for +`geneweaver-api`. The workflows are used to automatically build and test the project +when changes are pushed to the repository. + +Any file that starts with an underscore (`_`) is a "reusable workflow". These files +are not directly used by GitHub Actions, but are instead referenced by the workflows +files that do not start with an underscore. + +There are five reusable workflows: + +- Check Coverage (`_check-coverage-action.yml`): This workflow is used to check the code + coverage of the project. +- Format Lint (`_format-lint-action.yml`): This workflow is used to check the formatting + and linting of the project. +- Run Tests (`_run-tests-action.yml`): This workflow is used to run the tests for the + project. +- Skaffold Build (`_skaffold-build-k8s.yml`): This workflow is used to build the + Docker images for the project. +- Skaffold Deploy (`_skaffold-deploy-action.yml`): This workflow is used to deploy the + Docker images to kubernetes. + +There are two _main_ workflows that are used by GitHub Actions: + +- Pull Requests (`pull_requests.yml`): This workflow is used to build and test the + project when a pull request is opened. +- Release (`release.yml`): This workflow that is run whenever the version number changes + on the `main` branch. + +There are also three quality assurance workflows that are run on any change to the main +branch: + +- Coverage (`coverage.yml`): This workflow is used to check the code coverage of the + project. +- Style (`style.yml`): This workflow is used to check the formatting and linting of the + project. +- Tests (`tests.yml`): This workflow is used to run the tests for the project. + + +## Pull Requests + +The pull request workflow is run whenever a pull request is opened. This workflow +will: + +- Check the formatting and linting of the project. +- Run the tests for the project. +- Check the code coverage of the project. +- Build the Docker images (into the `test` registry) for the project. +- Deploy the Docker images (from the `test` registry) to kubernetes (into the `dev` + environment). + +## Release + +The release workflow is run whenever the version number changes on the `main` branch. +This workflow will: + +- Check the formatting and linting of the project. +- Run the tests for the project. +- Check the code coverage of the project. +- Build the Docker images (into the `prod` registry) for the project. +- Deploy the Docker images (from the `prod` registry) to kubernetes (into the `sqa` + environment). + - It will wait for approval from SQA before running this step +- If the version number is not a pre-release version (contains a letter) it will then: + - Deploy the Docker images (from the `prod` registry) to kubernetes (into the + `stage` environment). + - It will wait for approval from SQA before running this step + - Deploy the Docker images (from the `prod` registry) to kubernetes (into the `prod` + environment). + - It will wait for approval from SQA before running this step + - It will then create a draft GitHub release \ No newline at end of file diff --git a/.github/workflows/_check-coverage-action.yml b/.github/workflows/_check-coverage-action.yml new file mode 100644 index 0000000..0990df5 --- /dev/null +++ b/.github/workflows/_check-coverage-action.yml @@ -0,0 +1,102 @@ +name: 'Test Coverage Definition' +on: + workflow_call: + inputs: + coverage-module: + description: "Module to test coverage for" + type: string + required: true + python-version: + description: Python version to set up' + default: '3.11' + type: string + runner-os: + description: 'Runner OS' + default: 'ubuntu-latest' + type: string + upload-coverage: + description: 'Upload coverage results' + default: true + type: boolean + required-coverage: + description: 'Required coverage percentage' + default: 100 + type: string + show-test-traceback: + description: "Show traceback for failed tests" + type: string + default: "no" +jobs: + check_coverage: + runs-on: ${{ inputs.runner-os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + if [[ "$RUNNER_OS" == "macOS" ]]; then + echo "/Users/runner/.local/bin:$PATH" >> $GITHUB_PATH + fi + - name: Configure Poetry + run: poetry config virtualenvs.create false + - name: Install dependencies with Poetry + run: poetry install + - name: Test with pytest + run: | + poetry run pytest tests \ + --tb=${{ inputs.show-test-traceback }} \ + --cov=${{ inputs.coverage-module }} \ + --cov-report=term \ + --cov-report=html \ + --cov-fail-under=${{ inputs.required-coverage }} > coverage_report.txt + - name: Upload coverage report + if: '!cancelled()' + uses: actions/upload-artifact@v3 + with: + name: coverage-report + path: coverage_report.txt + - name: Upload coverage report + if: '!cancelled()' + uses: actions/upload-artifact@v3 + with: + name: coverage-report-html + path: htmlcov + comment-coverage-report: + needs: [ check_coverage ] + runs-on: ubuntu-latest + if: ${{always() && github.event_name == 'pull_request'}} + permissions: + pull-requests: write + steps: + - name: Download coverage report artifact + uses: actions/download-artifact@v3 + with: + name: coverage-report + - name: Read coverage report + id: read-coverage + run: | + echo "COVERAGE_REPORT<> $GITHUB_ENV + cat coverage_report.txt >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + - name: Find Comment + uses: peter-evans/find-comment@v2 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: '### Test Coverage Report' + - name: Create or update comment + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + edit-mode: replace + body: | + ### Test Coverage Report + ``` + ${{ env.COVERAGE_REPORT }} + ``` \ No newline at end of file diff --git a/.github/workflows/_format-lint-action.yml b/.github/workflows/_format-lint-action.yml new file mode 100644 index 0000000..8e41af4 --- /dev/null +++ b/.github/workflows/_format-lint-action.yml @@ -0,0 +1,32 @@ +name: 'Lint Code Definition' +on: + workflow_call: + inputs: + python-version: + description: 'Python version to set up' + required: true + default: '3.9' + type: string +jobs: + format-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + if [[ "$RUNNER_OS" == "macOS" ]]; then + echo "/Users/runner/.local/bin:$PATH" >> $GITHUB_PATH + fi + - name: Configure Poetry + run: poetry config virtualenvs.create false + - name: Install Black and Ruff + run: poetry install --only dev + - name: Run Ruff Linter + run: ruff src/ tests/ + - name: Run Black Formatter + run: black --check src/ tests/ diff --git a/.github/workflows/_run-tests-action.yml b/.github/workflows/_run-tests-action.yml new file mode 100644 index 0000000..ad82334 --- /dev/null +++ b/.github/workflows/_run-tests-action.yml @@ -0,0 +1,44 @@ +name: 'Python Tests Definition' +on: + workflow_call: + inputs: + python-version: + description: Python version to set up' + required: true + default: '3.9' + type: string + runner-os: + description: 'Runner OS' + required: true + default: 'ubuntu-latest' + type: string + upload-coverage: + description: 'Upload coverage results' + default: true + type: boolean + required-coverage: + description: 'Required coverage percentage' + default: 75 + type: string +jobs: + run-tests: + runs-on: ${{ inputs.runner-os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + if [[ "$RUNNER_OS" == "macOS" ]]; then + echo "/Users/runner/.local/bin:$PATH" >> $GITHUB_PATH + fi + - name: Configure Poetry + run: poetry config virtualenvs.create false + - name: Install dependencies with Poetry + run: poetry install + - name: Test with pytest + run: | + poetry run pytest tests diff --git a/.github/workflows/_skaffold-build-k8s.yml b/.github/workflows/_skaffold-build-k8s.yml new file mode 100644 index 0000000..2cbb096 --- /dev/null +++ b/.github/workflows/_skaffold-build-k8s.yml @@ -0,0 +1,34 @@ +name: 'Skaffold Build' +on: + workflow_call: + inputs: + default_image_repo: + description: 'Default image repo' + required: false + type: string + default: "us-docker.pkg.dev/jax-cs-registry/docker/geneweaver" +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCLOUD_REGISTRY_SA_KEY }}' + - name: Docker Login + run: gcloud auth configure-docker us-docker.pkg.dev,us-east1-docker.pkg.dev + - name: Build + run: | + skaffold build \ + --default-repo=${{ inputs.default_image_repo }} \ + --file-output=build.json + - name: Upload Build Artifact Information + uses: actions/upload-artifact@v3 + with: + name: build-artifact-json + path: build.json diff --git a/.github/workflows/_skaffold-deploy-k8s.yml b/.github/workflows/_skaffold-deploy-k8s.yml new file mode 100644 index 0000000..399ef14 --- /dev/null +++ b/.github/workflows/_skaffold-deploy-k8s.yml @@ -0,0 +1,41 @@ +name: 'Skaffold Deploy' +on: + workflow_call: + inputs: + environment: + description: 'Deployment environment/profile' + required: true + type: string +jobs: + deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - uses: actions/checkout@v3 + - name: Download Build Artifact Information + uses: actions/download-artifact@v3 + with: + name: build-artifact-json + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCLOUD_CLUSTER_SA_KEY }}' + - id: setup-gcloud + name: Setup Gcloud + uses: 'google-github-actions/setup-gcloud@v1' + - id: get-gke-credentials + name: Get GKE credentials + uses: 'google-github-actions/get-gke-credentials@v1' + with: + cluster_name: ${{ vars.CLUSTER_NAME }} + location: ${{ vars.CLUSTER_REGION }} + project_id: ${{ vars.CLUSTER_PROJECT }} + - name: Install Skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + - name: Deploy + run: | + skaffold deploy \ + --profile ${{ inputs.environment }} \ + --build-artifacts=build.json diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..70b8341 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,13 @@ +name: Coverage +on: + push: + branches: + - 'main' +jobs: + check-coverage: + uses: ./.github/workflows/_check-coverage-action.yml + permissions: + pull-requests: write + with: + required-coverage: ${{ vars.REQUIRED_COVERAGE }} + coverage-module: "geneweaver.aon" diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml new file mode 100644 index 0000000..fd1c4ad --- /dev/null +++ b/.github/workflows/pull_requests.yml @@ -0,0 +1,46 @@ +name: Pull Request Test, Build and Deploy +on: + pull_request: + branches: + - 'main' +jobs: + format-lint: + name: "Format and Lint" + uses: ./.github/workflows/_format-lint-action.yml + with: + python-version: '3.9' + check-coverage: + name: "Check Coverage" + needs: [format-lint] + uses: ./.github/workflows/_check-coverage-action.yml + permissions: + pull-requests: write + with: + required-coverage: ${{ vars.REQUIRED_COVERAGE }} + coverage-module: "geneweaver.aon" + test: + name: "Run Tests" + needs: [format-lint] + strategy: + matrix: + os: [ubuntu-latest] + python-version: ['3.10', '3.11'] + uses: ./.github/workflows/_run-tests-action.yml + with: + runner-os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + required-coverage: ${{ vars.REQUIRED_COVERAGE }} + build: + name: "Build: Dev" + needs: [test, check-coverage] + uses: ./.github/workflows/_skaffold-build-k8s.yml + secrets: inherit + with: + default_image_repo: "us-east1-docker.pkg.dev/jax-cs-registry/docker-test/geneweaver" + deploy: + name: "Deploy: Dev" + needs: [build] + uses: ./.github/workflows/_skaffold-deploy-k8s.yml + secrets: inherit + with: + environment: "jax-cluster-dev-10--dev" \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..57486fd --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,134 @@ +name: Build, Deploy and Release + +on: + push: + branches: + - 'main' + paths: + - 'pyproject.toml' + +permissions: + contents: write + +jobs: + format-lint: + name: "Format and Lint" + uses: ./.github/workflows/_format-lint-action.yml + with: + python-version: '3.9' + check-coverage: + name: "Check Coverage" + needs: [format-lint] + uses: ./.github/workflows/_check-coverage-action.yml + permissions: + pull-requests: write + with: + required-coverage: ${{ vars.REQUIRED_COVERAGE }} + coverage-module: "geneweaver.aon" + test: + name: "Run Tests" + needs: [check-coverage, format-lint] + strategy: + matrix: + os: [ubuntu-latest] + python-version: ['3.10', '3.11'] + uses: ./.github/workflows/_run-tests-action.yml + with: + runner-os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + required-coverage: ${{ vars.REQUIRED_COVERAGE }} + version: + needs: test + runs-on: ubuntu-latest + outputs: + should_release: ${{ steps.version_check.outputs.should_release }} + prerelease: ${{ steps.version_check.outputs.prerelease }} + version: ${{ steps.version_check.outputs.version }} + steps: + - name: Check out code + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + - name: Install dependencies + run: | + pip install toml + - name: Check for version change + id: version_check + run: | + # Extract version from pyproject.toml + version=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") + echo "Version=$version" + echo "version=$version" >> $GITHUB_OUTPUT + + # Check if this version tag already exists + if git rev-parse "v$version" >/dev/null 2>&1; then + echo "Version already released" + echo "should_release=true" >> $GITHUB_OUTPUT + else + echo "New version detected" + echo "should_release=false" >> $GITHUB_OUTPUT + fi + - name: Determine Release Type + id: release_type + run: | + version=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") + if [[ $version =~ [a-zA-Z] ]]; then + echo "Pre-release version detected" + echo "prerelease=true" >> $GITHUB_OUTPUT + else + echo "Full release version detected" + echo "prerelease=false" >> $GITHUB_OUTPUT + fi + build: + name: "Build" + needs: [ test, check-coverage, format-lint, version ] + if: ${{ needs.version.outputs.should_release }} == 'true' + uses: ./.github/workflows/_skaffold-build-k8s.yml + secrets: inherit + deploy_sqa: + name: "Deploy: SQA" + needs: [build, version] + if: ${{ needs.version.outputs.should_release }} == 'true' + uses: ./.github/workflows/_skaffold-deploy-k8s.yml + secrets: inherit + with: + environment: "jax-cluster-dev-10--sqa" + deploy_stage: + name: "Deploy: Stage" + needs: [ build, version, deploy_sqa] + if: ${{ needs.version.outputs.should_release }} == 'true' && ${{ needs.version.outputs.prerelease }} == 'false' + uses: ./.github/workflows/_skaffold-deploy-k8s.yml + secrets: inherit + with: + environment: "jax-cluster-prod-10--stage" + deploy_prod: + name: "Deploy: Prod" + needs: [ build, version, deploy_stage] + if: ${{ needs.version.outputs.should_release }} == 'true' && ${{ needs.version.outputs.prerelease }} == 'false' + uses: ./.github/workflows/_skaffold-deploy-k8s.yml + secrets: inherit + with: + environment: "jax-cluster-prod-10--prod" + release: + name: "Create Github Release Draft" + runs-on: ubuntu-latest + needs: [ deploy_prod ] + if: ${{ needs.version.outputs.should_release == 'true' && needs.version.outputs.prerelease == 'false'}} + steps: + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ steps.version_check.outputs.version }} + name: Release v${{ needs.version.outputs.version }} + draft: true + prerelease: ${{ needs.version.outputs.prerelease }} + body: | + Release v${{ needs.version.outputs.version }} + files: | + dist/* + LICENSE + README.md \ No newline at end of file diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml new file mode 100644 index 0000000..a590a19 --- /dev/null +++ b/.github/workflows/style.yml @@ -0,0 +1,10 @@ +name: Style +on: + push: + branches: + - 'main' +jobs: + format-lint: + uses: ./.github/workflows/_format-lint-action.yml + with: + python-version: '3.9' \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..9bde38e --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,16 @@ +name: Tests +on: + push: + branches: + - 'main' +jobs: + test: + strategy: + matrix: + os: [ubuntu-latest] + python-version: ['3.9', '3.10', '3.11'] + uses: ./.github/workflows/_run-tests-action.yml + with: + runner-os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + required-coverage: ${{ vars.REQUIRED_COVERAGE }} From 621a48cb7fa81783789365bcfc1fa29ead56c64c Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Tue, 20 Feb 2024 15:03:26 -0500 Subject: [PATCH 05/21] Bumping version number --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8f6e98a..7ebfebf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "geneweaver-aon" -version = "0.1.0" +version = "0.2.0" description = "A service to do Homology and Orthology mapping." authors = ["Sophie Kearney ", "Alexander Berger "] readme = "README.md" From aaf78e40020784cc1adde131dfc2c4e1e3f1a5ad Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 07:30:01 -0500 Subject: [PATCH 06/21] Increasing worker resources --- deploy/k8s/base/temporal_worker.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/k8s/base/temporal_worker.yaml b/deploy/k8s/base/temporal_worker.yaml index a290b6b..7aad097 100644 --- a/deploy/k8s/base/temporal_worker.yaml +++ b/deploy/k8s/base/temporal_worker.yaml @@ -49,8 +49,8 @@ spec: key: DB_NAME resources: requests: - cpu: 500m - memory: 256Mi + cpu: 1 + memory: 2Gi limits: - cpu: 2000m - memory: 1024Mi \ No newline at end of file + cpu: 2 + memory: 4Gi \ No newline at end of file From e950865dc69e1c91b43dfc0d12efe0c17b3de72d Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 07:30:39 -0500 Subject: [PATCH 07/21] Applying first pass at ruff and black formatting project wide --- src/geneweaver/aon/__init__.py | 1 + .../versions/4297df5638d7_first_commit.py | 1 + .../aon/alembic/versions/666568a64356_.py | 8 ++-- src/geneweaver/aon/api.py | 2 +- src/geneweaver/aon/cli/load.py | 21 +++++---- src/geneweaver/aon/cli/main.py | 2 +- src/geneweaver/aon/cli/setup.py | 7 +-- src/geneweaver/aon/cli/temporal.py | 45 +++++++++++-------- .../aon/controller/flask/healthcheck.py | 1 + src/geneweaver/aon/controller/genes.py | 2 +- src/geneweaver/aon/controller/versions.py | 3 +- src/geneweaver/aon/core/config.py | 1 + src/geneweaver/aon/core/config_class.py | 5 ++- src/geneweaver/aon/core/database.py | 1 - src/geneweaver/aon/core/schema_version.py | 11 +++-- src/geneweaver/aon/dependencies.py | 15 +++---- src/geneweaver/aon/load/agr/load.py | 5 ++- src/geneweaver/aon/load/agr/sources.py | 1 + src/geneweaver/aon/load/geneweaver/genes.py | 39 +--------------- .../aon/load/geneweaver/homologs.py | 2 + src/geneweaver/aon/load/geneweaver/species.py | 2 +- .../temporal/activities/download_source.py | 7 +-- .../aon/temporal/load_data_workflow.py | 16 +++---- src/geneweaver/aon/temporal/worker.py | 17 ++++--- 24 files changed, 98 insertions(+), 117 deletions(-) diff --git a/src/geneweaver/aon/__init__.py b/src/geneweaver/aon/__init__.py index 4de37c9..3ad463f 100755 --- a/src/geneweaver/aon/__init__.py +++ b/src/geneweaver/aon/__init__.py @@ -1,4 +1,5 @@ """The Geneweaver AON (Ortholog/Homolog Normalizer) Package.""" + try: from importlib import metadata except ImportError: diff --git a/src/geneweaver/aon/alembic/versions/4297df5638d7_first_commit.py b/src/geneweaver/aon/alembic/versions/4297df5638d7_first_commit.py index 09d820a..c192cec 100644 --- a/src/geneweaver/aon/alembic/versions/4297df5638d7_first_commit.py +++ b/src/geneweaver/aon/alembic/versions/4297df5638d7_first_commit.py @@ -5,6 +5,7 @@ Create Date: 2020-12-16 19:27:45.142317 """ + import sqlalchemy as sa from alembic import op diff --git a/src/geneweaver/aon/alembic/versions/666568a64356_.py b/src/geneweaver/aon/alembic/versions/666568a64356_.py index 0ef661e..995ee0a 100644 --- a/src/geneweaver/aon/alembic/versions/666568a64356_.py +++ b/src/geneweaver/aon/alembic/versions/666568a64356_.py @@ -5,13 +5,13 @@ Create Date: 2024-02-19 15:12:11.936663 """ -from alembic import op -import sqlalchemy as sa +import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. -revision = '666568a64356' -down_revision = '4297df5638d7' +revision = "666568a64356" +down_revision = "4297df5638d7" branch_labels = None depends_on = None diff --git a/src/geneweaver/aon/api.py b/src/geneweaver/aon/api.py index 8eebb7a..b415acc 100644 --- a/src/geneweaver/aon/api.py +++ b/src/geneweaver/aon/api.py @@ -1,6 +1,6 @@ """""" -from fastapi import APIRouter, FastAPI, Depends +from fastapi import APIRouter, Depends, FastAPI from geneweaver.aon import __version__ from geneweaver.aon import dependencies as deps from geneweaver.aon.controller import ( diff --git a/src/geneweaver/aon/cli/load.py b/src/geneweaver/aon/cli/load.py index f2877af..c5600b6 100644 --- a/src/geneweaver/aon/cli/load.py +++ b/src/geneweaver/aon/cli/load.py @@ -1,20 +1,21 @@ """CLI to load the database.""" + +from argparse import Namespace +from datetime import datetime +from gzip import BadGzipFile from pathlib import Path from typing import Optional, Tuple -from argparse import Namespace + import psycopg import typer -from pathlib import Path -from datetime import datetime -from alembic.config import Config from alembic import command -from gzip import BadGzipFile +from alembic.config import Config from geneweaver.aon.core.config import config from geneweaver.aon.core.database import SessionLocal from geneweaver.aon.core.schema_version import ( get_schema_version, - set_up_sessionmanager, mark_schema_version_load_complete, + set_up_sessionmanager, ) from geneweaver.aon.load import agr, geneweaver from geneweaver.aon.models import Version @@ -181,7 +182,7 @@ def gw(schema_id: int) -> bool: session, _ = set_up_sessionmanager(version) gw_load_msg = "Loading Geneweaver data: " - with Progress(transient=True) as progress: + with Progress() as progress: gw_load = progress.add_task(gw_load_msg + "Connecting", total=None) db = session() @@ -223,12 +224,14 @@ def homology(schema_id: int) -> bool: version = get_schema_version(schema_id) session, _ = set_up_sessionmanager(version) - with Progress(transient=True) as progress: + with Progress() as progress: db_load_msg = "Loading data into the database: " - db_load = progress.add_task(db_load_msg + "Adding Homology", total=None) + db_load = progress.add_task(db_load_msg + "Connecting...", total=None) db = session() + progress.update(db_load, completed=True, description=db_load_msg + "Loading Homology") + agr.load.add_homology(db) progress.update(db_load, completed=True, description=db_load_msg + "Complete") diff --git a/src/geneweaver/aon/cli/main.py b/src/geneweaver/aon/cli/main.py index 75db082..b475f09 100644 --- a/src/geneweaver/aon/cli/main.py +++ b/src/geneweaver/aon/cli/main.py @@ -1,6 +1,6 @@ import typer from geneweaver.aon import __version__ -from geneweaver.aon.cli import load, temporal, setup +from geneweaver.aon.cli import load, setup, temporal cli = typer.Typer(no_args_is_help=True, rich_markup_mode=True) diff --git a/src/geneweaver/aon/cli/setup.py b/src/geneweaver/aon/cli/setup.py index 805191a..3ce6874 100644 --- a/src/geneweaver/aon/cli/setup.py +++ b/src/geneweaver/aon/cli/setup.py @@ -1,10 +1,11 @@ """CLI to load the database.""" -import typer + from pathlib import Path -from alembic.config import Config + +import typer from alembic import command +from alembic.config import Config from geneweaver.aon.core.config import config - from rich.progress import Progress cli = typer.Typer(no_args_is_help=True, rich_markup_mode="rich") diff --git a/src/geneweaver/aon/cli/temporal.py b/src/geneweaver/aon/cli/temporal.py index ecc6154..84c180b 100644 --- a/src/geneweaver/aon/cli/temporal.py +++ b/src/geneweaver/aon/cli/temporal.py @@ -1,34 +1,37 @@ """Commands for working with temporal jobs.""" -import typer -from datetime import timedelta + from asyncio import run as aiorun -from temporalio.service import RPCError +from datetime import timedelta + +import typer +from geneweaver.aon.core.config import config +from geneweaver.aon.temporal import worker +from geneweaver.aon.temporal.load_data_workflow import GeneWeaverAonDataLoad from temporalio.client import ( Client, Schedule, ScheduleActionStartWorkflow, ScheduleIntervalSpec, ScheduleSpec, - ScheduleState, ) -from geneweaver.aon.core.config import config -from geneweaver.aon.temporal.load_data_workflow import GeneWeaverAonDataLoad -from geneweaver.aon.temporal import worker +from temporalio.service import RPCError + cli = typer.Typer(no_args_is_help=True, rich_markup_mode="rich") async def _clear_schedules(): """Clear all schedules.""" try: - client = await Client.connect(config.TEMPORAL_URI, - namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect( + config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE + ) handle = client.get_schedule_handle( "geneweaver-aon-agr-load-schedule", ) await handle.delete() - except RPCError as e: - typer.echo(f"No Schedule to delete.") + except RPCError: + typer.echo("No Schedule to delete.") typer.Exit(0) @@ -41,8 +44,9 @@ def clear_schedules(): async def _schedule_load(hour_frequency: int = 24): """Schedule a load job.""" await _clear_schedules() - client = await Client.connect(config.TEMPORAL_URI, - namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect( + config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE + ) await client.create_schedule( "geneweaver-aon-agr-load-schedule", Schedule( @@ -66,8 +70,9 @@ def schedule_load(hour_frequency: int = 24): async def _start_load(): """Start a load job.""" - client = await Client.connect(config.TEMPORAL_URI, - namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect( + config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE + ) await client.start_workflow( GeneWeaverAonDataLoad.run, id="geneweaver-aon-agr-load-workflow", @@ -83,8 +88,9 @@ def start_load(): async def _cancel_load(): """Cancel a load job.""" - client = await Client.connect(config.TEMPORAL_URI, - namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect( + config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE + ) await client.get_workflow_handle("geneweaver-aon-agr-load-workflow").cancel() @@ -96,8 +102,9 @@ def cancel_load(): async def _terminate_load(): """Cancel a load job.""" - client = await Client.connect(config.TEMPORAL_URI, - namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect( + config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE + ) await client.get_workflow_handle("geneweaver-aon-agr-load-workflow").terminate() diff --git a/src/geneweaver/aon/controller/flask/healthcheck.py b/src/geneweaver/aon/controller/flask/healthcheck.py index 136a608..8f468af 100644 --- a/src/geneweaver/aon/controller/flask/healthcheck.py +++ b/src/geneweaver/aon/controller/flask/healthcheck.py @@ -1,4 +1,5 @@ """Endpoints to check on the services health.""" + from datetime import datetime from flask_restx import Namespace, Resource diff --git a/src/geneweaver/aon/controller/genes.py b/src/geneweaver/aon/controller/genes.py index a9815d4..5eb9943 100644 --- a/src/geneweaver/aon/controller/genes.py +++ b/src/geneweaver/aon/controller/genes.py @@ -1,6 +1,6 @@ from typing import Optional -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends from geneweaver.aon import dependencies as deps from geneweaver.aon.enum import ReferenceGeneIDType from geneweaver.aon.service import convert as convert_service diff --git a/src/geneweaver/aon/controller/versions.py b/src/geneweaver/aon/controller/versions.py index f64e84b..1638ba8 100644 --- a/src/geneweaver/aon/controller/versions.py +++ b/src/geneweaver/aon/controller/versions.py @@ -1,4 +1,3 @@ -from typing import Optional from fastapi import APIRouter, Depends, Request from geneweaver.aon import dependencies as deps @@ -11,7 +10,7 @@ def get_versions( db: deps.Session = Depends(deps.session), ): - return db.query(Version).all() + return db.query(Version).filter(Version.load_complete == True).all() @router.get("/default") diff --git a/src/geneweaver/aon/core/config.py b/src/geneweaver/aon/core/config.py index 0cdd726..e5d7c99 100755 --- a/src/geneweaver/aon/core/config.py +++ b/src/geneweaver/aon/core/config.py @@ -1,4 +1,5 @@ """Application config instance.""" + from geneweaver.aon.core.config_class import Config config = Config() diff --git a/src/geneweaver/aon/core/config_class.py b/src/geneweaver/aon/core/config_class.py index 790ec01..5cabcec 100644 --- a/src/geneweaver/aon/core/config_class.py +++ b/src/geneweaver/aon/core/config_class.py @@ -1,6 +1,8 @@ """Config class definition.""" -from typing import Any, Dict, Optional + import logging +from typing import Any, Dict, Optional + from geneweaver.db.core.settings_class import Settings as DBSettings from pydantic import BaseSettings, validator @@ -65,7 +67,6 @@ def assemble_gw_db_settings( cls, v: Optional[DBSettings], values: Dict[str, Any] # noqa: N805 ) -> Optional[DBSettings]: """Build the database settings.""" - if isinstance(v, DBSettings): return v elif values.get("GW_DB_HOST") is None or values.get("GW_DB_NAME") is None: diff --git a/src/geneweaver/aon/core/database.py b/src/geneweaver/aon/core/database.py index 1e2bf1d..ccc40d0 100755 --- a/src/geneweaver/aon/core/database.py +++ b/src/geneweaver/aon/core/database.py @@ -3,7 +3,6 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker - BaseAGR = declarative_base() BaseGW = declarative_base() diff --git a/src/geneweaver/aon/core/schema_version.py b/src/geneweaver/aon/core/schema_version.py index 26fb772..4c2c184 100644 --- a/src/geneweaver/aon/core/schema_version.py +++ b/src/geneweaver/aon/core/schema_version.py @@ -1,14 +1,13 @@ """Utilities for getting and setting up schema version db connections.""" + import logging -from typing import Annotated, Optional, Union -from contextlib import asynccontextmanager +from typing import Optional, Tuple -from fastapi import FastAPI, Request, HTTPException from geneweaver.aon.core.config import config from geneweaver.aon.core.database import BaseAGR, BaseGW -from sqlalchemy import create_engine, event -from sqlalchemy.orm import sessionmaker, Session from geneweaver.aon.models import Version +from sqlalchemy import create_engine, Engine +from sqlalchemy.orm import Session, sessionmaker logger = logging.getLogger("uvicorn.error") @@ -57,7 +56,7 @@ def mark_schema_version_load_complete(version_id: int): session.close() -def set_up_sessionmanager(version: Optional[Version]) -> sessionmaker: +def set_up_sessionmanager(version: Optional[Version]) -> Tuple[sessionmaker, Tuple[Engine, Engine]]: """Set up the session manager. :param version: The schema version to use. diff --git a/src/geneweaver/aon/dependencies.py b/src/geneweaver/aon/dependencies.py index 63ad439..442ddf3 100644 --- a/src/geneweaver/aon/dependencies.py +++ b/src/geneweaver/aon/dependencies.py @@ -1,21 +1,19 @@ """Dependency injection for the AON FastAPI application.""" + import logging -from typing import Annotated, Optional, Union from contextlib import asynccontextmanager +from typing import Annotated, Optional, Union -from fastapi import FastAPI, Request, HTTPException +from fastapi import FastAPI, HTTPException, Request from geneweaver.aon.core.config import config -from geneweaver.aon.core.database import BaseAGR, BaseGW -from sqlalchemy import create_engine, event -from sqlalchemy.orm import sessionmaker, Session -from geneweaver.aon.models import Version from geneweaver.aon.core.schema_version import ( get_latest_schema_version, - get_schema_versions, get_schema_version, + get_schema_versions, set_up_sessionmanager, set_up_sessionmanager_by_schema, ) +from sqlalchemy.orm import sessionmaker, Session logger = logging.getLogger("uvicorn.error") @@ -42,7 +40,8 @@ async def lifespan(app: FastAPI) -> None: ) else: default_version_id = next( - (v.id for v in schema_versions if v.schema_name == config.DEFAULT_SCHEMA), None + (v.id for v in schema_versions if v.schema_name == config.DEFAULT_SCHEMA), + None, ) app.default_schema_version_id = default_version_id diff --git a/src/geneweaver/aon/load/agr/load.py b/src/geneweaver/aon/load/agr/load.py index 93c36f9..d9df2c0 100644 --- a/src/geneweaver/aon/load/agr/load.py +++ b/src/geneweaver/aon/load/agr/load.py @@ -1,10 +1,11 @@ """Functions used to load the database.""" + from itertools import chain, islice -from geneweaver.core import enum from geneweaver.aon.models import Algorithm, Gene, Homology, Ortholog, Species -from sqlalchemy.sql import text +from geneweaver.core import enum from sqlalchemy.orm import Session +from sqlalchemy.sql import text TAXON_ID_MAP = { enum.Species.RATTUS_NORVEGICUS: 10116, diff --git a/src/geneweaver/aon/load/agr/sources.py b/src/geneweaver/aon/load/agr/sources.py index 0a09650..87e125c 100644 --- a/src/geneweaver/aon/load/agr/sources.py +++ b/src/geneweaver/aon/load/agr/sources.py @@ -1,4 +1,5 @@ """Download and prepare external source data for Geneweaver AON.""" + import gzip from pathlib import Path from typing import Optional diff --git a/src/geneweaver/aon/load/geneweaver/genes.py b/src/geneweaver/aon/load/geneweaver/genes.py index 5ae6966..a5e9428 100644 --- a/src/geneweaver/aon/load/geneweaver/genes.py +++ b/src/geneweaver/aon/load/geneweaver/genes.py @@ -1,6 +1,6 @@ """Code to add genes from geneweaver gene table to agr gn_gene table.""" -from geneweaver.aon.controller.flask.controller import convert_species_ode_to_agr -from geneweaver.aon.models import Gene + +from geneweaver.aon.models import Gene, Species from geneweaver.core.enum import GeneIdentifier from psycopg import Cursor from sqlalchemy.orm import Session @@ -81,38 +81,3 @@ def add_missing_genes(db: Session, geneweaver_cursor: Cursor): db.add_all(genes) print("committing") db.commit() - - # results_returned = True - # limit = 10000 - # offset = 9600 - # while results_returned: - # print(f"Complete: {offset}") - # geneweaver_cursor.execute( - # """ - # SELECT ode_ref_id, gdb_id, sp_id FROM extsrc.gene WHERE sp_id IN (6,10,11) - # AND gdb_id in (1,2,20) LIMIT %(limit)s OFFSET %(offset)s; - # """, {"limit": limit, "offset": offset} - # ) - # gw_genes = geneweaver_cursor.fetchall() - # offset = offset + limit - # print(f"Found {offset}") - # - # i = 0 - # genes = [] - # for g in gw_genes: - # gn_ref_id = g[0] - # gn_prefix = convert_gdb_to_prefix(g[1]) - # # sp_id = convert_species_ode_to_agr(int(g[2])) - # sp_id = int(g[2]) - # - # gene = Gene(gn_ref_id=gn_ref_id, gn_prefix=gn_prefix, sp_id=sp_id) - # genes.append(gene) - # - # print("Adding genes") - # db.add_all(genes) - # db.commit() - # - # if len(genes) < 1000: - # results_returned = False - # print("done") - # break diff --git a/src/geneweaver/aon/load/geneweaver/homologs.py b/src/geneweaver/aon/load/geneweaver/homologs.py index 99bb336..c508a0f 100644 --- a/src/geneweaver/aon/load/geneweaver/homologs.py +++ b/src/geneweaver/aon/load/geneweaver/homologs.py @@ -1,6 +1,8 @@ """Code for adding homolog/ortholog information from the geneweaver database.""" + import itertools from typing import Optional + from geneweaver.aon.controller.flask.controller import convert_ode_ref_to_agr from geneweaver.aon.models import Gene, Geneweaver_Gene, Ortholog from psycopg import Cursor, sql diff --git a/src/geneweaver/aon/load/geneweaver/species.py b/src/geneweaver/aon/load/geneweaver/species.py index 0f759d0..56adba6 100644 --- a/src/geneweaver/aon/load/geneweaver/species.py +++ b/src/geneweaver/aon/load/geneweaver/species.py @@ -1,5 +1,5 @@ """Code for adding missing species information.""" -from geneweaver.core import enum + from geneweaver.aon.models import Species from sqlalchemy.orm import Session diff --git a/src/geneweaver/aon/temporal/activities/download_source.py b/src/geneweaver/aon/temporal/activities/download_source.py index 04ece50..04f9f2e 100644 --- a/src/geneweaver/aon/temporal/activities/download_source.py +++ b/src/geneweaver/aon/temporal/activities/download_source.py @@ -1,14 +1,15 @@ -from temporalio import activity from typing import Optional, Tuple + from geneweaver.aon.cli.load import ( - get_data, agr_release_exists, create_schema, - load_agr, + get_data, gw, homology, + load_agr, mark_schema_version_load_complete, ) +from temporalio import activity @activity.defn diff --git a/src/geneweaver/aon/temporal/load_data_workflow.py b/src/geneweaver/aon/temporal/load_data_workflow.py index f496d7f..248bc38 100644 --- a/src/geneweaver/aon/temporal/load_data_workflow.py +++ b/src/geneweaver/aon/temporal/load_data_workflow.py @@ -1,18 +1,18 @@ from datetime import timedelta -from typing import Optional, Tuple +from typing import Optional from temporalio import workflow from temporalio.common import RetryPolicy with workflow.unsafe.imports_passed_through(): from geneweaver.aon.temporal.activities.download_source import ( - get_data_activity, - release_exists_activity, create_schema_activity, + get_data_activity, load_agr_activity, load_gw_activity, load_homology_activity, mark_load_complete_activity, + release_exists_activity, ) @@ -48,7 +48,7 @@ async def run(self, release: Optional[str] = None) -> bool: schedule_to_close_timeout=timedelta(seconds=3600), retry_policy=RetryPolicy( maximum_attempts=1, - ) + ), ) gw_load_success = await workflow.execute_activity( @@ -57,16 +57,16 @@ async def run(self, release: Optional[str] = None) -> bool: schedule_to_close_timeout=timedelta(seconds=36000), retry_policy=RetryPolicy( maximum_attempts=1, - ) + ), ) homology_load_success = await workflow.execute_activity( load_homology_activity, schema_id, - schedule_to_close_timeout=timedelta(seconds=6000), + schedule_to_close_timeout=timedelta(seconds=360), retry_policy=RetryPolicy( - maximum_attempts=1, - ) + maximum_attempts=3, + ), ) if agr_load_success and gw_load_success and homology_load_success: diff --git a/src/geneweaver/aon/temporal/worker.py b/src/geneweaver/aon/temporal/worker.py index cee7ab4..cf4d649 100644 --- a/src/geneweaver/aon/temporal/worker.py +++ b/src/geneweaver/aon/temporal/worker.py @@ -1,25 +1,24 @@ import asyncio -from temporalio.client import Client -from temporalio.worker import Worker - +from geneweaver.aon.core.config import config from geneweaver.aon.temporal.activities.download_source import ( - get_data_activity, - release_exists_activity, create_schema_activity, + get_data_activity, load_agr_activity, load_gw_activity, load_homology_activity, mark_load_complete_activity, + release_exists_activity, ) from geneweaver.aon.temporal.load_data_workflow import GeneWeaverAonDataLoad - -from geneweaver.aon.core.config import config +from temporalio.client import Client +from temporalio.worker import Worker async def main(): - client = await Client.connect(config.TEMPORAL_URI, - namespace=config.TEMPORAL_NAMESPACE) + client = await Client.connect( + config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE + ) worker = Worker( client, From 2b079046c7c8b168a4680abbfc96d9e49066c9bc Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 08:30:45 -0500 Subject: [PATCH 08/21] Partially fixing ruff issues --- pyproject.toml | 3 + src/geneweaver/aon/alembic/__init__.py | 1 + src/geneweaver/aon/api.py | 2 +- src/geneweaver/aon/cli/__init__.py | 4 + src/geneweaver/aon/cli/load.py | 4 +- .../aon/controller/flask/controller.py | 194 +++++----- src/geneweaver/aon/controller/genes.py | 19 - src/geneweaver/aon/controller/versions.py | 1 - src/geneweaver/aon/core/schema_version.py | 4 +- src/geneweaver/aon/enum.py | 5 + src/geneweaver/aon/load/add_missing_info.py | 77 ---- src/geneweaver/aon/load/agr/__init__.py | 4 +- .../load/fill_missing_species_information.py | 355 ------------------ .../aon/load/geneweaver/__init__.py | 4 +- .../aon/load/geneweaver/homologs.py | 49 +-- src/geneweaver/aon/load/geneweaver/species.py | 22 -- src/geneweaver/aon/models.py | 31 +- src/geneweaver/aon/service/convert.py | 16 +- src/geneweaver/aon/service/species.py | 36 +- src/geneweaver/aon/service/utils.py | 9 + src/geneweaver/aon/temporal/__init__.py | 1 + .../aon/temporal/activities/__init__.py | 1 + .../temporal/activities/download_source.py | 9 + .../aon/temporal/load_data_workflow.py | 5 + src/geneweaver/aon/temporal/worker.py | 5 +- 25 files changed, 239 insertions(+), 622 deletions(-) delete mode 100644 src/geneweaver/aon/load/add_missing_info.py delete mode 100644 src/geneweaver/aon/load/fill_missing_species_information.py delete mode 100644 src/geneweaver/aon/load/geneweaver/species.py diff --git a/pyproject.toml b/pyproject.toml index 7ebfebf..fe75192 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,9 @@ pytest-cov = "^4.1.0" [tool.ruff] select = ['F', 'E', 'W', 'A', 'C90', 'N', 'B', 'ANN', 'D', 'I', 'ERA', 'PD', 'NPY', 'PT'] +exclude = [ + 'src/geneweaver/aon/controller/flask/controller.py' + ] [tool.ruff.per-file-ignores] "tests/*" = ["ANN001", "ANN201"] diff --git a/src/geneweaver/aon/alembic/__init__.py b/src/geneweaver/aon/alembic/__init__.py index e69de29..daf47b5 100644 --- a/src/geneweaver/aon/alembic/__init__.py +++ b/src/geneweaver/aon/alembic/__init__.py @@ -0,0 +1 @@ +"""The alembic module for managing database migrations.""" diff --git a/src/geneweaver/aon/api.py b/src/geneweaver/aon/api.py index b415acc..f3a53ac 100644 --- a/src/geneweaver/aon/api.py +++ b/src/geneweaver/aon/api.py @@ -1,4 +1,4 @@ -"""""" +"""The root of the GeneWeaver AON API.""" from fastapi import APIRouter, Depends, FastAPI from geneweaver.aon import __version__ diff --git a/src/geneweaver/aon/cli/__init__.py b/src/geneweaver/aon/cli/__init__.py index e69de29..c8b5a4a 100644 --- a/src/geneweaver/aon/cli/__init__.py +++ b/src/geneweaver/aon/cli/__init__.py @@ -0,0 +1,4 @@ +"""Command line definitions for Geneweaver AON. + +Use `gwaon --help` to see the available commands. +""" diff --git a/src/geneweaver/aon/cli/load.py b/src/geneweaver/aon/cli/load.py index c5600b6..30d4f37 100644 --- a/src/geneweaver/aon/cli/load.py +++ b/src/geneweaver/aon/cli/load.py @@ -230,7 +230,9 @@ def homology(schema_id: int) -> bool: db = session() - progress.update(db_load, completed=True, description=db_load_msg + "Loading Homology") + progress.update( + db_load, completed=True, description=db_load_msg + "Loading Homology" + ) agr.load.add_homology(db) diff --git a/src/geneweaver/aon/controller/flask/controller.py b/src/geneweaver/aon/controller/flask/controller.py index b6be96f..0e047e8 100755 --- a/src/geneweaver/aon/controller/flask/controller.py +++ b/src/geneweaver/aon/controller/flask/controller.py @@ -5,9 +5,9 @@ from geneweaver.aon.models import ( Algorithm, Gene, - Geneweaver_Gene, - Geneweaver_GeneDB, - Geneweaver_Species, + GeneweaverGene, + GeneweaverGeneDB, + GeneweaverSpecies, Homology, Ortholog, OrthologAlgorithms, @@ -137,7 +137,7 @@ def convertAGRtoODE(gn_id): ref = ref[ind:] ode_gene_id = ( - db.query(Geneweaver_Gene).filter(Geneweaver_Gene.ode_ref_id == ref).first() + db.query(GeneweaverGene).filter(GeneweaverGene.ode_ref_id == ref).first() ).ode_gene_id return ode_gene_id @@ -196,10 +196,10 @@ class get_orthologs_by_from_gene(Resource): def get(self, ode_ref_id, ode_gene_id): # find gene and search orthologs based on gene_id gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == ode_gene_id, - Geneweaver_Gene.ode_ref_id == ode_ref_id, + GeneweaverGene.ode_gene_id == ode_gene_id, + GeneweaverGene.ode_ref_id == ode_ref_id, ) .first() ).gdb_id @@ -223,10 +223,10 @@ class get_orthologs_by_to_gene(Resource): @NS.marshal_with(ortholog_model) def get(self, ode_ref_id, ode_gene_id): gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == ode_gene_id, - Geneweaver_Gene.ode_ref_id == ode_ref_id, + GeneweaverGene.ode_gene_id == ode_gene_id, + GeneweaverGene.ode_ref_id == ode_ref_id, ) .first() ).gdb_id @@ -270,18 +270,18 @@ class get_orthologs_by_to_and_from_gene(Resource): @NS.marshal_with(ortholog_model) def get(self, from_ode_ref_id, from_ode_gene_id, to_ode_ref_id, to_ode_gene_id): to_gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == to_ode_gene_id, - Geneweaver_Gene.ode_ref_id == to_ode_ref_id, + GeneweaverGene.ode_gene_id == to_ode_gene_id, + GeneweaverGene.ode_ref_id == to_ode_ref_id, ) .first() ).gdb_id from_gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == from_ode_gene_id, - Geneweaver_Gene.ode_ref_id == from_ode_ref_id, + GeneweaverGene.ode_gene_id == from_ode_gene_id, + GeneweaverGene.ode_ref_id == from_ode_ref_id, ) .first() ).gdb_id @@ -333,10 +333,10 @@ def get(self, from_ode_ref_id, from_ode_gene_id, best): else: modified_best = True gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == from_ode_gene_id, - Geneweaver_Gene.ode_ref_id == from_ode_ref_id, + GeneweaverGene.ode_gene_id == from_ode_gene_id, + GeneweaverGene.ode_ref_id == from_ode_ref_id, ) .first() ).gdb_id @@ -384,18 +384,18 @@ def get( modified_best = True # find from and to gene objects using given information to_gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == to_ode_gene_id, - Geneweaver_Gene.ode_ref_id == to_ode_ref_id, + GeneweaverGene.ode_gene_id == to_ode_gene_id, + GeneweaverGene.ode_ref_id == to_ode_ref_id, ) .first() ).gdb_id from_gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == from_ode_gene_id, - Geneweaver_Gene.ode_ref_id == from_ode_ref_id, + GeneweaverGene.ode_gene_id == from_ode_gene_id, + GeneweaverGene.ode_ref_id == from_ode_ref_id, ) .first() ).gdb_id @@ -461,18 +461,18 @@ def get( inp = True to_gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == to_ode_gene_id, - Geneweaver_Gene.ode_ref_id == to_ode_ref_id, + GeneweaverGene.ode_gene_id == to_ode_gene_id, + GeneweaverGene.ode_ref_id == to_ode_ref_id, ) .first() ).gdb_id from_gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == from_ode_gene_id, - Geneweaver_Gene.ode_ref_id == from_ode_ref_id, + GeneweaverGene.ode_gene_id == from_ode_gene_id, + GeneweaverGene.ode_ref_id == from_ode_ref_id, ) .first() ).gdb_id @@ -579,10 +579,10 @@ def get(self, ode_ref_id, ode_gene_id): # find gene gdb_id to use the convertODEtoAGR function that converts the # geneweaver ode_ref_id into the agr gn_ref_id gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == ode_gene_id, - Geneweaver_Gene.ode_ref_id == ode_ref_id, + GeneweaverGene.ode_gene_id == ode_gene_id, + GeneweaverGene.ode_ref_id == ode_ref_id, ) .first() ).gdb_id @@ -619,10 +619,10 @@ class get_gene_species_name(Resource): @NS.doc("returns the species of a specified gene") def get(self, ode_ref_id, ode_gene_id): gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == ode_gene_id, - Geneweaver_Gene.ode_ref_id == ode_ref_id, + GeneweaverGene.ode_gene_id == ode_gene_id, + GeneweaverGene.ode_ref_id == ode_ref_id, ) .first() ).gdb_id @@ -1108,7 +1108,7 @@ def get(self, gn_id): ref = ref[ind:] # find matching ode_gene_id ode_gene_id = ( - db.query(Geneweaver_Gene).filter(Geneweaver_Gene.ode_ref_id == ref).first() + db.query(GeneweaverGene).filter(GeneweaverGene.ode_ref_id == ref).first() ).ode_gene_id if not ode_gene_id: abort(404, message="matching ode_gene_id not found") @@ -1125,10 +1125,10 @@ class id_convert_ode_to_agr(Resource): @NS.doc("converts an ode gene id to the corresponding agr gene id") def get(self, ode_gene_id, ode_ref_id): gdb_id = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == ode_gene_id, - Geneweaver_Gene.ode_ref_id == ode_ref_id, + GeneweaverGene.ode_gene_id == ode_gene_id, + GeneweaverGene.ode_ref_id == ode_ref_id, ) .first() ).gdb_id @@ -1150,7 +1150,7 @@ class get_ode_gene_by_gdb_id(Resource): @NS.doc("return all ode_genes with the specified gdb_id") @NS.marshal_with(gw_gene_model) def get(self, gdb_id): - genes = db.query(Geneweaver_Gene).filter(Geneweaver_Gene.gdb_id == gdb_id).all() + genes = db.query(GeneweaverGene).filter(GeneweaverGene.gdb_id == gdb_id).all() if not genes: abort(404, message="No genes were found with specified gdb_id") return genes @@ -1168,8 +1168,8 @@ class get_ode_gene_by_gene_id(Resource): @NS.marshal_with(gw_gene_model) def get(self, ode_gene_id): genes = ( - db.query(Geneweaver_Gene) - .filter(Geneweaver_Gene.ode_gene_id == ode_gene_id) + db.query(GeneweaverGene) + .filter(GeneweaverGene.ode_gene_id == ode_gene_id) .all() ) if not genes: @@ -1190,15 +1190,15 @@ class get_ode_gene_by_species(Resource): @NS.marshal_with(gw_gene_model) def get(self, ode_gene_id, sp_name): sp_id = ( - db.query(Geneweaver_Species) - .filter(Geneweaver_Species.sp_name == sp_name) + db.query(GeneweaverSpecies) + .filter(GeneweaverSpecies.sp_name == sp_name) .first() ).sp_id genes = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == ode_gene_id, - Geneweaver_Gene.sp_id == sp_id, + GeneweaverGene.ode_gene_id == ode_gene_id, + GeneweaverGene.sp_id == sp_id, ) .all() ) @@ -1223,8 +1223,8 @@ def convert_ode_ref_to_agr(ode_ref): # where necessary for the AGR format. ref = ode_ref gdb_id = ( - db.query(Geneweaver_Gene.gdb_id) - .filter(Geneweaver_Gene.ode_ref_id == ode_ref) + db.query(GeneweaverGene.gdb_id) + .filter(GeneweaverGene.ode_ref_id == ode_ref) .first() ) if gdb_id: @@ -1268,8 +1268,8 @@ def convert_agr_ref_to_ode(gn_ref_id): def convert_species_ode_to_agr(ode_sp_id): # find the species name, return None if not found in the geneweaver db species_name = ( - db.query(Geneweaver_Species.sp_name) - .filter(Geneweaver_Species.sp_id == ode_sp_id) + db.query(GeneweaverSpecies.sp_name) + .filter(GeneweaverSpecies.sp_id == ode_sp_id) .first() ) if species_name: @@ -1292,8 +1292,8 @@ def convert_species_agr_to_ode(agr_sp_id): return None # get the geneweaver species id ode_sp_id = ( - db.query(Geneweaver_Species.sp_id) - .filter(Geneweaver_Species.sp_name == species_name) + db.query(GeneweaverSpecies.sp_id) + .filter(GeneweaverSpecies.sp_name == species_name) .first() ) if ode_sp_id: @@ -1332,8 +1332,8 @@ class get_homology_by_ode_gene_id(Resource): def get(self, ode_gene_id): # find the gn_ids for any gene with the given ode_gene_id result = ( - db.query(Geneweaver_Gene.ode_ref_id) - .filter(Geneweaver_Gene.ode_gene_id == ode_gene_id) + db.query(GeneweaverGene.ode_ref_id) + .filter(GeneweaverGene.ode_gene_id == ode_gene_id) .all() ) ode_refs = [] @@ -1364,8 +1364,8 @@ class get_homologous_ode_gene_ids_for_gene(Resource): def get(self, ode_ref_id, gdb_name): agr_ref_id = convert_ode_ref_to_agr(ode_ref_id) ( - db.query(Geneweaver_GeneDB.gdb_id) - .filter(Geneweaver_GeneDB.gdb_name == gdb_name) + db.query(GeneweaverGeneDB.gdb_id) + .filter(GeneweaverGeneDB.gdb_name == gdb_name) .first() ) @@ -1389,8 +1389,8 @@ def get(self, ode_ref_id, gdb_name): ] ode_gene_ids = ( - db.query(Geneweaver_Gene.ode_gene_id) - .filter(Geneweaver_Gene.ode_ref_id.in_(homologous_refs)) + db.query(GeneweaverGene.ode_gene_id) + .filter(GeneweaverGene.ode_ref_id.in_(homologous_refs)) .all() ) ode_gene_ids = [l[0] for l in set(ode_gene_ids)] @@ -1413,8 +1413,8 @@ def get(self): # find the gn_ids for any gene with the given ode_gene_id result = ( - db.query(Geneweaver_Gene.ode_ref_id) - .filter(Geneweaver_Gene.ode_gene_id.in_(ode_gene_ids)) + db.query(GeneweaverGene.ode_ref_id) + .filter(GeneweaverGene.ode_gene_id.in_(ode_gene_ids)) .all() ) ode_refs = [] @@ -1454,8 +1454,8 @@ def get(self, hom_id, target_gdb_id): # find the sp_id from the given gdb_id target_sp_id = ( - db.query(Geneweaver_GeneDB.sp_id) - .filter(Geneweaver_GeneDB.gdb_id == target_gdb_id) + db.query(GeneweaverGeneDB.sp_id) + .filter(GeneweaverGeneDB.gdb_id == target_gdb_id) .first() ) target_sp_id = convert_species_ode_to_agr(target_sp_id[0]) @@ -1500,10 +1500,10 @@ def get(self, from_ode_gene_id, gdb_id): # database, so it is filtered out in the search. agr_compatible_gdb_ids = [10, 11, 12, 13, 14, 15, 16] genes = ( - db.query(Geneweaver_Gene.ode_ref_id) + db.query(GeneweaverGene.ode_ref_id) .filter( - Geneweaver_Gene.ode_gene_id == from_ode_gene_id, - Geneweaver_Gene.gdb_id.in_(agr_compatible_gdb_ids), + GeneweaverGene.ode_gene_id == from_ode_gene_id, + GeneweaverGene.gdb_id.in_(agr_compatible_gdb_ids), ) .all() ) @@ -1615,15 +1615,15 @@ def get(self): ode_ref_id = gs1_gn_ids[gn_id] ode_gene_id = ( - db.query(Geneweaver_Gene.ode_gene_id) - .filter(Geneweaver_Gene.ode_ref_id == ode_ref_id) + db.query(GeneweaverGene.ode_gene_id) + .filter(GeneweaverGene.ode_ref_id == ode_ref_id) .first() )[0] gene_symbol = ( - db.query(Geneweaver_Gene.ode_ref_id) + db.query(GeneweaverGene.ode_ref_id) .filter( - Geneweaver_Gene.ode_gene_id == ode_gene_id, - Geneweaver_Gene.gdb_id == 7, + GeneweaverGene.ode_gene_id == ode_gene_id, + GeneweaverGene.gdb_id == 7, ) .first() )[0] @@ -1696,16 +1696,16 @@ def get(self): homologous_new_species_gw_refs.append(convert_agr_ref_to_ode(r)) ode_gene_ids = ( - db.query(Geneweaver_Gene.ode_gene_id) - .filter(Geneweaver_Gene.ode_ref_id.in_(homologous_new_species_gw_refs)) + db.query(GeneweaverGene.ode_gene_id) + .filter(GeneweaverGene.ode_ref_id.in_(homologous_new_species_gw_refs)) .all() ) gene_symbols = ( - db.query(Geneweaver_Gene.ode_ref_id) + db.query(GeneweaverGene.ode_ref_id) .filter( - Geneweaver_Gene.ode_gene_id.in_(ode_gene_ids), - Geneweaver_Gene.gdb_id == 7, - Geneweaver_Gene.ode_pref is True, + GeneweaverGene.ode_gene_id.in_(ode_gene_ids), + GeneweaverGene.gdb_id == 7, + GeneweaverGene.ode_pref is True, ) .all() ) @@ -1721,8 +1721,8 @@ class if_gene_has_homolog(Resource): def get(self, ode_gene_id): ref = ( - db.query(Geneweaver_Gene.ode_ref_id) - .filter(Geneweaver_Gene.ode_gene_id == ode_gene_id) + db.query(GeneweaverGene.ode_ref_id) + .filter(GeneweaverGene.ode_gene_id == ode_gene_id) .all() ) result = 0 @@ -1755,14 +1755,14 @@ def get(self, sym, orig_species, homologous_species): # get the sp_id from the species name orig_sp_id = ( - db.query(Geneweaver_Species.sp_id) - .filter(Geneweaver_Species.sp_name == orig_species) + db.query(GeneweaverSpecies.sp_id) + .filter(GeneweaverSpecies.sp_name == orig_species) .first() ) # get the target homologous species sp_id gdb_id = ( - db.query(Geneweaver_GeneDB.gdb_id) - .filter(Geneweaver_GeneDB.sp_id == orig_sp_id) + db.query(GeneweaverGeneDB.gdb_id) + .filter(GeneweaverGeneDB.sp_id == orig_sp_id) .first() ) homologous_sp_id = ( @@ -1774,10 +1774,10 @@ def get(self, sym, orig_species, homologous_species): data = {} for s in symbols: gene_id = ( - db.query(Geneweaver_Gene.ode_gene_id) + db.query(GeneweaverGene.ode_gene_id) .filter( - Geneweaver_Gene.ode_ref_id == s, - Geneweaver_Gene.sp_id.in_(orig_sp_id), + GeneweaverGene.ode_ref_id == s, + GeneweaverGene.sp_id.in_(orig_sp_id), ) .first() ) @@ -1787,10 +1787,10 @@ def get(self, sym, orig_species, homologous_species): continue else: ref = ( - db.query(Geneweaver_Gene.ode_ref_id) + db.query(GeneweaverGene.ode_ref_id) .filter( - Geneweaver_Gene.ode_gene_id == gene_id, - Geneweaver_Gene.gdb_id.in_(gdb_id), + GeneweaverGene.ode_gene_id == gene_id, + GeneweaverGene.gdb_id.in_(gdb_id), ) .first() ) @@ -1831,17 +1831,17 @@ def get(self, sym, orig_species, homologous_species): for o in ortho_refs: # get the ode_gene_id from the ode_ref_id ortho_id = ( - db.query(Geneweaver_Gene.ode_gene_id) - .filter(Geneweaver_Gene.ode_ref_id == o) + db.query(GeneweaverGene.ode_gene_id) + .filter(GeneweaverGene.ode_ref_id == o) .first() ) # convert the ode_gene_id to the gene symbol ortho_sym = ( - db.query(Geneweaver_Gene.ode_ref_id) + db.query(GeneweaverGene.ode_ref_id) .filter( - Geneweaver_Gene.ode_gene_id == ortho_id, - Geneweaver_Gene.gdb_id == 7, - Geneweaver_Gene.ode_pref is True, + GeneweaverGene.ode_gene_id == ortho_id, + GeneweaverGene.gdb_id == 7, + GeneweaverGene.ode_pref is True, ) .first() ) diff --git a/src/geneweaver/aon/controller/genes.py b/src/geneweaver/aon/controller/genes.py index 5eb9943..774effe 100644 --- a/src/geneweaver/aon/controller/genes.py +++ b/src/geneweaver/aon/controller/genes.py @@ -44,22 +44,3 @@ def get_gene_by_ref_id( if ref_id_type == ReferenceGeneIDType.GW: ref_id = convert_service.ode_ref_to_agr(db, ref_id) return genes_service.gene_by_ref_id(ref_id, db) - - -# @router.get("/covert/{id}") -# def covert_gene_id( -# id: str, -# from_type: ReferenceGeneIDType = ReferenceGeneIDType.AON, -# to_type: ReferenceGeneIDType = ReferenceGeneIDType.GW, -# geneweaver_id_type: Optional[ReferenceGeneIDType] = None, -# db: deps.Session = Depends(deps.session), -# ): -# """Convert gene id.""" -# if from_type == ReferenceGeneIDType.GW and to_type == ReferenceGeneIDType.AON: -# return convert_service.ode_ref_to_agr(db, id) -# -# if from_type == ReferenceGeneIDType.AON and to_type == ReferenceGeneIDType.GW: -# return convert_service.agr_ref_to_ode(id) -# -# else: -# raise HTTPException(400, detail="Invalid gene id type conversion.") diff --git a/src/geneweaver/aon/controller/versions.py b/src/geneweaver/aon/controller/versions.py index 1638ba8..d87d3d9 100644 --- a/src/geneweaver/aon/controller/versions.py +++ b/src/geneweaver/aon/controller/versions.py @@ -1,4 +1,3 @@ - from fastapi import APIRouter, Depends, Request from geneweaver.aon import dependencies as deps from geneweaver.aon.models import Version diff --git a/src/geneweaver/aon/core/schema_version.py b/src/geneweaver/aon/core/schema_version.py index 4c2c184..809974e 100644 --- a/src/geneweaver/aon/core/schema_version.py +++ b/src/geneweaver/aon/core/schema_version.py @@ -56,7 +56,9 @@ def mark_schema_version_load_complete(version_id: int): session.close() -def set_up_sessionmanager(version: Optional[Version]) -> Tuple[sessionmaker, Tuple[Engine, Engine]]: +def set_up_sessionmanager( + version: Optional[Version], +) -> Tuple[sessionmaker, Tuple[Engine, Engine]]: """Set up the session manager. :param version: The schema version to use. diff --git a/src/geneweaver/aon/enum.py b/src/geneweaver/aon/enum.py index 37ffee8..c6f7fb1 100644 --- a/src/geneweaver/aon/enum.py +++ b/src/geneweaver/aon/enum.py @@ -1,11 +1,16 @@ +"""Enumerations specifically for the GW AON API.""" + from enum import Enum class ReferenceGeneIDType(Enum): + """Enum for defining gene reference id types.""" AON = "aon" GW = "gw" class InternalGeneIDType(Enum): + """Enum for defining internal gene id types.""" + AON = "aon" GW = "gw" diff --git a/src/geneweaver/aon/load/add_missing_info.py b/src/geneweaver/aon/load/add_missing_info.py deleted file mode 100644 index 17ae116..0000000 --- a/src/geneweaver/aon/load/add_missing_info.py +++ /dev/null @@ -1,77 +0,0 @@ -from csv import reader - -from geneweaver.aon.core.database import SessionLocal -from geneweaver.aon.models import Gene, Ortholog, Species - -db = SessionLocal() - -gene_file_path = "missing_info/missing_genes.csv" -ortholog_file_path = "missing_info/missing_orthologs.csv" - - -def add_missing_species(): - """:description: adds three missing species to sp_species table.""" - species_dict = { - "Gallus gallus": 9031, - "Canis familiaris": 9615, - "Macaca mulatta": 9544, - } - i = 8 - for s in species_dict.keys(): - species = Species(sp_id=i, sp_name=s, sp_taxon_id=species_dict[s]) - db.add(species) - i += 1 - db.commit() - - -def add_missing_genes(file_path): - """:param: file_path - file path to file missing_genes.csv that contains all missing genes - to be added to the database - :description: adds genes of the three missing species to the gn_gene table - """ - with open(file_path, "r") as f: - csv_reader = reader(f) - next(csv_reader) - gene_file_list = list(csv_reader) - - gene_objects = [] - for g in gene_file_list: - gene = Gene(gn_id=int(g[0]), gn_ref_id=g[1], gn_prefix=g[2], sp_id=int(g[3])) - gene_objects.append(gene) - db.bulk_save_objects(gene_objects) - db.commit() - - -def add_missing_orthologs(file_path): - """:param: file_path - file path to file missing_orthologs.csv that contains all missing - orthologs to be added to the database - :description: adds orthologs of the three missing species to the gn_gene table - """ - with open(file_path, "r") as f: - csv_reader = reader(f) - next(csv_reader) - ortholog_file_list = list(csv_reader) - - ortholog_objects = [] - - for o in ortholog_file_list: - ortholog = Ortholog( - ort_id=int(o[0]), - from_gene=int(o[1]), - to_gene=int(o[2]), - ort_is_best=bool(o[3]), - ort_is_best_revised=bool(o[4]), - ort_is_best_is_adjusted=bool(o[5]), - ort_num_possible_match_algorithms=int(o[6]), - ort_source_name=o[7], - ) - ortholog_objects.append(ortholog) - - db.bulk_save_objects(ortholog_objects) - db.commit() - - -if __name__ == "__main__": - add_missing_species() - add_missing_genes(gene_file_path) - add_missing_orthologs(ortholog_file_path) diff --git a/src/geneweaver/aon/load/agr/__init__.py b/src/geneweaver/aon/load/agr/__init__.py index 20a1fce..e6cabb1 100644 --- a/src/geneweaver/aon/load/agr/__init__.py +++ b/src/geneweaver/aon/load/agr/__init__.py @@ -1 +1,3 @@ -from . import load, sources +"""Module for the AGR based loading code.""" + +from . import load, sources # noqa: F401 diff --git a/src/geneweaver/aon/load/fill_missing_species_information.py b/src/geneweaver/aon/load/fill_missing_species_information.py deleted file mode 100644 index e2c9d26..0000000 --- a/src/geneweaver/aon/load/fill_missing_species_information.py +++ /dev/null @@ -1,355 +0,0 @@ -import itertools -import time - -from geneweaver.aon.controller.flask.controller import ( - convert_ode_ref_to_agr, - convert_species_ode_to_agr, -) -from geneweaver.aon.core.database import SessionLocal -from geneweaver.aon.models import Gene, Geneweaver_Gene, Ortholog, Species -from psycopg_pool import ConnectionPool -from sqlalchemy.ext.declarative import declarative_base - -start_time = time.time() - -BASE = declarative_base() - -db = SessionLocal() - -########################################################################################## -# connection to both databases using connection pool - - -class GeneWeaverConnectionPool(ConnectionPool): - """Extend ConnectionPool to initialize the search_path.""" - - def __init__(self, minconn, maxconn, *args, **kwargs) -> None: - ConnectionPool.__init__(self, minconn, maxconn, *args, **kwargs) - - def _connect(self, key=None): - """Create a new connection and set its search_path.""" - conn = super(GeneWeaverConnectionPool, self)._connect(key) - conn.set_client_encoding("UTF-8") - cursor = conn.cursor() - cursor.execute("SET search_path TO production, extsrc, odestatic;") - conn.commit() - - return conn - - -# the global threaded connection pool that should be used for all DB -# connections in this application - -# TODO - update database information to relevant databases -pool = GeneWeaverConnectionPool( - 5, - 20, - database="geneweaver", - user="user", - password="pass", - host="localhost", - port="2222", -) -pool_agr = GeneWeaverConnectionPool( - 5, 20, database="agr", user="user", password="pass", host="localhost", port="5432" -) - - -# creates cursor for both geneweaver and agr databases -class PooledCursor(object): - """This is the cursor for the Geneweaver database - A cursor obtained from a connection pool. This is suitable for using in a with ... as ...: construct (the - underlying connection will be automatically returned to the connection pool. - """ - - def __init__(self, conn_pool=pool, rollback_on_exception=False) -> None: - self.conn_pool = conn_pool - self.rollback_on_exception = rollback_on_exception - self.connection = None - - def __enter__(self): - self.connection = self.conn_pool.getconn() - return self.connection.cursor() - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.connection is not None: - if self.rollback_on_exception and exc_type is not None: - self.connection.rollback() - - self.conn_pool.putconn(self.connection) - - -class PooledCursorAGR(object): - """This is the cursor for the AGR database.""" - - def __init__(self, conn_pool=pool_agr, rollback_on_exception=False) -> None: - self.conn_pool = conn_pool - self.rollback_on_exception = rollback_on_exception - self.connection = None - - def __enter__(self): - self.connection = self.conn_pool.getconn() - return self.connection.cursor() - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.connection is not None: - if self.rollback_on_exception and exc_type is not None: - self.connection.rollback() - - self.conn_pool.putconn(self.connection) - - -########################################################################################## - - -def add_missing_species(): - """:description: adds three missing species to sp_species table.""" - species_dict = { - "Gallus gallus": 9031, - "Canis familiaris": 9615, - "Macaca mulatta": 9544, - } - i = 8 - for s in species_dict.keys(): - species = Species(sp_id=i, sp_name=s, sp_taxon_id=species_dict[s]) - db.add(species) - i += 1 - db.commit() - - -########################################################################################## - - -def convert_gdb_to_prefix(gdb_id): - """:param: gdb_id - gdb_id from genedb table in geneweaver database, used as key for - gn_prefix in agr gn_gene table because in agr, each prefix corresponds to - one genedb - :return: gn_prefix from gdb_dict corresponding to param gdb_id - :description: converts gdb_id to gn_prefix to communicate between agr and geneweaver - databases - """ - gdb_dict = { - 1: "entrez", - 2: "ensembl", - 3: "ensembl_protein", - 4: "ensembl_transcript", - 5: "unigene", - 7: "symbol", - 8: "unannotated", - 10: "MGI", - 11: "HGNC", - 12: "RGD", - 13: "ZFIN", - 14: "FB", - 15: "WB", - 16: "SGD", - 17: "miRBase", - 20: "CGNC", - 21: "Variant", - } - return gdb_dict[int(gdb_id)] - - -def add_missing_genes(): - """:description: adds genes from geneweaver gene table for the three missing species. - parses information from this table to create Gene objects to go into gn_gene - table in agr. - """ - # query for a list of geneweaver genes from Gallus gallus (sp_id=10, gdb_id=20), - # Canis familiaris(sp_id=11, gdb_id=2), and Macaca mulatta (sp_id=6, gdb_id=1) - with PooledCursor() as cursor: - cursor.execute( - """ - SELECT ode_ref_id, gdb_id, sp_id FROM extsrc.gene WHERE sp_id IN (6,10,11) - AND gdb_id in (1,2,20); - """ - ) - gw_genes = cursor.fetchall() - - i = 0 - genes = [] - for g in gw_genes: - gn_ref_id = g[0] - gn_prefix = convert_gdb_to_prefix(g[1]) - sp_id = convert_species_ode_to_agr(int(g[2])) - - gene = Gene(gn_ref_id=gn_ref_id, gn_prefix=gn_prefix, sp_id=sp_id) - genes.append(gene) - - # adds genes to database 1000 at a time - if i % 1000 == 0 and i != 0: - db.bulk_save_objects(genes) - db.commit() - genes = [] - i = 0 - else: - i += 1 - - -########################################################################################## - - -def get_homolog_information(): - with PooledCursorAGR() as cursorAGR: - # get all ode_gene_ids of the genes in the gn_gene table for the 3 missing species - cursorAGR.execute( - """ - select ode_gene_id from geneweaver.gene where ode_ref_id in ( - select gn_ref_id from public.gn_gene where sp_id in (8,9,10)) - ; - """ - ) - output = cursorAGR.fetchall() - - # put output into list format - gw_genes = [] - for g in output: - gw_genes.append(str(g[0])) - - with PooledCursor() as cursor: - # get hom_id, ode_gene_id, and sp_id from the geneweaver homology table for any homolog - # that is a member of a cluster that contains a gene in agr and of the 3 missing species, - # also orders by hom_id to make it easier to parse - cursor.execute( - """ - select hom_id, ode_gene_id, sp_id from extsrc.homology h1 where h1.hom_id in ( - select hom_id from extsrc.homology h2 where ode_gene_id in ({})) - order by hom_id; - """.format( - ",".join([str(i) for i in gw_genes]) - ) - ) - homologs = cursor.fetchall() - - return homologs - - -def add_missing_orthologs(homologs): - # starting hom_id is the first hom_id of the first homolog, keeps track of current cluster - curr_hom_id = homologs[0][0] - # will hold all homolog genes that are not from the new species - homolog_set = [] - # holds all homolog genes from the new species - new_species_homologs = [] - - for h in homologs: - # h is in current cluster - if h[0] == curr_hom_id: - # check if h is part of the new species - if h[2] in [6, 10, 11]: - new_species_homologs.append(h) - # if h is not in new species, check if the gene is in the agr database. if it is, - # add to homolog_set - else: - # check if in agr database by getting ode_ref_id, converting that to gn_ref_id, - # and searching gn_gene with that agr_ref_id. if all queries are valid, gene is - # in agr. - gw_gene_ref = ( - db.query(Geneweaver_Gene) - .filter( - Geneweaver_Gene.ode_gene_id == h[1], - Geneweaver_Gene.sp_id == h[2], - Geneweaver_Gene.gdb_id.in_([10, 11, 12, 13, 14, 15, 16]), - ) - .first() - ) - if gw_gene_ref: - agr_ref = convert_ode_ref_to_agr(gw_gene_ref.ode_ref_id) - gene_in_agr = ( - db.query(Gene).filter(Gene.gn_ref_id == agr_ref).first() - ) - if gene_in_agr: - homolog_set.append(h) - - # h is not in current cluster, new homology set - # add new relations from new_species_homologs to rest of homolog_set - # reset homolog_set and new_species_homologs - if h[0] != curr_hom_id: - orthos = [] - from_agr_genes = {} - - # iterate through new_species_homologs to get the gn_id of each gene - for n in new_species_homologs: - n_gw_ref = ( - db.query(Geneweaver_Gene) - .filter( - Geneweaver_Gene.ode_gene_id == n[1], - Geneweaver_Gene.sp_id == n[2], - Geneweaver_Gene.gdb_id.in_([1, 2, 10]), - ) - .first() - ) - n_agr_gene = ( - db.query(Gene).filter(Gene.gn_ref_id == n_gw_ref.ode_ref_id).first() - ) - from_agr_genes[n[1]] = n_agr_gene.gn_id - - # get permutations of every combination between new_species_homologs and - # homolog_set - pairs = list(itertools.product(new_species_homologs, homolog_set)) - - for p in pairs: - # from gene is first pairs object, to gene is second - f = p[0] - t = p[1] - - # get to_gene gn_ref_id - t_gene_ref = ( - db.query(Geneweaver_Gene) - .filter( - Geneweaver_Gene.ode_gene_id == t[1], - Geneweaver_Gene.sp_id == t[2], - Geneweaver_Gene.gdb_id.in_([10, 11, 12, 13, 14, 15, 16]), - ) - .first() - ) - t_agr_ref = convert_ode_ref_to_agr(t_gene_ref.ode_ref_id) - - # check that to gene is in agr and geneweaver database - if not t_gene_ref or not t_agr_ref: - continue - agr_to_gene = db.query(Gene).filter(Gene.gn_ref_id == t_agr_ref).first() - if not agr_to_gene: - continue - - to_gene = agr_to_gene.gn_id - from_gene = from_agr_genes[f[1]] - ort_is_best = True - ort_is_best_revised = True - ort_is_best_adjusted = True - ort_num_possible_match_algorithms = 0 - ortho = Ortholog( - from_gene=from_gene, - to_gene=to_gene, - ort_is_best=ort_is_best, - ort_is_best_revised=ort_is_best_revised, - ort_is_best_is_adjusted=ort_is_best_adjusted, - ort_num_possible_match_algorithms=ort_num_possible_match_algorithms, - ort_source_name="Homologene", - ) - orthos.append(ortho) - db.bulk_save_objects(orthos) - db.commit() - - # reset homolog_set and new_species_homologs - homolog_set = [] - new_species_homologs = [] - - # add current gene to appropriate list - if h[2] in [6, 10, 11]: - new_species_homologs.append(h) - else: - homolog_set.append(h) - - # reset curr_hom_id - curr_hom_id = h[0] - - -########################################################################################## - -if __name__ == "__main__": - add_missing_species() - add_missing_genes() - h = get_homolog_information() - add_missing_orthologs(h) - db.close() diff --git a/src/geneweaver/aon/load/geneweaver/__init__.py b/src/geneweaver/aon/load/geneweaver/__init__.py index 1b59daf..fb7c613 100644 --- a/src/geneweaver/aon/load/geneweaver/__init__.py +++ b/src/geneweaver/aon/load/geneweaver/__init__.py @@ -1 +1,3 @@ -from . import genes, homologs, species +"""Module for the geneweaver based loading code.""" + +from . import genes, homologs # noqa: F401 diff --git a/src/geneweaver/aon/load/geneweaver/homologs.py b/src/geneweaver/aon/load/geneweaver/homologs.py index c508a0f..443765d 100644 --- a/src/geneweaver/aon/load/geneweaver/homologs.py +++ b/src/geneweaver/aon/load/geneweaver/homologs.py @@ -4,7 +4,7 @@ from typing import Optional from geneweaver.aon.controller.flask.controller import convert_ode_ref_to_agr -from geneweaver.aon.models import Gene, Geneweaver_Gene, Ortholog +from geneweaver.aon.models import Gene, GeneweaverGene, Ortholog from psycopg import Cursor, sql from sqlalchemy.orm import Session @@ -56,17 +56,17 @@ def add_missing_orthologs_2(db: Session, homologs): This refactor stores more information in memory so that it doesn't need to make as many database calls, which can be slow when using the cloud-sql-proxy. - Homologs should be a list of tuples, where each tuple contains the hom_id, ode_gene_id, - and sp_id of a gene in the geneweaver database. + Homologs should be a list of tuples, where each tuple contains the hom_id, + ode_gene_id, and sp_id of a gene in the geneweaver database. """ geneweaver_genes = {} agr_genes = {} gw_gene_query = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id.in_([h[1] for h in homologs]), - Geneweaver_Gene.sp_id.in_([6, 10, 11]), + GeneweaverGene.ode_gene_id.in_([h[1] for h in homologs]), + GeneweaverGene.sp_id.in_([6, 10, 11]), ) .all() ) @@ -84,7 +84,8 @@ def add_missing_orthologs_2(db: Session, homologs): def add_missing_orthologs(db: Session, homologs): - # starting hom_id is the first hom_id of the first homolog, keeps track of current cluster + # starting hom_id is the first hom_id of the first homolog, + # keeps track of current cluster curr_hom_id = homologs[0][0] # will hold all homolog genes that are not from the new species homolog_set = [] @@ -97,18 +98,18 @@ def add_missing_orthologs(db: Session, homologs): # check if h is part of the new species if h[2] in [6, 10, 11]: new_species_homologs.append(h) - # if h is not in new species, check if the gene is in the agr database. if it is, - # add to homolog_set + # if h is not in new species, check if the gene is in the agr database. + # if it is, add to homolog_set else: - # check if in agr database by getting ode_ref_id, converting that to gn_ref_id, - # and searching gn_gene with that agr_ref_id. if all queries are valid, gene is - # in agr. + # check if in agr database by getting ode_ref_id, converting that to + # gn_ref_id, and searching gn_gene with that agr_ref_id. if all + # queries are valid, gene is in agr. gw_gene_ref = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == h[1], - Geneweaver_Gene.sp_id == h[2], - Geneweaver_Gene.gdb_id.in_([10, 11, 12, 13, 14, 15, 16]), + GeneweaverGene.ode_gene_id == h[1], + GeneweaverGene.sp_id == h[2], + GeneweaverGene.gdb_id.in_([10, 11, 12, 13, 14, 15, 16]), ) .first() ) @@ -130,11 +131,11 @@ def add_missing_orthologs(db: Session, homologs): # iterate through new_species_homologs to get the gn_id of each gene for n in new_species_homologs: n_gw_ref = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == n[1], - Geneweaver_Gene.sp_id == n[2], - Geneweaver_Gene.gdb_id.in_([1, 2, 10]), + GeneweaverGene.ode_gene_id == n[1], + GeneweaverGene.sp_id == n[2], + GeneweaverGene.gdb_id.in_([1, 2, 10]), ) .first() ) @@ -154,11 +155,11 @@ def add_missing_orthologs(db: Session, homologs): # get to_gene gn_ref_id t_gene_ref = ( - db.query(Geneweaver_Gene) + db.query(GeneweaverGene) .filter( - Geneweaver_Gene.ode_gene_id == t[1], - Geneweaver_Gene.sp_id == t[2], - Geneweaver_Gene.gdb_id.in_([10, 11, 12, 13, 14, 15, 16]), + GeneweaverGene.ode_gene_id == t[1], + GeneweaverGene.sp_id == t[2], + GeneweaverGene.gdb_id.in_([10, 11, 12, 13, 14, 15, 16]), ) .first() ) diff --git a/src/geneweaver/aon/load/geneweaver/species.py b/src/geneweaver/aon/load/geneweaver/species.py deleted file mode 100644 index 56adba6..0000000 --- a/src/geneweaver/aon/load/geneweaver/species.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Code for adding missing species information.""" - -from geneweaver.aon.models import Species -from sqlalchemy.orm import Session - - -def add_missing_species(db: Session) -> None: - """Adds three missing species to sp_species table. - - :param db: The database session. - """ - species_dict = { - "Gallus gallus": 9031, - "Canis familiaris": 9615, - "Macaca mulatta": 9544, - } - i = 8 - for s in species_dict.keys(): - species = Species(sp_id=i, sp_name=s, sp_taxon_id=species_dict[s]) - db.add(species) - i += 1 - db.commit() diff --git a/src/geneweaver/aon/models.py b/src/geneweaver/aon/models.py index 2e4f416..405e9a7 100755 --- a/src/geneweaver/aon/models.py +++ b/src/geneweaver/aon/models.py @@ -17,6 +17,8 @@ class Version(BaseAGR): + """Schema version table.""" + __tablename__ = "schema_version" __table_args__ = {"schema": "versions"} @@ -28,8 +30,9 @@ class Version(BaseAGR): class Gene(BaseAGR): - __tablename__ = "gn_gene" + """Gene table.""" + __tablename__ = "gn_gene" gn_id = Column(Integer, primary_key=True) # id gn_ref_id = Column("gn_ref_id", String, unique=True) gn_prefix = Column("gn_prefix", String) @@ -37,6 +40,8 @@ class Gene(BaseAGR): class Species(BaseAGR): + """Species table.""" + __tablename__ = "sp_species" sp_id = Column(Integer, primary_key=True) # id sp_name = Column(String, nullable=False) # name @@ -44,6 +49,8 @@ class Species(BaseAGR): class OrthologAlgorithms(BaseAGR): + """Ortholog algorithms table.""" + __tablename__ = "ora_ortholog_algorithms" ora_id = Column(Integer, primary_key=True) alg_id = Column(ForeignKey("alg_algorithm.alg_id")) @@ -51,6 +58,8 @@ class OrthologAlgorithms(BaseAGR): class Ortholog(BaseAGR): + """Ortholog table.""" + __tablename__ = "ort_ortholog" ort_id = Column(Integer, primary_key=True) # id from_gene = Column(ForeignKey("gn_gene.gn_id")) @@ -69,12 +78,16 @@ class Ortholog(BaseAGR): class Algorithm(BaseAGR): + """Algorithm table.""" + __tablename__ = "alg_algorithm" alg_id = Column(Integer, primary_key=True) # id alg_name = Column(String, unique=True) # name class Homology(BaseAGR): + """Homology table.""" + __tablename__ = "hom_homology" hom_id = Column(Integer) gn_id = Column(ForeignKey("gn_gene.gn_id")) @@ -83,8 +96,12 @@ class Homology(BaseAGR): __table_args__ = (PrimaryKeyConstraint("hom_id", "gn_id"),) -# The following models correspond to tables in the geneweaver database, so they are created using BaseGW -class Geneweaver_Species(BaseGW): +# The following models correspond to tables in the geneweaver database, +# so they are created using BaseGW + +class GeneweaverSpecies(BaseGW): + """Geneweaver species table.""" + __tablename__ = "species" __table_args__ = {"schema": "odestatic"} sp_id = Column(Integer, primary_key=True, unique=True) @@ -96,7 +113,9 @@ class Geneweaver_Species(BaseGW): sp_source_data = Column(Text) -class Geneweaver_Gene(BaseGW): +class GeneweaverGene(BaseGW): + """Geneweaver gene table.""" + __tablename__ = "gene" ode_gene_id = Column(BIGINT) ode_ref_id = Column(VARCHAR) @@ -111,7 +130,9 @@ class Geneweaver_Gene(BaseGW): ) -class Geneweaver_GeneDB(BaseGW): +class GeneweaverGeneDB(BaseGW): + """Geneweaver gene database table.""" + __tablename__ = "genedb" __table_args__ = {"schema": "odestatic"} gdb_id = Column(Integer, primary_key=True, unique=True) diff --git a/src/geneweaver/aon/service/convert.py b/src/geneweaver/aon/service/convert.py index 21efab5..47c3bc8 100644 --- a/src/geneweaver/aon/service/convert.py +++ b/src/geneweaver/aon/service/convert.py @@ -1,6 +1,6 @@ from geneweaver.aon.models import ( - Geneweaver_Gene, - Geneweaver_Species, + GeneweaverGene, + GeneweaverSpecies, Species, ) from geneweaver.core.enum import GeneIdentifier @@ -17,8 +17,8 @@ def ode_ref_to_agr(db: Session, ode_ref): ref = ode_ref gdb_id = ( - db.query(Geneweaver_Gene.gdb_id) - .filter(Geneweaver_Gene.ode_ref_id == ode_ref) + db.query(GeneweaverGene.gdb_id) + .filter(GeneweaverGene.ode_ref_id == ode_ref) .first() ) if gdb_id: @@ -64,8 +64,8 @@ def agr_ref_to_ode(gn_ref_id): def species_ode_to_agr(db: Session, ode_sp_id): # find the species name, return None if not found in the geneweaver db species_name = ( - db.query(Geneweaver_Species.sp_name) - .filter(Geneweaver_Species.sp_id == ode_sp_id) + db.query(GeneweaverSpecies.sp_name) + .filter(GeneweaverSpecies.sp_id == ode_sp_id) .first() ) if species_name: @@ -88,8 +88,8 @@ def species_agr_to_ode(db: Session, agr_sp_id): return None # get the geneweaver species id ode_sp_id = ( - db.query(Geneweaver_Species.sp_id) - .filter(Geneweaver_Species.sp_name == species_name) + db.query(GeneweaverSpecies.sp_id) + .filter(GeneweaverSpecies.sp_name == species_name) .first() ) if ode_sp_id: diff --git a/src/geneweaver/aon/service/species.py b/src/geneweaver/aon/service/species.py index e07a5db..ec28511 100644 --- a/src/geneweaver/aon/service/species.py +++ b/src/geneweaver/aon/service/species.py @@ -1,10 +1,17 @@ +"""Species database queries.""" from typing import Optional -from geneweaver.aon.models import Geneweaver_Species, Species +from geneweaver.aon.models import GeneweaverSpecies, Species from sqlalchemy.orm import Session def get_species(db: Session, name: Optional[str] = None): + """Get species. + + :param db: The database session. + :param name: The species name to search for. + :return: All species that match the queries. + """ base_query = db.query(Species) if name is not None: base_query = base_query.filter(Species.sp_name == name) @@ -13,19 +20,32 @@ def get_species(db: Session, name: Optional[str] = None): def species_by_id(db: Session, species_id: int): - print(str(db.query(Species))) + """Get species by id. + + :param db: The database session. + :param species_id: The species id to search for. + :return: The species info for the provided id. + """ return db.query(Species).get(species_id) def species_by_taxon_id(db: Session, taxon_id: int): - return db.query(Species).filter(Species.sp_taxon_id == taxon_id).all() + """Get species by taxon id. + :param db: The database session. + :param taxon_id: The taxon id to search for. + :return: All species that match the queries. + """ + return db.query(Species).filter(Species.sp_taxon_id == taxon_id).all() -def geneweaver_id(db: Session, species_id: int): - return convert_species_agr_to_ode(db, species_id) +def convert_species_agr_to_ode(db: Session, agr_sp_id: int): + """Convert the species id from AGR to Geneweaver. -def convert_species_agr_to_ode(db: Session, agr_sp_id): + :param db: The database session. + :param agr_sp_id: The species id from AGR. + :return: The species id from Geneweaver. + """ # find the species name, return None if not found in the AGR-normalizer db species_name = db.query(Species.sp_name).filter(Species.sp_id == agr_sp_id).first() if species_name: @@ -34,8 +54,8 @@ def convert_species_agr_to_ode(db: Session, agr_sp_id): return None # get the geneweaver species id ode_sp_id = ( - db.query(Geneweaver_Species.sp_id) - .filter(Geneweaver_Species.sp_name == species_name) + db.query(GeneweaverSpecies.sp_id) + .filter(GeneweaverSpecies.sp_name == species_name) .first() ) if ode_sp_id: diff --git a/src/geneweaver/aon/service/utils.py b/src/geneweaver/aon/service/utils.py index eb87cb5..c54d9e4 100644 --- a/src/geneweaver/aon/service/utils.py +++ b/src/geneweaver/aon/service/utils.py @@ -1,3 +1,5 @@ +"""Utility functions for AON services.""" + from typing import Optional from sqlalchemy.orm import Query @@ -6,6 +8,13 @@ def apply_paging( query: Query, start: Optional[int] = None, limit: Optional[int] = None ) -> Query: + """Apply paging to a query. + + :param query: The query to apply paging to. + :param start: The start index for the query. + :param limit: The limit for the query. + :return: The query with paging applied. + """ if start is not None: query = query.offset(start) if limit is not None: diff --git a/src/geneweaver/aon/temporal/__init__.py b/src/geneweaver/aon/temporal/__init__.py index e69de29..6a8fa85 100644 --- a/src/geneweaver/aon/temporal/__init__.py +++ b/src/geneweaver/aon/temporal/__init__.py @@ -0,0 +1 @@ +"""Definitions for workflows/activitities using temporal.""" diff --git a/src/geneweaver/aon/temporal/activities/__init__.py b/src/geneweaver/aon/temporal/activities/__init__.py index e69de29..e048985 100644 --- a/src/geneweaver/aon/temporal/activities/__init__.py +++ b/src/geneweaver/aon/temporal/activities/__init__.py @@ -0,0 +1 @@ +"""Temporal activities definitions.""" diff --git a/src/geneweaver/aon/temporal/activities/download_source.py b/src/geneweaver/aon/temporal/activities/download_source.py index 04f9f2e..2e4ed2c 100644 --- a/src/geneweaver/aon/temporal/activities/download_source.py +++ b/src/geneweaver/aon/temporal/activities/download_source.py @@ -1,3 +1,5 @@ +"""TemporalIO activities for downloading AGR data.""" + from typing import Optional, Tuple from geneweaver.aon.cli.load import ( @@ -14,34 +16,41 @@ @activity.defn async def get_data_activity(release: Optional[str] = None) -> Tuple[str, str]: + """Get AGR data for a release.""" return get_data(release) @activity.defn async def release_exists_activity(release: str) -> bool: + """Check if an AGR release exists.""" return agr_release_exists(release) @activity.defn async def create_schema_activity(release: str) -> Tuple[str, int]: + """Create a new schema version.""" return create_schema(release) @activity.defn async def load_agr_activity(orthology_file: str, schema_id: int) -> bool: + """Load AGR data for a schema version.""" return load_agr(orthology_file, schema_id) @activity.defn async def load_gw_activity(schema_id: int) -> bool: + """Load GeneWeaver data for a schema version.""" return gw(schema_id) @activity.defn async def load_homology_activity(schema_id: int) -> bool: + """Load homology data for a schema version.""" return homology(schema_id) @activity.defn async def mark_load_complete_activity(schema_id: int) -> bool: + """Mark the load of a schema version as complete.""" return mark_schema_version_load_complete(schema_id) diff --git a/src/geneweaver/aon/temporal/load_data_workflow.py b/src/geneweaver/aon/temporal/load_data_workflow.py index 248bc38..184e705 100644 --- a/src/geneweaver/aon/temporal/load_data_workflow.py +++ b/src/geneweaver/aon/temporal/load_data_workflow.py @@ -1,3 +1,5 @@ +"""GeneWeaver AON data load workflow definition.""" + from datetime import timedelta from typing import Optional @@ -18,8 +20,11 @@ @workflow.defn class GeneWeaverAonDataLoad: + """GeneWeaver AON data load workflow.""" + @workflow.run async def run(self, release: Optional[str] = None) -> bool: + """Run the gene weaver data load workflow.""" orthology_file, release = await workflow.execute_activity( get_data_activity, release, diff --git a/src/geneweaver/aon/temporal/worker.py b/src/geneweaver/aon/temporal/worker.py index cf4d649..c4a7030 100644 --- a/src/geneweaver/aon/temporal/worker.py +++ b/src/geneweaver/aon/temporal/worker.py @@ -1,3 +1,5 @@ +"""Temporal worker for the GeneWeaver AON data load workflow.""" + import asyncio from geneweaver.aon.core.config import config @@ -15,7 +17,8 @@ from temporalio.worker import Worker -async def main(): +async def main() -> None: + """Run the worker.""" client = await Client.connect( config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE ) From 9ba282e490c29db3b425b6c4a0127b7fd6032f68 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 09:23:20 -0500 Subject: [PATCH 09/21] Formatting fixes for the services code --- src/geneweaver/aon/service/__init__.py | 4 ++ src/geneweaver/aon/service/algorithms.py | 27 ++++++++++-- src/geneweaver/aon/service/convert.py | 51 ++++++++++++++++++----- src/geneweaver/aon/service/genes.py | 53 ++++++++++++++++++++---- src/geneweaver/aon/service/homologs.py | 23 ++++++++-- src/geneweaver/aon/service/orthologs.py | 52 ++++++++++++++++++----- src/geneweaver/aon/service/species.py | 11 ++--- 7 files changed, 181 insertions(+), 40 deletions(-) diff --git a/src/geneweaver/aon/service/__init__.py b/src/geneweaver/aon/service/__init__.py index e69de29..5199a1a 100644 --- a/src/geneweaver/aon/service/__init__.py +++ b/src/geneweaver/aon/service/__init__.py @@ -0,0 +1,4 @@ +"""Service functions for the AON API. + +These functions mostly deal with querying the AON database and returning the results. +""" diff --git a/src/geneweaver/aon/service/algorithms.py b/src/geneweaver/aon/service/algorithms.py index 0b34644..4e971dd 100644 --- a/src/geneweaver/aon/service/algorithms.py +++ b/src/geneweaver/aon/service/algorithms.py @@ -1,14 +1,35 @@ +"""Service code for interacting with the Algorithm table.""" + +from typing import List, Type + from geneweaver.aon.models import Algorithm from sqlalchemy.orm import Session -def all_algorithms(db: Session): +def all_algorithms(db: Session) -> List[Type[Algorithm]]: + """Get all algorithms. + + :param db: The database session. + :return: All algorithms. + """ return db.query(Algorithm).all() -def algorithm_by_id(algorithm_id: int, db: Session): +def algorithm_by_id(db: Session, algorithm_id: int) -> Type[Algorithm]: + """Get algorithm by ID. + + :param db: The database session. + :param algorithm_id: The algorithm ID. + :return: The algorithm with the provided ID. + """ return db.query(Algorithm).get(algorithm_id) -def algorithm_by_name(algorithm_name: str, db: Session): +def algorithm_by_name(db: Session, algorithm_name: str) -> List[Type[Algorithm]]: + """Get algorithm by name. + + :param algorithm_name: The algorithm name. + :param db: The database session. + :return: The algorithm with the provided name. + """ return db.query(Algorithm).filter(Algorithm.alg_name == algorithm_name).all() diff --git a/src/geneweaver/aon/service/convert.py b/src/geneweaver/aon/service/convert.py index 47c3bc8..9c6e592 100644 --- a/src/geneweaver/aon/service/convert.py +++ b/src/geneweaver/aon/service/convert.py @@ -1,3 +1,6 @@ +"""Convert between Geneweaver and AON ID formats.""" +from typing import Optional + from geneweaver.aon.models import ( GeneweaverGene, GeneweaverSpecies, @@ -6,14 +9,22 @@ from geneweaver.core.enum import GeneIdentifier from sqlalchemy.orm import Session - # converter functions using first char - these functions improve efficiency and # are used here instead of convertODEtoAGR and convertAGRtoGW because they require # gdb_id and are used more broadly. -def ode_ref_to_agr(db: Session, ode_ref): - # AGR only contains data from some gene data sources geneewaver has and the format is - # slightly different for the reference ids, so this fuction adds prefixes to the ids - # where necessary for the AGR format. + + +def ode_ref_to_agr(db: Session, ode_ref: str) -> str: + """Convert a gene reference ID from Geneweaver to AGR format. + + AGR only contains data from some gene data sources Geneweaver has and the format is + slightly different for the reference ids, so this function adds prefixes to the ids + where necessary for the AGR format. + + :param db: The database session. + :param ode_ref: The gene reference ID from Geneweaver. + :return: The gene reference ID in AGR format. + """ ref = ode_ref gdb_id = ( @@ -40,10 +51,16 @@ def ode_ref_to_agr(db: Session, ode_ref): return ref -def agr_ref_to_ode(gn_ref_id): - # All gene ref ids in AGR contain the gene source database prefix followed by a colon, - # but geneweaver only has some genes in this format. This function adjusts - # the ref ids to match the geneweaver format +def agr_ref_to_ode(gn_ref_id: str) -> str: + """Convert a gene reference ID from AGR to Geneweaver format. + + All gene ref ids in AGR contain the gene source database prefix followed by a colon, + but geneweaver only has some genes in this format. This function adjusts the ref ids + to match the geneweaver format. + + :param gn_ref_id: The gene reference ID in AGR format. + :return: The gene reference ID in Geneweaver format. + """ ref = gn_ref_id prefix = ref[0 : ref.find(":")] @@ -61,7 +78,13 @@ def agr_ref_to_ode(gn_ref_id): return ref -def species_ode_to_agr(db: Session, ode_sp_id): +def species_ode_to_agr(db: Session, ode_sp_id: int) -> Optional[int]: + """Convert a species ID from Geneweaver to AGR format. + + :param db: The database session. + :param ode_sp_id: The species ID from Geneweaver. + :return: The species ID in AGR format. + """ # find the species name, return None if not found in the geneweaver db species_name = ( db.query(GeneweaverSpecies.sp_name) @@ -79,7 +102,13 @@ def species_ode_to_agr(db: Session, ode_sp_id): return agr_sp_id -def species_agr_to_ode(db: Session, agr_sp_id): +def species_agr_to_ode(db: Session, agr_sp_id: int) -> Optional[int]: + """Convert a species ID from AGR to Geneweaver format. + + :param db: The database session. + :param agr_sp_id: The species ID from AGR. + :return: The species ID in Geneweaver format. + """ # find the species name, return None if not found in the AGR-normalizer db species_name = db.query(Species.sp_name).filter(Species.sp_id == agr_sp_id).first() if species_name: diff --git a/src/geneweaver/aon/service/genes.py b/src/geneweaver/aon/service/genes.py index b708ef2..73a000f 100644 --- a/src/geneweaver/aon/service/genes.py +++ b/src/geneweaver/aon/service/genes.py @@ -1,4 +1,5 @@ -from typing import Optional +"""Module with database functions for genes.""" +from typing import List, Optional, Type from geneweaver.aon.models import Gene from geneweaver.aon.service.utils import apply_paging @@ -11,7 +12,16 @@ def get_genes( prefix: Optional[str] = None, start: Optional[int] = None, limit: Optional[int] = 1000, -): +) -> List[Type[Gene]]: + """Get all genes with optional filtering. + + :param db: The database session. + :param species_id: The species id to filter by. + :param prefix: The gene prefix to filter by. + :param start: The start index for paging. + :param limit: The limit for paging. + :return: All genes with optional filtering. + """ query = db.query(Gene) if species_id is not None: query = query.filter(Gene.sp_id == species_id) @@ -21,22 +31,51 @@ def get_genes( return query.all() -def gene_by_id(gene_id: int, db: Session): +def gene_by_id(db: Session, gene_id: int) -> Type[Gene]: + """Get a gene by id. + + :param db: The database session. + :param gene_id: The gene id to search for. + :return: The gene with the id. + """ return db.query(Gene).get(gene_id) -def gene_by_ref_id(ref_id: str, db: Session): +def gene_by_ref_id(db: Session, ref_id: str) -> List[Type[Gene]]: + """Get a gene by reference id. + + :param db: The database session. + :param ref_id: The reference id to search for. + :return: The gene with the reference id. + """ return db.query(Gene).filter(Gene.gn_ref_id == ref_id).all() -def genes_by_prefix(prefix: str, db: Session): +def genes_by_prefix(db: Session, prefix: str) -> List[Type[Gene]]: + """Get all genes by prefix. + + :param db: The database session. + :param prefix: The gene prefix to search for. + :return: All genes with the prefix. + """ return db.query(Gene).filter(Gene.gn_prefix == prefix).all() -def gene_prefixes(db: Session): +def gene_prefixes(db: Session) -> List[str]: + """Get all gene prefixes. + + :param db: The database session. + :return: All gene prefixes. + """ results = db.query(Gene).distinct(Gene.gn_prefix).all() return [r.gn_prefix for r in results] -def genes_by_species_id(db: Session, species_id): +def genes_by_species_id(db: Session, species_id: int) -> List[Type[Gene]]: + """Get all genes for a species. + + :param db: The database session. + :param species_id: The species id to search for. + :return: All genes for the species. + """ return db.query(Gene).filter(Gene.sp_id == species_id).all() diff --git a/src/geneweaver/aon/service/homologs.py b/src/geneweaver/aon/service/homologs.py index c7d03fc..d3f7bac 100644 --- a/src/geneweaver/aon/service/homologs.py +++ b/src/geneweaver/aon/service/homologs.py @@ -1,4 +1,5 @@ -from typing import Optional +"""Module with functions for getting homologs from the database.""" +from typing import List, Optional, Type from geneweaver.aon.models import Homology from geneweaver.aon.service.utils import apply_paging @@ -13,7 +14,18 @@ def get_homologs( gene_id: Optional[int] = None, start: Optional[int] = None, limit: Optional[int] = 1000, -): +) -> List[Type[Homology]]: + """Get homologs with optional filters. + + :param db: The database session. + :param homolog_id: The homolog ID. + :param source_name: The source name. + :param species_id: The species ID. + :param gene_id: The gene ID. + :param start: The start index for paging. + :param limit: The number of results to return. + :return: The homologs with optional filters. + """ base_query = db.query(Homology) if homolog_id is not None: base_query = base_query.filter(Homology.hom_id == homolog_id) @@ -28,6 +40,11 @@ def get_homologs( return base_query.all() -def homolog_sources(db: Session): +def homolog_sources(db: Session) -> List[str]: + """Get all homolog sources. + + :param db: The database session. + :return: All homolog sources. + """ results = db.query(Homology.hom_source_name).distinct().all() return [r.hom_source_name for r in results] diff --git a/src/geneweaver/aon/service/orthologs.py b/src/geneweaver/aon/service/orthologs.py index 0c8d104..da8f40d 100644 --- a/src/geneweaver/aon/service/orthologs.py +++ b/src/geneweaver/aon/service/orthologs.py @@ -1,4 +1,5 @@ -from typing import Optional, Type +"""Module with functions for querying orthologs from the database.""" +from typing import List, Optional, Type from geneweaver.aon.models import Algorithm, Gene, Ortholog, OrthologAlgorithms from geneweaver.aon.service.utils import apply_paging @@ -6,7 +7,11 @@ def get_ortholog_from_gene(db: Session, ortholog_id: int) -> Optional[Type[Gene]]: - """Get ortholog by id.""" + """Get ortholog by id. + + :param db: The database session. + :param ortholog_id: The ortholog id to query. + """ ortholog = get_ortholog(db, ortholog_id) if ortholog is None: return None @@ -14,7 +19,11 @@ def get_ortholog_from_gene(db: Session, ortholog_id: int) -> Optional[Type[Gene] def get_ortholog_to_gene(db: Session, ortholog_id: int) -> Optional[Type[Gene]]: - """Get ortholog by id.""" + """Get ortholog by id. + + :param db: The database session. + :param ortholog_id: The ortholog id to query. + """ ortholog = get_ortholog(db, ortholog_id) if ortholog is None: return None @@ -33,7 +42,22 @@ def get_orthologs( revised: Optional[bool] = None, start: Optional[int] = None, limit: Optional[int] = 1000, -): +) -> List[Type[Ortholog]]: + """Get orthologs with dynamic optional filters. + + :param db: The database session. + :param from_species: The species to get orthologs from. + :param to_species: The species to get orthologs to. + :param from_gene_id: The gene id to get orthologs from. + :param to_gene_id: The gene id to get orthologs to. + :param algorithm_id: The algorithm id to get orthologs from. + :param possible_match_algorithms: The number of possible match algorithms. + :param best: The best orthologs. + :param revised: The revised orthologs. + :param start: The start index for paging. + :param limit: The limit for paging. + :return: The orthologs for the provided query. + """ query = db.query(Ortholog) if algorithm_id is not None: @@ -44,15 +68,15 @@ def get_orthologs( ) if from_species is not None: - FromGene = aliased(Gene) - query = query.join(FromGene, Ortholog.from_gene == FromGene.gn_id).filter( - FromGene.sp_id == from_species + from_gene = aliased(Gene) + query = query.join(from_gene, Ortholog.from_gene == from_gene.gn_id).filter( + from_gene.sp_id == from_species ) if to_species is not None: - ToGene = aliased(Gene) - query = query.join(ToGene, Ortholog.to_gene == ToGene.gn_id).filter( - ToGene.sp_id == to_species + to_gene = aliased(Gene) + query = query.join(to_gene, Ortholog.to_gene == to_gene.gn_id).filter( + to_gene.sp_id == to_species ) if from_gene_id: @@ -77,5 +101,11 @@ def get_orthologs( return query.all() -def get_ortholog(db: Session, ortholog_id: int): +def get_ortholog(db: Session, ortholog_id: int) -> Type[Ortholog]: + """Get ortholog by id. + + :param db: The database session. + :param ortholog_id: The ortholog id to query. + :return: The ortholog for the provided id. + """ return db.query(Ortholog).get(ortholog_id) diff --git a/src/geneweaver/aon/service/species.py b/src/geneweaver/aon/service/species.py index ec28511..ca6e437 100644 --- a/src/geneweaver/aon/service/species.py +++ b/src/geneweaver/aon/service/species.py @@ -1,11 +1,12 @@ """Species database queries.""" -from typing import Optional + +from typing import List, Optional, Type from geneweaver.aon.models import GeneweaverSpecies, Species from sqlalchemy.orm import Session -def get_species(db: Session, name: Optional[str] = None): +def get_species(db: Session, name: Optional[str] = None) -> List[Type[Species]]: """Get species. :param db: The database session. @@ -19,7 +20,7 @@ def get_species(db: Session, name: Optional[str] = None): return base_query.all() -def species_by_id(db: Session, species_id: int): +def species_by_id(db: Session, species_id: int) -> Type[Species]: """Get species by id. :param db: The database session. @@ -29,7 +30,7 @@ def species_by_id(db: Session, species_id: int): return db.query(Species).get(species_id) -def species_by_taxon_id(db: Session, taxon_id: int): +def species_by_taxon_id(db: Session, taxon_id: int) -> List[Type[Species]]: """Get species by taxon id. :param db: The database session. @@ -39,7 +40,7 @@ def species_by_taxon_id(db: Session, taxon_id: int): return db.query(Species).filter(Species.sp_taxon_id == taxon_id).all() -def convert_species_agr_to_ode(db: Session, agr_sp_id: int): +def convert_species_agr_to_ode(db: Session, agr_sp_id: int) -> Optional[int]: """Convert the species id from AGR to Geneweaver. :param db: The database session. From 2a6446e581e9c7d0a68cda7d100c2811235f773a Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 09:23:39 -0500 Subject: [PATCH 10/21] Formatting fixes for the controller code --- src/geneweaver/aon/controller/__init__.py | 1 + src/geneweaver/aon/controller/algorithms.py | 6 ++++-- src/geneweaver/aon/controller/flask/__init__.py | 1 + src/geneweaver/aon/controller/flask/healthcheck.py | 1 + src/geneweaver/aon/controller/genes.py | 6 ++++-- src/geneweaver/aon/controller/homologs.py | 2 ++ src/geneweaver/aon/controller/orthologs.py | 10 +++++++--- src/geneweaver/aon/controller/species.py | 4 +++- src/geneweaver/aon/controller/versions.py | 14 +++++++++++++- 9 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/geneweaver/aon/controller/__init__.py b/src/geneweaver/aon/controller/__init__.py index e69de29..380239f 100644 --- a/src/geneweaver/aon/controller/__init__.py +++ b/src/geneweaver/aon/controller/__init__.py @@ -0,0 +1 @@ +"""Controller definitions for the AON API.""" diff --git a/src/geneweaver/aon/controller/algorithms.py b/src/geneweaver/aon/controller/algorithms.py index 4e3056f..5f9927d 100644 --- a/src/geneweaver/aon/controller/algorithms.py +++ b/src/geneweaver/aon/controller/algorithms.py @@ -1,3 +1,5 @@ +"""Controller definitions for the Algorithms API.""" + from typing import Optional from fastapi import APIRouter, Depends @@ -13,7 +15,7 @@ def get_algorithms( ): """Get all algorithms.""" if name is not None: - return algorithms_service.algorithm_by_name(name, db) + return algorithms_service.algorithm_by_name(db, name) return algorithms_service.all_algorithms(db) @@ -21,4 +23,4 @@ def get_algorithms( @router.get("/{algorithm_id}") def get_algorithm(algorithm_id: int, db: deps.Session = Depends(deps.session)): """Get algorithm by id.""" - return algorithms_service.algorithm_by_id(algorithm_id, db) + return algorithms_service.algorithm_by_id(db, algorithm_id) diff --git a/src/geneweaver/aon/controller/flask/__init__.py b/src/geneweaver/aon/controller/flask/__init__.py index e69de29..e500e6b 100644 --- a/src/geneweaver/aon/controller/flask/__init__.py +++ b/src/geneweaver/aon/controller/flask/__init__.py @@ -0,0 +1 @@ +"""Legacy Flask app for the project. This is no longer used.""" diff --git a/src/geneweaver/aon/controller/flask/healthcheck.py b/src/geneweaver/aon/controller/flask/healthcheck.py index 8f468af..697906b 100644 --- a/src/geneweaver/aon/controller/flask/healthcheck.py +++ b/src/geneweaver/aon/controller/flask/healthcheck.py @@ -15,4 +15,5 @@ class Healthcheck(Resource): @NS.doc(security=None) def get(self): + """Get the current application health.""" return {"status": "Available", "timestamp": datetime.now().isoformat()} diff --git a/src/geneweaver/aon/controller/genes.py b/src/geneweaver/aon/controller/genes.py index 774effe..bac73bb 100644 --- a/src/geneweaver/aon/controller/genes.py +++ b/src/geneweaver/aon/controller/genes.py @@ -1,3 +1,5 @@ +"""Controller definitions for the Genes API.""" + from typing import Optional from fastapi import APIRouter, Depends @@ -31,7 +33,7 @@ def get_gene_prefixes(db: deps.Session = Depends(deps.session)): @router.get("/{gene_id}") def get_gene(gene_id: int, db: deps.Session = Depends(deps.session)): """Get gene by id.""" - return genes_service.gene_by_id(gene_id, db) + return genes_service.gene_by_id(db, gene_id) @router.get("/by-ref-id/{ref_id}") @@ -43,4 +45,4 @@ def get_gene_by_ref_id( """Get gene by reference id.""" if ref_id_type == ReferenceGeneIDType.GW: ref_id = convert_service.ode_ref_to_agr(db, ref_id) - return genes_service.gene_by_ref_id(ref_id, db) + return genes_service.gene_by_ref_id(db, ref_id) diff --git a/src/geneweaver/aon/controller/homologs.py b/src/geneweaver/aon/controller/homologs.py index 8e974ca..8c8e7fa 100644 --- a/src/geneweaver/aon/controller/homologs.py +++ b/src/geneweaver/aon/controller/homologs.py @@ -1,3 +1,5 @@ +"""Controller definition for the homologs API.""" + from typing import Optional from fastapi import APIRouter, Depends, HTTPException diff --git a/src/geneweaver/aon/controller/orthologs.py b/src/geneweaver/aon/controller/orthologs.py index 983f9b3..6bd38fe 100644 --- a/src/geneweaver/aon/controller/orthologs.py +++ b/src/geneweaver/aon/controller/orthologs.py @@ -1,3 +1,5 @@ +"""API endpoints for orthologs.""" + from typing import Optional from fastapi import APIRouter, Depends, HTTPException @@ -19,7 +21,7 @@ def get_orthologs( paging_params: dict = Depends(deps.paging_parameters), db: deps.Session = Depends(deps.session), ): - """Get ortholog by id.""" + """Get orthologs with optional filtering.""" return orthologs_service.get_orthologs( db, from_species=from_species, @@ -46,7 +48,7 @@ def get_orthologs_by_id( @router.get("/{ortholog_id}/from_gene") -def get_ortholog_id_gene(ortholog_id: int, db: deps.Session = Depends(deps.session)): +def get_ortholog_id_to_gene(ortholog_id: int, db: deps.Session = Depends(deps.session)): """Get ortholog by id.""" orthologs = orthologs_service.get_ortholog_from_gene(db, ortholog_id) if not orthologs: @@ -55,7 +57,9 @@ def get_ortholog_id_gene(ortholog_id: int, db: deps.Session = Depends(deps.sessi @router.get("/{ortholog_id}/to_gene") -def get_ortholog_id_gene(ortholog_id: int, db: deps.Session = Depends(deps.session)): +def get_ortholog_id_from_gene( + ortholog_id: int, db: deps.Session = Depends(deps.session) +): """Get ortholog by id.""" orthologs = orthologs_service.get_ortholog_to_gene(db, ortholog_id) if not orthologs: diff --git a/src/geneweaver/aon/controller/species.py b/src/geneweaver/aon/controller/species.py index 0224955..667083e 100644 --- a/src/geneweaver/aon/controller/species.py +++ b/src/geneweaver/aon/controller/species.py @@ -1,3 +1,5 @@ +"""API Controller definition for species.""" + from typing import Optional from fastapi import APIRouter, Depends, HTTPException @@ -16,7 +18,7 @@ def get_species(name: Optional[str] = None, db: deps.Session = Depends(deps.sess @router.get("/{species_id}") -def get_species(species_id: int, db: deps.Session = Depends(deps.session)): +def get_species_by_id(species_id: int, db: deps.Session = Depends(deps.session)): """Get species by id.""" return species_service.species_by_id(db, species_id) diff --git a/src/geneweaver/aon/controller/versions.py b/src/geneweaver/aon/controller/versions.py index d87d3d9..c2d27c7 100644 --- a/src/geneweaver/aon/controller/versions.py +++ b/src/geneweaver/aon/controller/versions.py @@ -1,3 +1,5 @@ +"""API endpoints for the getting data load versions.""" + from fastapi import APIRouter, Depends, Request from geneweaver.aon import dependencies as deps from geneweaver.aon.models import Version @@ -9,7 +11,12 @@ def get_versions( db: deps.Session = Depends(deps.session), ): - return db.query(Version).filter(Version.load_complete == True).all() + """Get all versions. + + A version is a specific data load in the database. This endpoint returns + all versions in the database that are currently available. + """ + return db.query(Version).filter(Version.load_complete == True).all() # noqa: E712 @router.get("/default") @@ -17,4 +24,9 @@ def current_default_version( request: Request, db: deps.Session = Depends(deps.session), ): + """Get the default schema version ID. + + This endpoint returns the default schema version ID. This is the version + of the schema that the API will use if no version is specified. + """ return request.app.default_schema_version_id From 518f9e0f07c491fb3d1c9a822084c871ecbd8895 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 09:27:36 -0500 Subject: [PATCH 11/21] Fixing formatting the core module --- src/geneweaver/aon/core/__init__.py | 1 + src/geneweaver/aon/core/database.py | 1 + src/geneweaver/aon/core/exceptions.py | 5 ----- src/geneweaver/aon/core/schema_version.py | 20 ++++++++++++-------- 4 files changed, 14 insertions(+), 13 deletions(-) delete mode 100755 src/geneweaver/aon/core/exceptions.py diff --git a/src/geneweaver/aon/core/__init__.py b/src/geneweaver/aon/core/__init__.py index e69de29..684b406 100644 --- a/src/geneweaver/aon/core/__init__.py +++ b/src/geneweaver/aon/core/__init__.py @@ -0,0 +1 @@ +"""Core functionality for the Geneweaver AON application.""" diff --git a/src/geneweaver/aon/core/database.py b/src/geneweaver/aon/core/database.py index ccc40d0..e5d39ef 100755 --- a/src/geneweaver/aon/core/database.py +++ b/src/geneweaver/aon/core/database.py @@ -1,3 +1,4 @@ +"""Root database module, for uses other than the FastAPI application.""" from geneweaver.aon.core.config import config from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base diff --git a/src/geneweaver/aon/core/exceptions.py b/src/geneweaver/aon/core/exceptions.py deleted file mode 100755 index 529f07c..0000000 --- a/src/geneweaver/aon/core/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Exceptions created for the AON service.""" - - -class GwOrthoNormError(Exception): - pass diff --git a/src/geneweaver/aon/core/schema_version.py b/src/geneweaver/aon/core/schema_version.py index 809974e..a429a66 100644 --- a/src/geneweaver/aon/core/schema_version.py +++ b/src/geneweaver/aon/core/schema_version.py @@ -1,23 +1,24 @@ """Utilities for getting and setting up schema version db connections.""" import logging -from typing import Optional, Tuple +from typing import List, Optional, Tuple from geneweaver.aon.core.config import config from geneweaver.aon.core.database import BaseAGR, BaseGW from geneweaver.aon.models import Version -from sqlalchemy import create_engine, Engine +from sqlalchemy import Engine, create_engine from sqlalchemy.orm import Session, sessionmaker logger = logging.getLogger("uvicorn.error") -def get_latest_schema_version(): +def get_latest_schema_version() -> Optional[Version]: + """Get the latest schema version.""" engine = create_engine(config.DB.URI) session = Session(bind=engine) version = ( session.query(Version) - .filter(Version.load_complete == True) + .filter(Version.load_complete == True) # noqa: E712 .order_by(Version.id.desc()) .first() ) @@ -25,12 +26,13 @@ def get_latest_schema_version(): return version -def get_schema_versions(): +def get_schema_versions() -> List[Version]: + """Get all schema versions.""" engine = create_engine(config.DB.URI) session = Session(bind=engine) versions = ( session.query(Version) - .filter(Version.load_complete == True) + .filter(Version.load_complete == True) # noqa: E712 .order_by(Version.id.desc()) .all() ) @@ -38,7 +40,8 @@ def get_schema_versions(): return versions -def get_schema_version(version_id: int): +def get_schema_version(version_id: int) -> Optional[Version]: + """Get a schema version by ID.""" engine = create_engine(config.DB.URI) session = Session(bind=engine) version = session.query(Version).get(version_id) @@ -46,7 +49,8 @@ def get_schema_version(version_id: int): return version -def mark_schema_version_load_complete(version_id: int): +def mark_schema_version_load_complete(version_id: int) -> None: + """Mark a schema version as loaded.""" engine = create_engine(config.DB.URI) session = Session(bind=engine) version = session.query(Version).get(version_id) From d047deb0e6023bcbf6c174ea2ae499118265bd0c Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 10:24:45 -0500 Subject: [PATCH 12/21] More formatting fixes in dependencies, core, enum, models and services --- pyproject.toml | 1 + src/geneweaver/aon/core/database.py | 1 + src/geneweaver/aon/dependencies.py | 21 +++++++++++++-------- src/geneweaver/aon/enum.py | 1 + src/geneweaver/aon/models.py | 1 + src/geneweaver/aon/service/convert.py | 1 + src/geneweaver/aon/service/genes.py | 1 + src/geneweaver/aon/service/homologs.py | 1 + src/geneweaver/aon/service/orthologs.py | 1 + 9 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fe75192..57b2cf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ exclude = [ [tool.ruff.per-file-ignores] "tests/*" = ["ANN001", "ANN201"] "src/*" = ["ANN101"] +"src/geneweaver/aon/controller/*" = ["B008", "ANN201"] [build-system] requires = ["poetry-core"] diff --git a/src/geneweaver/aon/core/database.py b/src/geneweaver/aon/core/database.py index e5d39ef..94dfade 100755 --- a/src/geneweaver/aon/core/database.py +++ b/src/geneweaver/aon/core/database.py @@ -1,4 +1,5 @@ """Root database module, for uses other than the FastAPI application.""" + from geneweaver.aon.core.config import config from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base diff --git a/src/geneweaver/aon/dependencies.py b/src/geneweaver/aon/dependencies.py index 442ddf3..d04a3b6 100644 --- a/src/geneweaver/aon/dependencies.py +++ b/src/geneweaver/aon/dependencies.py @@ -13,12 +13,14 @@ set_up_sessionmanager, set_up_sessionmanager_by_schema, ) -from sqlalchemy.orm import sessionmaker, Session +from sqlalchemy.orm import Session, sessionmaker logger = logging.getLogger("uvicorn.error") DEFAULT_ALGORITHM_ID = config.DEFAULT_ALGORITHM_ID +Session = Session + @asynccontextmanager async def lifespan(app: FastAPI) -> None: @@ -58,6 +60,7 @@ async def lifespan(app: FastAPI) -> None: def version_id(version_id: int, request: Request) -> None: + """Set the schema version ID for a request.""" logger.info(f"Setting schema version to {version_id}.") request.state.schema_version_id = version_id @@ -67,23 +70,25 @@ def session(request: Request) -> sessionmaker: try: schema_version = request.state.schema_version_id try: - session = request.app.session_managers[schema_version]() - except KeyError: + _session = request.app.session_managers[schema_version]() + except KeyError as e: version = get_schema_version(schema_version) if version is not None and version.id not in request.app.session_managers: ( request.app.session_managers[version.id], request.app.engines[version.id], ) = set_up_sessionmanager(version) - session = request.app.session_managers[version.id]() + _session = request.app.session_managers[version.id]() else: - raise HTTPException(status_code=404, detail="Schema version not found.") + raise HTTPException( + status_code=404, detail="Schema version not found." + ) from e except AttributeError: - session = request.app.session() + _session = request.app.session() - yield session + yield _session - session.close() + _session.close() async def paging_parameters( diff --git a/src/geneweaver/aon/enum.py b/src/geneweaver/aon/enum.py index c6f7fb1..1ff3492 100644 --- a/src/geneweaver/aon/enum.py +++ b/src/geneweaver/aon/enum.py @@ -5,6 +5,7 @@ class ReferenceGeneIDType(Enum): """Enum for defining gene reference id types.""" + AON = "aon" GW = "gw" diff --git a/src/geneweaver/aon/models.py b/src/geneweaver/aon/models.py index 405e9a7..2223639 100755 --- a/src/geneweaver/aon/models.py +++ b/src/geneweaver/aon/models.py @@ -99,6 +99,7 @@ class Homology(BaseAGR): # The following models correspond to tables in the geneweaver database, # so they are created using BaseGW + class GeneweaverSpecies(BaseGW): """Geneweaver species table.""" diff --git a/src/geneweaver/aon/service/convert.py b/src/geneweaver/aon/service/convert.py index 9c6e592..ed2e36e 100644 --- a/src/geneweaver/aon/service/convert.py +++ b/src/geneweaver/aon/service/convert.py @@ -1,4 +1,5 @@ """Convert between Geneweaver and AON ID formats.""" + from typing import Optional from geneweaver.aon.models import ( diff --git a/src/geneweaver/aon/service/genes.py b/src/geneweaver/aon/service/genes.py index 73a000f..d5c0833 100644 --- a/src/geneweaver/aon/service/genes.py +++ b/src/geneweaver/aon/service/genes.py @@ -1,4 +1,5 @@ """Module with database functions for genes.""" + from typing import List, Optional, Type from geneweaver.aon.models import Gene diff --git a/src/geneweaver/aon/service/homologs.py b/src/geneweaver/aon/service/homologs.py index d3f7bac..7528ee6 100644 --- a/src/geneweaver/aon/service/homologs.py +++ b/src/geneweaver/aon/service/homologs.py @@ -1,4 +1,5 @@ """Module with functions for getting homologs from the database.""" + from typing import List, Optional, Type from geneweaver.aon.models import Homology diff --git a/src/geneweaver/aon/service/orthologs.py b/src/geneweaver/aon/service/orthologs.py index da8f40d..fe28943 100644 --- a/src/geneweaver/aon/service/orthologs.py +++ b/src/geneweaver/aon/service/orthologs.py @@ -1,4 +1,5 @@ """Module with functions for querying orthologs from the database.""" + from typing import List, Optional, Type from geneweaver.aon.models import Algorithm, Gene, Ortholog, OrthologAlgorithms From fdf55e9f052a1db6f932911c3cb43f19e9e6e879 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 12:58:57 -0500 Subject: [PATCH 13/21] Removing flask controller, updating geneweaver-testing, adding namespace docstring --- poetry.lock | 22 +- pyproject.toml | 5 +- .../aon/controller/flask/__init__.py | 1 - .../aon/controller/flask/controller.py | 1854 ----------------- .../aon/controller/flask/healthcheck.py | 19 - tests/__init__.py | 1 + 6 files changed, 13 insertions(+), 1889 deletions(-) delete mode 100644 src/geneweaver/aon/controller/flask/__init__.py delete mode 100755 src/geneweaver/aon/controller/flask/controller.py delete mode 100644 src/geneweaver/aon/controller/flask/healthcheck.py diff --git a/poetry.lock b/poetry.lock index f7834ef..9d9e9eb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -887,13 +887,13 @@ psycopg = {version = ">=3.1.13,<4.0.0", extras = ["binary"]} [[package]] name = "geneweaver-testing" -version = "0.1.0" +version = "0.1.1" description = "A library to standardize testing of GeneWeaver pacakges." optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "geneweaver_testing-0.1.0-py3-none-any.whl", hash = "sha256:c2082d342b82a5d65c6ec11364f7da2e596995c0fe4e8e8d1a0bfdff83e78251"}, - {file = "geneweaver_testing-0.1.0.tar.gz", hash = "sha256:1b62713b4c8c63e7a1a330f2719be803e8ed54cb113097cc90942bb66fd61b7c"}, + {file = "geneweaver_testing-0.1.1-py3-none-any.whl", hash = "sha256:01036bc3ab69fc98eca7ab8b3766290fc923b9a57e311c73e02d18d20dac5741"}, + {file = "geneweaver_testing-0.1.1.tar.gz", hash = "sha256:83d1e14cb53cf57c751c061887df8313129ca1d7400bf2075d94170a9098c230"}, ] [package.dependencies] @@ -990,13 +990,13 @@ files = [ [[package]] name = "httpcore" -version = "1.0.3" +version = "1.0.4" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-1.0.3-py3-none-any.whl", hash = "sha256:9a6a501c3099307d9fd76ac244e08503427679b1e81ceb1d922485e2f2462ad2"}, - {file = "httpcore-1.0.3.tar.gz", hash = "sha256:5c0f9546ad17dac4d0772b0808856eb616eb8b48ce94f49ed819fd6982a8a544"}, + {file = "httpcore-1.0.4-py3-none-any.whl", hash = "sha256:ac418c1db41bade2ad53ae2f3834a3a0f5ae76b56cf5aa497d2d033384fc7d73"}, + {file = "httpcore-1.0.4.tar.gz", hash = "sha256:cb2839ccfcba0d2d3c1131d3c3e26dfc327326fbe7a5dc0dbfe9f6c9151bb022"}, ] [package.dependencies] @@ -1007,7 +1007,7 @@ h11 = ">=0.13,<0.15" asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<0.24.0)"] +trio = ["trio (>=0.22.0,<0.25.0)"] [[package]] name = "httptools" @@ -1059,13 +1059,13 @@ test = ["Cython (>=0.29.24,<0.30.0)"] [[package]] name = "httpx" -version = "0.26.0" +version = "0.27.0" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, - {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, + {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"}, + {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"}, ] [package.dependencies] @@ -3866,4 +3866,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "60a8c0f81852e9088280552dfee0a147b1528aa7b04fe1512f00c79784e6d5b9" +content-hash = "763a993efdf2ffceae84c84fee39908634e72038e8cfda9c33908234b3eea54d" diff --git a/pyproject.toml b/pyproject.toml index 57b2cf8..aeb315f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ typer = {extras = ["all"], version = "^0.9.0"} temporalio = "^1.5.0" [tool.poetry.group.dev.dependencies] -geneweaver-testing = "^0.1.0" +geneweaver-testing = "^0.1.1" xmlrunner = "1.7.7" notebook = "^7.0.7" pytest = "^7.4.0" @@ -42,9 +42,6 @@ pytest-cov = "^4.1.0" [tool.ruff] select = ['F', 'E', 'W', 'A', 'C90', 'N', 'B', 'ANN', 'D', 'I', 'ERA', 'PD', 'NPY', 'PT'] -exclude = [ - 'src/geneweaver/aon/controller/flask/controller.py' - ] [tool.ruff.per-file-ignores] "tests/*" = ["ANN001", "ANN201"] diff --git a/src/geneweaver/aon/controller/flask/__init__.py b/src/geneweaver/aon/controller/flask/__init__.py deleted file mode 100644 index e500e6b..0000000 --- a/src/geneweaver/aon/controller/flask/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Legacy Flask app for the project. This is no longer used.""" diff --git a/src/geneweaver/aon/controller/flask/controller.py b/src/geneweaver/aon/controller/flask/controller.py deleted file mode 100755 index 0e047e8..0000000 --- a/src/geneweaver/aon/controller/flask/controller.py +++ /dev/null @@ -1,1854 +0,0 @@ -"""Definition of our API interface - Endpoints query the AGR database.""" - -from flask_restx import Namespace, Resource, abort, fields, reqparse -from geneweaver.aon.core.database import SessionLocal -from geneweaver.aon.models import ( - Algorithm, - Gene, - GeneweaverGene, - GeneweaverGeneDB, - GeneweaverSpecies, - Homology, - Ortholog, - OrthologAlgorithms, - Species, -) - -NS = Namespace("agr-service", description="Endpoints to query database") -db = SessionLocal() - -parser = reqparse.RequestParser() - -# MODELS - correspond with models in models.py file, allow for output in JSON format -algorithm_model = NS.model( - "algorithms", {"alg_id": fields.Integer(), "alg_name": fields.String()} -) - -ortholog_model = NS.model( - "orthologs", - { - "ort_id": fields.Integer(), - "from_gene": fields.Integer(), - "to_gene": fields.Integer(), - "ort_is_best": fields.Boolean(), - "ort_is_best_revised": fields.Boolean(), - "ort_is_best_adjusted": fields.Boolean(), - "ort_num_possible_match_algorithms": fields.Integer(), - }, -) - -gene_model = NS.model( - "genes", - { - "gn_id": fields.Integer(), - "gn_ref_id": fields.String(), - "gn_prefix": fields.String(), - "sp_id": fields.Integer(), - }, -) - -species_model = NS.model( - "species", - { - "sp_id": fields.Integer(), - "sp_name": fields.String(), - "sp_taxon_id": fields.Integer(), - }, -) - -ortholog_algorithms_model = NS.model( - "ortholog_algorithms", - { - "ora_id": fields.Integer(), - "alg_id": fields.Integer(), - "ort_id": fields.Integer(), - }, -) - -gw_gene_model = NS.model( - "geneweaver_genes", - { - "ode_gene_id": fields.Integer(), - "ode_ref_id": fields.String(), - "gdb_id": fields.Integer(), - "sp_id": fields.Integer(), - "ode_pref": fields.Boolean(), - "ode_date": fields.Date(), - "old_ode_gene_ids": fields.Integer(), - }, -) - -homology_model = NS.model( - "homologs", - { - "hom_id": fields.Integer(), - "gn_id": fields.Integer(), - "sp_id": fields.Integer(), - "hom_source_name": fields.String(), - }, -) - - -# CONVERTER FUNCTIONS - convert parameters to communicate between databases -def convertODEtoAGR(ode_ref, gdb_id): - # convert the ref_ids into how the agr ref ids are stored, same values but formatted - # slightly different in each database - """:description: converts into agr gene_id using the ode_ref_id and ode_gene_id - (both used as primary key in geneweaver.gene table) - :param ode_ref - ode_ref_id of gene - ode_gene_id - ode_gene_id of gene - :return: agr ref id (gn_ref_id from gn_gene table). - """ - ref = ode_ref - gdb_id = int(gdb_id) - # in agr database, each species only comes from one gdb_id, so these can be used - # to differentiate how the ref id should be modified - if gdb_id in [10, 11, 12, 13, 14, 15, 16]: - if gdb_id == 15: - prefix = "WB" - ref = prefix + ":" + ode_ref - if gdb_id == 14: - prefix = "FB" - ref = prefix + ":" + ode_ref - if gdb_id == 16: - prefix = "SGD" - ref = prefix + ":" + ode_ref - if gdb_id == 13: - prefix = "ZFIN" - ref = prefix + ":" + ode_ref - if gdb_id == 12: - ref = ode_ref[:3] + ":" + ode_ref[3:] - return ref - - -def convertAGRtoODE(gn_id): - """:description: converts an agr gene_id into the ode gene object - :param gn_id - integer gene id from gn_gene table in agr database - :return: ode_gene_id - integer gene id from gene table in geneweaver database. - """ - agr_gene = db.query(Gene).filter(Gene.gn_id == gn_id).first() - ref = agr_gene.gn_ref_id - prefix = agr_gene.gn_prefix - # convert the ref ids into the format they are stored in the geneweaver gene table - if prefix == "RGD": - ref = ref.replace(":", "") - elif prefix == "WB" or prefix == "FB" or prefix == "SGD" or prefix == "ZFIN": - ind = ref.find(":") + 1 - ref = ref[ind:] - - ode_gene_id = ( - db.query(GeneweaverGene).filter(GeneweaverGene.ode_ref_id == ref).first() - ).ode_gene_id - return ode_gene_id - - -######## -# alg_algorithm Table Endpoints -######## -@NS.route("/get_algorithm_by_name/") -class get_algorithm_by_name(Resource): - """:param alg_name: string of full species name, case sensitive - :return: alg_id and alg_name for selected algorithm - """ - - @NS.doc("returns algorithm object with specified name") - @NS.marshal_with(algorithm_model) - def get(self, alg_name): - result = db.query(Algorithm).filter(Algorithm.alg_name == alg_name).first() - return result - - -@NS.route("/all_algorithms") -class all_algorithms(Resource): - """:return: alg_id and alg_name for each algorithm""" - - @NS.doc("returns all algorithms") - @NS.marshal_with(algorithm_model) - def get(self): - return db.query(Algorithm).all() - - -######## -# ort_ortholog Table Endpoints -######## -@NS.route("/all_orthologs") -class all_orthologs(Resource): - """:return: all ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, - ort_is_best_adjusted, and ort_num_possible_match_algorithms) - """ - - @NS.doc("returns all orthologs") - @NS.marshal_with(ortholog_model) - def get(self): - return db.query(Ortholog).all() - - -@NS.route("/get_orthologs_by_from_gene//") -class get_orthologs_by_from_gene(Resource): - """:param ode_ref_id - ode_ref_id of from gene - ode_gene_id - ode_gene_id of from gene - :return: all ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for any ortholog with specified from_gene - """ - - @NS.doc("returns orthologs from a specified gene") - @NS.marshal_with(ortholog_model) - def get(self, ode_ref_id, ode_gene_id): - # find gene and search orthologs based on gene_id - gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == ode_gene_id, - GeneweaverGene.ode_ref_id == ode_ref_id, - ) - .first() - ).gdb_id - gn_ref_id = convertODEtoAGR(ode_ref_id, gdb_id) - gn_id = db.query(Gene.gn_id).filter(Gene.gn_ref_id == gn_ref_id).first() - result = db.query(Ortholog).filter(Ortholog.from_gene == gn_id).all() - if not result: - abort(404, message="Could not find any orthologs from the specified gene") - return result - - -@NS.route("/get_orthologs_by_to_gene//") -class get_orthologs_by_to_gene(Resource): - """:param ode_ref_id - ode_ref_id of to gene - ode_gene_id - ode_gene_id of to gene - :return: all ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for any ortholog with specified to_gene - """ - - @NS.doc("returns orthologs to a specified gene") - @NS.marshal_with(ortholog_model) - def get(self, ode_ref_id, ode_gene_id): - gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == ode_gene_id, - GeneweaverGene.ode_ref_id == ode_ref_id, - ) - .first() - ).gdb_id - gn_ref_id = convertODEtoAGR(ode_ref_id, gdb_id) - gn_id = (db.query(Gene).filter(Gene.gn_ref_id == gn_ref_id).first()).gn_id - result = db.query(Ortholog).filter(Ortholog.to_gene == gn_id).all() - if not result: - abort(404, message="Could not find any orthologs to the specified gene") - return result - - -@NS.route("/get_ortholog_by_id/") -class get_ortholog_by_id(Resource): - """:param ort_id - ort_id from ort_ortholog table - :return: all ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for any ortholog with specified ort_id - """ - - @NS.doc("returns orthologs with specified id") - @NS.marshal_with(ortholog_model) - def get(self, ort_id): - result = db.query(Ortholog).filter(Ortholog.ort_id == ort_id).all() - if not result: - abort(404, message="Could not find any orthologs with that ortholog id") - return result - - -@NS.route( - "/get_orthologs_by_to_and_from_gene////" -) -class get_orthologs_by_to_and_from_gene(Resource): - """:param from_ode_ref_id - ode_ref_id of from gene - from_ode_gene_id - ode_gene_id of from gene - to_ode_ref_id - ode_ref_id of to gene - to_ode_gene_id - ode_gene_id of to gene - :return: all ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for any ortholog with specified from_gene and to_gene - """ - - @NS.doc("returns all orthologs to and from the specified genes") - @NS.marshal_with(ortholog_model) - def get(self, from_ode_ref_id, from_ode_gene_id, to_ode_ref_id, to_ode_gene_id): - to_gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == to_ode_gene_id, - GeneweaverGene.ode_ref_id == to_ode_ref_id, - ) - .first() - ).gdb_id - from_gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == from_ode_gene_id, - GeneweaverGene.ode_ref_id == from_ode_ref_id, - ) - .first() - ).gdb_id - - # converting geneweaver refs to query gn_gene table - from_gn_ref = convertODEtoAGR(from_ode_ref_id, from_gdb_id) - to_gn_ref = convertODEtoAGR(to_ode_ref_id, to_gdb_id) - - to_agr_gn_id = ( - db.query(Gene).filter(Gene.gn_ref_id == to_gn_ref).first() - ).gn_id - from_agr_gn_id = ( - db.query(Gene).filter(Gene.gn_ref_id == from_gn_ref).first() - ).gn_id - - result = ( - db.query(Ortholog) - .filter( - Ortholog.from_gene == from_agr_gn_id, Ortholog.to_gene == to_agr_gn_id - ) - .all() - ) - if not result: - abort( - 404, message="Could not find any orthologs with that from and to gene" - ) - return result - - -@NS.route( - "/get_orthologs_by_from_gene_and_best///" -) -class get_orthologs_by_from_gene_and_best(Resource): - """:param from_ode_ref_id - ode_ref_id of from gene - from_ode_gene_id - ode_gene_id of from gene - best - boolean to query the ort_is_best column in ortholog table - :return: all ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for any ortholog from a specific gene and T or F for - the ort_is_best column - """ - - @NS.doc("returns all orthologs from specified gene and by the best variable") - @NS.marshal_with(ortholog_model) - def get(self, from_ode_ref_id, from_ode_gene_id, best): - # best variable is a string, must convert it to a bool to use in query - best = best.upper() - if best == "FALSE" or best == "F": - modified_best = False - else: - modified_best = True - gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == from_ode_gene_id, - GeneweaverGene.ode_ref_id == from_ode_ref_id, - ) - .first() - ).gdb_id - gn_ref = convertODEtoAGR(from_ode_ref_id, gdb_id) - agr_gn_id = (db.query(Gene).filter(Gene.gn_ref_id == gn_ref).first()).gn_id - result = ( - db.query(Ortholog) - .filter( - Ortholog.from_gene == agr_gn_id, Ortholog.ort_is_best == modified_best - ) - .all() - ) - if not result: - abort( - 404, - message="Could not find any orthologs with that from gene and ort_is_best value", - ) - return result - - -@NS.route( - "/get_orthologs_by_from_to_gene_and_best/////" -) -class get_orthologs_by_from_to_gene_and_best(Resource): - """:param from_ode_ref_id - ode_ref_id of from gene - from_ode_gene_id - ode_gene_id of from gene - to_ode_ref_id - ode_ref_id of to gene - to_ode_gene_id - ode_gene_id of to gene - best - boolean to query the ort_is_best column in ortholog table - :return: all ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for any ortholog with specified from_gene and to_gene - and T or F for the ort_is_best column - """ - - @NS.doc("returns all orthologs from and to specified gene and by the best variable") - @NS.marshal_with(ortholog_model) - def get( - self, from_ode_ref_id, from_ode_gene_id, to_ode_ref_id, to_ode_gene_id, best - ): - # best variable is a string, must convert it to a bool to use in query - best = best.upper() - if best == "FALSE" or best == "F": - modified_best = False - else: - modified_best = True - # find from and to gene objects using given information - to_gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == to_ode_gene_id, - GeneweaverGene.ode_ref_id == to_ode_ref_id, - ) - .first() - ).gdb_id - from_gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == from_ode_gene_id, - GeneweaverGene.ode_ref_id == from_ode_ref_id, - ) - .first() - ).gdb_id - from_gn_ref = convertODEtoAGR(from_ode_ref_id, from_gdb_id) - to_gn_ref = convertODEtoAGR(to_ode_ref_id, to_gdb_id) - - to_agr_gn_id = ( - db.query(Gene).filter(Gene.gn_ref_id == to_gn_ref).first() - ).gn_id - from_agr_gn_id = ( - db.query(Gene).filter(Gene.gn_ref_id == from_gn_ref).first() - ).gn_id - - result = ( - db.query(Ortholog) - .filter( - Ortholog.from_gene == from_agr_gn_id, - Ortholog.to_gene == to_agr_gn_id, - Ortholog.ort_is_best == modified_best, - ) - .all() - ) - if not result: - abort( - 404, - message="Could not find any orthologs with that " - "from_gene, to_gene and ort_is_best value", - ) - return result - - -@NS.route( - "/get_orthologs_by_from_to_gene_and_revised/////" -) -class get_orthologs_by_from_to_gene_and_revised(Resource): - """:param from_ode_ref_id - ode_ref_id of from gene - from_ode_gene_id - ode_gene_id of from gene - to_ode_ref_id - ode_ref_id of to gene - to_ode_gene_id - ode_gene_id of to gene - ort_best_revised - boolean to query the ort_is_best_revised column in ortholog table - :return: all ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for any ortholog with specified from_gene and to_gene - and T or F for the ort_is_best_revised column - """ - - @NS.doc( - "returns all orthologs from and to specified gene and by the ort_best_revised variable" - ) - @NS.marshal_with(ortholog_model) - def get( - self, - from_ode_ref_id, - from_ode_gene_id, - to_ode_ref_id, - to_ode_gene_id, - ort_best_revised, - ): - # convert string ort_best_revised into a bool to be used in later queries - ort_best_revised = ort_best_revised.upper() - if ort_best_revised == "FALSE" or ort_best_revised == "F": - inp = False - else: - inp = True - - to_gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == to_ode_gene_id, - GeneweaverGene.ode_ref_id == to_ode_ref_id, - ) - .first() - ).gdb_id - from_gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == from_ode_gene_id, - GeneweaverGene.ode_ref_id == from_ode_ref_id, - ) - .first() - ).gdb_id - from_gn_ref = convertODEtoAGR(from_ode_ref_id, from_gdb_id) - to_gn_ref = convertODEtoAGR(to_ode_ref_id, to_gdb_id) - - to_agr_gn_id = ( - db.query(Gene).filter(Gene.gn_ref_id == to_gn_ref).first() - ).gn_id - from_agr_gn_id = ( - db.query(Gene).filter(Gene.gn_ref_id == from_gn_ref).first() - ).gn_id - - result = ( - db.query(Ortholog) - .filter( - Ortholog.from_gene == from_agr_gn_id, - Ortholog.to_gene == to_agr_gn_id, - Ortholog.ort_is_best_revised == inp, - ) - .all() - ) - if not result: - abort( - 404, - message="Could not find any orthologs with that from_gene," - " to_gene and ort_is_best_revised value", - ) - return result - - -@NS.route("/get_from_gene_of_ortholog_by_id/") -class get_from_gene_of_ortholog_by_id(Resource): - """:param ort_id: id from ortholog table - :return: gene info (gn_id, gn_ref_id, gn_prefix, sp_id) of the from gene for that ortholog - """ - - @NS.doc("return from_gene object of a ortholog") - @NS.marshal_with(gene_model) - def get(self, ort_id): - ortholog = db.query(Ortholog).filter(Ortholog.ort_id == ort_id).first() - result = db.query(Gene).filter(Gene.gn_id == ortholog.from_gene).first() - if not ortholog: - abort(404, message="Could not find any orthologs with the given parameters") - return result - - -@NS.route("/get_to_gene_of_ortholog_by_id/") -class get_to_gene_of_ortholog_by_id(Resource): - """:param ort_id: id from ortholog table - :return: gene info (gn_id, gn_ref_id, gn_prefix, sp_id) of the to gene for that ortholog - """ - - @NS.doc("return to_gene object of a specific ortholog") - @NS.marshal_with(gene_model) - def get(self, ort_id): - ortho = db.query(Ortholog).filter(Ortholog.ort_id == ort_id).first() - result = db.query(Gene).filter(Gene.gn_id == ortho.to_gene).first() - if not ortho: - abort(404, message="Could not find any orthologs with the given parameters") - return result - - -######## -# gn_gene Table Endpoints -######## -@NS.route("/all_genes") -class all_genes(Resource): - """:return: all gene info (id, ref_id, gn_prefix, species)""" - - @NS.doc("return all genes") - @NS.marshal_with(gene_model) - def get(self): - return db.query(Gene).all() - - -@NS.route("/get_genes_by_prefix/") -class get_genes_by_prefix(Resource): - """:param: gn_prefix - :return: gene info (id, ref_id, gn_prefix, species) for genes with given prefix - """ - - @NS.doc("return all genes with specified prefix") - @NS.marshal_with(gene_model) - def get(self, gn_prefix): - gn_prefix = gn_prefix.upper() - result = db.query(Gene).filter(Gene.gn_prefix == gn_prefix).all() - if not result: - abort(404, message="Could not find any genes with that prefix") - return result - - -@NS.route("/get_genes_by_ode_gene_id//") -class get_genes_by_ode_gene_id(Resource): - """:param ode_ref_id - ode_ref_id of gene - ode_gene_id - ode_gene_id of gene - :return: gene info (gn_id, gn_ref_id, gn_prefix, sp_id) for agr gene, endpoint version of - convertODEtoAGR() - """ - - @NS.doc("return gene with specified ode_ref_id and ode_gene_id") - @NS.marshal_with(gene_model) - def get(self, ode_ref_id, ode_gene_id): - # find gene gdb_id to use the convertODEtoAGR function that converts the - # geneweaver ode_ref_id into the agr gn_ref_id - gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == ode_gene_id, - GeneweaverGene.ode_ref_id == ode_ref_id, - ) - .first() - ).gdb_id - gn_ref_id = convertODEtoAGR(ode_ref_id, gdb_id) - gene = db.query(Gene).filter(Gene.gn_ref_id == gn_ref_id).first() - if not gene: - abort(404, message="Could not find any matching genes") - return gene - - -@NS.route("/get_genes_by_species/") -class get_genes_by_species(Resource): - """:param: sp_name - string for species name, case sensitive - :return: info (gn_id, gn_ref_id, gn_prefix, sp_id) for genes of given species - """ - - @NS.doc("returns ode_gene_ids for genes of a certain species") - @NS.marshal_with(gene_model) - def get(self, sp_name): - species = db.query(Species).filter(Species.sp_name == sp_name).first() - genes = db.query(Gene).filter(Gene.sp_id == species.sp_id).all() - if not genes: - abort(404, message="Could not find any genes with that species") - return genes - - -@NS.route("/get_gene_species_name//") -class get_gene_species_name(Resource): - """:param ode_ref_id - ode_ref_id of gene - ode_gene_id - ode_gene_id of gene - :return: species name of gene - """ - - @NS.doc("returns the species of a specified gene") - def get(self, ode_ref_id, ode_gene_id): - gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == ode_gene_id, - GeneweaverGene.ode_ref_id == ode_ref_id, - ) - .first() - ).gdb_id - gn_ref_id = convertODEtoAGR(ode_ref_id, gdb_id) - agr_species = (db.query(Gene).filter(Gene.gn_ref_id == gn_ref_id).first()).sp_id - result = db.query(Species.sp_name).filter(Species.sp_id == agr_species).first() - if not result: - abort(404, message="Species not found for that gn_ref_id") - return result - - -######## -# sp_species Table Endpoints -######## -@NS.route("/all_species") -class all_species(Resource): - """:return: species info (id, name, sp_taxon_id) for all species""" - - @NS.doc("return all species") - @NS.marshal_with(species_model) - def get(self): - return db.query(Species).all() - - -@NS.route("/get_species_by_id/") -class get_species_by_id(Resource): - """:param: sp_id - :return: species info (sp_id, sp_name, sp_taxon_id) for species by id - """ - - @NS.doc("return species specified by id") - @NS.marshal_with(species_model) - def get(self, sp_id): - return db.query(Species).filter(Species.sp_id == sp_id).all() - - -@NS.route("/get_sp_id_by_hom_id/") -class get_sp_id_by_hom_id(Resource): - """:param: hom_id - :return: species id - """ - - @NS.doc("return species specified by hom id") - def get(self, hom_id): - sp_ids = [] - result = db.query(Homology).filter(Homology.hom_id == hom_id).all() - for r in result: - sp_ids.append(r.sp_id) - return sp_ids - - -@NS.route("/get_species_homologs_list", methods=["GET", "POST"]) -class get_species_homologs_list(Resource): - """:param: hom_id - list of hom_ids - :return: species id - """ - - @NS.expect(parser) - def get(self): - parser.add_argument("hom_ids", type=int, action="append") - data = parser.parse_args() - hom_ids = data["hom_ids"] - - result = db.query(Homology.sp_id).filter(Homology.hom_id.in_(hom_ids)).all() - homologous_species = list(set(list(zip(*result))[0])) - - return homologous_species - - -######## -# ora_ortholog_algorithms Table Endpoints -######## -@NS.route("/get_orthologs_by_num_algorithms/") -class get_orthologs_by_num_algorithms(Resource): - """:param: num - number of algorithms - :return: ortholog info ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for all orthologs with num_algorithms - """ - - @NS.doc("return all orthologs with specified ort_num_possible_match_algorithms") - @NS.marshal_with(ortholog_model) - def get(self, num): - result = ( - db.query(Ortholog) - .filter(Ortholog.ort_num_possible_match_algorithms == num) - .all() - ) - if not result: - abort( - 404, - message="Could not find any orthologs with that number " - "of possible match algorithms", - ) - return result - - -@NS.route("/get_ortholog_by_algorithm/") -class get_ortholog_by_algorithm(Resource): - """:param: alg_name - str algorithm by name - :return: ortholog info ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for all orthologs with that algorithm - """ - - @NS.doc("return all orthologs for an algorithm") - @NS.marshal_with(ortholog_algorithms_model) - def get(self, alg_name): - # get algorithm id from string of algorithm name - alg_id = ( - db.query(Algorithm.alg_id).filter(Algorithm.alg_name == alg_name).first() - ) - orthologs = ( - db.query(OrthologAlgorithms) - .filter(OrthologAlgorithms.alg_id == alg_id) - .all() - ) - return orthologs - - -######## -# hom_homology table endpoints -######## -@NS.route("/all_homology") -class all_homology(Resource): - """:param: None - :return: All rows in homology table - """ - - @NS.doc("returns all rows of homology table") - @NS.marshal_with(homology_model) - def get(self): - return db.query(Homology).all() - - -@NS.route("/get_homology_by_id/") -class get_homology_by_id(Resource): - """:param: hom_id - hom_id of set of desired homologs - Note: hom_id is not the primary key, any set of genes with the same - hom_id are homologs. - :return: All homology rows with given hom_id - """ - - @NS.doc("returns all homology rows with given hom_id") - @NS.marshal_with(homology_model) - def get(self, hom_id): - homologs = db.query(Homology).filter(Homology.hom_id == hom_id).all() - if not homologs: - abort(404, message="There are not homologs with given hom_id") - else: - return homologs - - -@NS.route("/get_homology_by_gene/") -class get_homology_by_gene(Resource): - """:param: gn_id - gene id from gn_gene table in agr database - :return: All rows in homology table that have a matching gn_id to - the given gn_id - """ - - @NS.doc("returns all homology rows with given gn_id") - @NS.marshal_with(homology_model) - def get(self, gn_id): - homologs = db.query(Homology).filter(Homology.gn_id == gn_id).all() - if not homologs: - abort(404, message="There are not homologs with that gn_id") - else: - return homologs - - -@NS.route("/get_homology_by_species/") -class get_homology_by_species(Resource): - """:param: sp_id - species id from sp_species table in agr database - :return: All rows in homology table that have a matching gn_id to - the given sp_id - """ - - @NS.doc("returns all homology rows with given sp_id") - @NS.marshal_with(homology_model) - def get(self, sp_id): - homologs = db.query(Homology).filter(Homology.sp_id == sp_id).all() - if not homologs: - abort(404, message="There are not homologs with that sp_id") - else: - return homologs - - -@NS.route("/get_homology_by_id_and_species//") -class get_homology_by_id_and_species(Resource): - """:param: hom_id - hom_id of set of desired homologs - sp_id - species id from sp_species table in agr database - :return: All rows in homology table that have a matching gn_id to - the given gn_id and a matching sp_id to the given sp_id - """ - - @NS.doc("returns all homology rows with given gn_id and sp_id") - @NS.marshal_with(homology_model) - def get(self, hom_id, sp_id): - homologs = ( - db.query(Homology) - .filter(Homology.hom_id == hom_id, Homology.sp_id == sp_id) - .all() - ) - if not homologs: - abort(404, message="There are not homologs with that hom_id and sp_id") - else: - return homologs - - -@NS.route("/get_homology_by_id_and_source//") -class get_homology_by_id_and_source(Resource): - """:param: hom_id - hom_id of set of desired homologs - hom_source_name - either 'AGR' or 'Homologene' to denote where the - homologous relationship came from - :return: All homology rows with given hom_id and hom_source_name - """ - - @NS.doc("returns all homology rows with given hom_id and hom_source_name") - @NS.marshal_with(homology_model) - def get(self, hom_id, hom_source_name): - homologs = ( - db.query(Homology) - .filter( - Homology.hom_id == hom_id, Homology.hom_source_name == hom_source_name - ) - .all() - ) - if not homologs: - abort( - 404, - message="There are not homologs with given hom_id and hom_source_name", - ) - else: - return homologs - - -@NS.route("/get_homology_by_gene_and_source//") -class get_homology_by_gene_and_source(Resource): - """:param: gn_id - gene id from gn_gene table in agr database - hom_source_name - either 'AGR' or 'Homologene' to denote where the - homologous relationship came from - :return: All homology rows with given gn_id and hom_source_name - """ - - @NS.doc("returns all homology rows with given gn_id and hom_source_name") - @NS.marshal_with(homology_model) - def get(self, gn_id, hom_source_name): - homologs = ( - db.query(Homology) - .filter( - Homology.gn_id == gn_id, Homology.hom_source_name == hom_source_name - ) - .all() - ) - if not homologs: - abort( - 404, - message="There are not homologs with given gn_id and hom_source_name", - ) - else: - return homologs - - -######## -# ort_ortholog and sp_species table endpoints -######## -@NS.route("/get_ortholog_by_from_species/") -class get_ortholog_by_from_species(Resource): - """:param: sp_name - str, case sensitive, from gene species - :return: ortholog info ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for all orthologs from given species - """ - - @NS.doc("return all orthologs from given species") - @NS.marshal_with(ortholog_model) - def get(self, sp_name): - # get species id from sp_name string - sp_id = db.query(Species.sp_id).filter(Species.sp_name == sp_name).first() - # find all genes with specified species and make a list of all the gene ids - genes = db.query(Gene).filter(Gene.sp_id == sp_id).all() - gene_ids = [] - for g in genes: - gene_ids.append(g.gn_id) - # query orthologs for any from genes in the list of gene_ids - from_orthos = db.query(Ortholog).filter(Ortholog.from_gene.in_(gene_ids)).all() - if not from_orthos: - abort(404, message="Could not find any matching orthologs") - return from_orthos - - -@NS.route("/get_ortholog_by_to_species/") -class get_ortholog_by_to_species(Resource): - """:param: sp_name - str, case sensitive, to gene species - :return: ortholog info ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for all orthologs to given species - """ - - @NS.doc("return all orthologs to a given species") - @NS.marshal_with(ortholog_model) - def get(self, sp_name): - # get sp_id from sp_name - sp_id = db.query(Species.sp_id).filter(Species.sp_name == sp_name).first() - genes = db.query(Gene).filter(Gene.sp_id == sp_id).all() - gene_ids = [] - for g in genes: - gene_ids.append(g.gn_id) - to_orthos = db.query(Ortholog).filter(Ortholog.to_gene.in_(gene_ids)).all() - if not to_orthos: - abort(404, message="Could not find any matching orthologs") - return to_orthos - - -@NS.route("/get_ortholog_by_to_and_from_species//") -class get_ortholog_by_to_and_from_species(Resource): - """:param: to_sp_name - str, case sensitive, to gene species name - from_sp_name - str, case sensitive, from gene species name - :return: ortholog info ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for all orthologs to and from the given species - """ - - @NS.doc("return all orthologs to and from given species") - @NS.marshal_with(ortholog_model) - def get(self, to_sp_name, from_sp_name): - # get both species ids - to_sp_id = db.query(Species.sp_id).filter(Species.sp_name == to_sp_name).first() - from_sp_id = ( - db.query(Species.sp_id).filter(Species.sp_name == from_sp_name).first() - ) - - # create a list of all gene_ids to and from the species - to_genes = db.query(Gene).filter(Gene.sp_id == to_sp_id).all() - from_genes = db.query(Gene).filter(Gene.sp_id == from_sp_id).all() - to_gn_ids = [] - from_gn_ids = [] - for g in to_genes: - to_gn_ids.append(g.gn_id) - for g in from_genes: - from_gn_ids.append(g.gn_id) - - orthos = ( - db.query(Ortholog) - .filter( - Ortholog.to_gene.in_(to_gn_ids), Ortholog.from_gene.in_(from_gn_ids) - ) - .all() - ) - if not orthos: - abort(404, message="Could not find any matching orthologs") - return orthos - - -@NS.route("/get_orthologous_species//") -class get_orthologous_species(Resource): - """:params: ode_gene_id - ode_gene_id to find orthologous species for - ode_ref_id - ode_ref_id to find orthologous species for - :return: list of ode species ids that given gene has orthologus genes to. - """ - - def get(self, ode_gene_id, ode_ref_id): - agr_ref = convert_ode_ref_to_agr(ode_ref_id) - agr_gene_id = db.query(Gene.gn_id).filter(Gene.gn_ref_id == agr_ref).first() - orthologous_from_genes = ( - db.query(Ortholog.from_gene).filter(Ortholog.to_gene == agr_gene_id).all() - ) - orthologous_to_genes = ( - db.query(Ortholog.to_gene).filter(Ortholog.from_gene == agr_gene_id).all() - ) - all_orthologs = orthologous_to_genes + orthologous_from_genes - all_orthologs = list(list(zip(*all_orthologs))[0]) - species = [] - for o in all_orthologs: - species.append( - convert_species_agr_to_ode( - db.query(Gene.sp_id).filter(Gene.gn_id == o).first()[0] - ) - ) - species = list(set(species)) - - return species - - -@NS.route( - "/get_ortholog_by_to_from_species_and_algorithm///" -) -class get_ortholog_by_to_from_species_and_algorithm(Resource): - """:param: to_sp_name - str, case sensitive, to gene species name - from_sp_name - str, case sensitive, from gene species name - alg_name - str, algoirthm name - :return: ortholog info ortholog info (ort_id, from_gene, to_gene, ort_is_best, ort_is_best_revised, ort_is_best_adjusted, - and ort_num_possible_match_algorithms) for all orthologs to and from the given species - and by algorithm - """ - - @NS.doc("return all orthologs to and from given species with specific algorithm") - @NS.marshal_with(ortholog_model) - def get(self, to_sp_name, from_sp_name, alg_name): - to_sp_id = db.query(Species.sp_id).filter(Species.sp_name == to_sp_name).first() - from_sp_id = ( - db.query(Species.sp_id).filter(Species.sp_name == from_sp_name).first() - ) - - to_genes = db.query(Gene.gn_id).filter(Gene.sp_id == to_sp_id).all() - from_genes = db.query(Gene.gn_id).filter(Gene.sp_id == from_sp_id).all() - - to_gn_ids = [] - from_gn_ids = [] - - for g in to_genes: - to_gn_ids.append(g) - for g in from_genes: - from_gn_ids.append(g) - - # get algorithm id from algorithm string - algo_id = ( - db.query(Algorithm.alg_id).filter(Algorithm.alg_name == alg_name).first() - ) - - orthos_algorithm = ( - db.query(OrthologAlgorithms) - .filter(OrthologAlgorithms.alg_id == algo_id) - .all() - ) - - # get list of ortholog ids using the algorithm - ort_ids = [] - for o in orthos_algorithm: - ort_ids.append(o.ort_id) - - # filter for Orthologs with from and to genes with given species and orthologs - # using specified algorithm - orthos = ( - db.query(Ortholog) - .filter( - Ortholog.to_gene.in_(to_gn_ids), - Ortholog.from_gene.in_(from_gn_ids), - Ortholog.ort_id.in_(ort_ids), - ) - .all() - ) - - if not orthos: - abort(404, message="Could not find any matching orthologs") - return orthos - - -################################################# -# AGR and Geneweaver Database Endpoints -# The following endpoints allow for connections to be made between the agr -# database and the geneweweaver database by converting between indentifiers -# for species and source databases -################################################# - - -@NS.route("/agr_to_geneweaver_species/") -class agr_to_geneweaver_species(Resource): - """:param: sp_id - agr species id - :return: geneweaver species id - """ - - @NS.doc( - "translate an AGR species id to the corresponding species id in the geneweaver database" - ) - def get(self, sp_id): - return convert_species_agr_to_ode(int(sp_id)) - - -# similar to the convertAGRtoODE function -@NS.route("/id_convert_agr_to_ode/") -class id_convert_agr_to_ode(Resource): - """:param: gn_id - gene id from gn_gene table in agr database - :return: ode_gene_id of corresponding gene in geneweaver database - """ - - @NS.doc("converts an agr gene id to the corresponding ode_gene_id") - def get(self, gn_id): - agr_gene = db.query(Gene).filter(Gene.gn_id == gn_id).first() - # edits the ref id to be in the format of the ode_ref_id, then search geneweaver.gene - ref = agr_gene.gn_ref_id - prefix = agr_gene.gn_prefix - # different formatting based on prefix - if prefix == "RGD": - ref = ref.replace(":", "") - elif prefix == "WB" or prefix == "FB" or prefix == "SGD" or prefix == "ZFIN": - ind = ref.find(":") + 1 - ref = ref[ind:] - # find matching ode_gene_id - ode_gene_id = ( - db.query(GeneweaverGene).filter(GeneweaverGene.ode_ref_id == ref).first() - ).ode_gene_id - if not ode_gene_id: - abort(404, message="matching ode_gene_id not found") - return ode_gene_id - - -@NS.route("/id_convert_ode_to_agr//") -class id_convert_ode_to_agr(Resource): - """:param: ode_ref_id - ode_ref_id of gene - ode_gene_id - ode_gene_id of gene - :return: agr gene id of corresponding gene - """ - - @NS.doc("converts an ode gene id to the corresponding agr gene id") - def get(self, ode_gene_id, ode_ref_id): - gdb_id = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == ode_gene_id, - GeneweaverGene.ode_ref_id == ode_ref_id, - ) - .first() - ).gdb_id - gn_ref_id = convertODEtoAGR(ode_ref_id, gdb_id) - gn_id = (db.query(Gene).filter(Gene.gn_ref_id == gn_ref_id).first()).gn_id - if not gn_id: - abort(404, message="No matching agr gene found") - return gn_id - - -@NS.route("/get_ode_gene_by_gdb_id/") -class get_ode_gene_by_gdb_id(Resource): - """:param: gdb_id - :return: gene info (ode_gene_id, ode_ref_id, gdb_id, sp_id, - ode_pref, ode_date, old_ode_gene_ids) of genes with - gdb_id - """ - - @NS.doc("return all ode_genes with the specified gdb_id") - @NS.marshal_with(gw_gene_model) - def get(self, gdb_id): - genes = db.query(GeneweaverGene).filter(GeneweaverGene.gdb_id == gdb_id).all() - if not genes: - abort(404, message="No genes were found with specified gdb_id") - return genes - - -@NS.route("/get_ode_gene_by_gene_id/") -class get_ode_gene_by_gene_id(Resource): - """:param: ode_gene_id - :return: gene info (ode_gene_id, ode_ref_id, gdb_id, sp_id, - ode_pref, ode_date, old_ode_gene_ids) of genes with - same ode_gene_id as given - """ - - @NS.doc("return all ode_genes with the same ode_gene_id") - @NS.marshal_with(gw_gene_model) - def get(self, ode_gene_id): - genes = ( - db.query(GeneweaverGene) - .filter(GeneweaverGene.ode_gene_id == ode_gene_id) - .all() - ) - if not genes: - abort(404, message="No genes were found matching that ode_gene_id") - return genes - - -@NS.route("/get_ode_gene_by_species//") -class get_ode_gene_by_species(Resource): - """:param: ode_gene_id - sp_name - case sensitive - :return: gene info (ode_gene_id, ode_ref_id, gdb_id, sp_id, - ode_pref, ode_date, old_ode_gene_ids) of genes with - same ode_gene_id as given and within same species - """ - - @NS.doc("return all genes with matching ode_gene_id and species") - @NS.marshal_with(gw_gene_model) - def get(self, ode_gene_id, sp_name): - sp_id = ( - db.query(GeneweaverSpecies) - .filter(GeneweaverSpecies.sp_name == sp_name) - .first() - ).sp_id - genes = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id == ode_gene_id, - GeneweaverGene.sp_id == sp_id, - ) - .all() - ) - if not genes: - abort( - 404, message="No genes were found matching that ode_gene_id and species" - ) - return genes - - -################################################# -# GW-AGR Integration Endpoints -################################################# - - -# converter functions using first char - these functions improve efficiency and -# are used here instead of convertODEtoAGR and convertAGRtoGW because they require -# gdb_id and are used more broadly. -def convert_ode_ref_to_agr(ode_ref): - # AGR only contains data from some gene data sources geneewaver has and the format is - # slightly different for the reference ids, so this fuction adds prefixes to the ids - # where necessary for the AGR format. - ref = ode_ref - gdb_id = ( - db.query(GeneweaverGene.gdb_id) - .filter(GeneweaverGene.ode_ref_id == ode_ref) - .first() - ) - if gdb_id: - gdb_id = gdb_id[0] - - if gdb_id == 15: - ref = "WB:" + ode_ref - elif gdb_id == 16: - ref = "SGD:" + ode_ref - elif gdb_id == 14: - ref = "FB:" + ode_ref - elif gdb_id == 13: - ref = "ZFIN:" + ode_ref - elif gdb_id == 12: - ref = ode_ref[:3] + ":" + ode_ref[3:] - - return ref - - -def convert_agr_ref_to_ode(gn_ref_id): - # All gene ref ids in AGR contain the gene source database prefix followed by a colon, - # but geneweaver only has some genes in this format. This function adjusts - # the ref ids to match the geneweaver format - ref = gn_ref_id - prefix = ref[0 : ref.find(":")] - - # genes with this prefix will have the prefix removed - gdb_to_remove_prefix = ["WB", "FB", "SGD", "ZFIN"] - - # RGD only requires removing the colon - if prefix == "RGD": - ref = ref.replace(":", "") - elif prefix in gdb_to_remove_prefix: - ind = ref.find(":") + 1 - ref = ref[ind:] - - # all other gene ref ids will be returned the same if not altered in the above steps - return ref - - -def convert_species_ode_to_agr(ode_sp_id): - # find the species name, return None if not found in the geneweaver db - species_name = ( - db.query(GeneweaverSpecies.sp_name) - .filter(GeneweaverSpecies.sp_id == ode_sp_id) - .first() - ) - if species_name: - species_name = species_name[0] - else: - return None - # get the AGR-normalizer species id from the species name - agr_sp_id = db.query(Species.sp_id).filter(Species.sp_name == species_name).first() - if agr_sp_id: - agr_sp_id = agr_sp_id[0] - return agr_sp_id - - -def convert_species_agr_to_ode(agr_sp_id): - # find the species name, return None if not found in the AGR-normalizer db - species_name = db.query(Species.sp_name).filter(Species.sp_id == agr_sp_id).first() - if species_name: - species_name = species_name[0] - else: - return None - # get the geneweaver species id - ode_sp_id = ( - db.query(GeneweaverSpecies.sp_id) - .filter(GeneweaverSpecies.sp_name == species_name) - .first() - ) - if ode_sp_id: - ode_sp_id = ode_sp_id[0] - return ode_sp_id - - -@NS.route("/get_ort_id_if_gene_is_ortholog//") -class get_ort_id_if_gene_is_ortholog(Resource): - """:param ode_ref_id - ode_ref_id of to gene - ode_gene_id - ode_gene_id of to gene - :return: list of ortholog ids that have specified gene as the from_gene - """ - - @NS.doc( - "returns the ortholog id filtering by the ortholog from gene, including the gdb_id" - ) - def get(self, ode_gene_id, ode_ref_id): - ref = convert_ode_ref_to_agr(ode_ref_id) - # find matching agr gene and filter the ortholog table with the agr gene id - gn_id = db.query(Gene.gn_id).filter(Gene.gn_ref_id == ref).first() - ort_id = ( - db.query(Ortholog.ort_id).filter( - (Ortholog.from_gene == gn_id) | (Ortholog.to_gene == gn_id) - ) - ).all() - return ort_id - - -@NS.route("/get_homology_by_ode_gene_id/") -class get_homology_by_ode_gene_id(Resource): - """:param ode_gene_id - ode_gene_id used to search for homology - :return: list of hom_ids that contain the gene given as input - """ - - def get(self, ode_gene_id): - # find the gn_ids for any gene with the given ode_gene_id - result = ( - db.query(GeneweaverGene.ode_ref_id) - .filter(GeneweaverGene.ode_gene_id == ode_gene_id) - .all() - ) - ode_refs = [] - for r in result: - ode_refs.append(convert_ode_ref_to_agr(r[0])) - gn_ids = db.query(Gene.gn_id).filter(Gene.gn_ref_id.in_(ode_refs)).all() - gn_ids = list(zip(*gn_ids))[0] - - hom_ids = [] - if len(gn_ids) != 0: - hom_ids = db.query(Homology.hom_id).filter(Homology.gn_id.in_(gn_ids)).all() - - # changes the output from the query to a list without repeats, rather than a list - # of lists - if len(hom_ids) != 0: - hom_ids = [l[0] for l in set(hom_ids)] - - return hom_ids - - -@NS.route("/get_homologous_ode_gene_ids_for_gene//") -class get_homologous_ode_gene_ids_for_gene(Resource): - """:param ode_ref_id - ode_ref_id used to search for homologous genes to this gene - gdb_name - name of gdb of the given ode_ref_id - :return: list of ode_gene_ids that are homologous to the given ode_ref_id - """ - - def get(self, ode_ref_id, gdb_name): - agr_ref_id = convert_ode_ref_to_agr(ode_ref_id) - ( - db.query(GeneweaverGeneDB.gdb_id) - .filter(GeneweaverGeneDB.gdb_name == gdb_name) - .first() - ) - - gn_id = db.query(Gene.gn_id).filter(Gene.gn_ref_id == agr_ref_id).all() - gn_id = [l[0] for l in set(gn_id)] - - hom_ids = db.query(Homology.hom_id).filter(Homology.gn_id.in_(gn_id)).all() - hom_ids = [l[0] for l in set(hom_ids)] - print(hom_ids) - - homologous_gn_ids = ( - db.query(Homology.gn_id).filter(Homology.hom_id.in_(hom_ids)).all() - ) - homologous_gn_ids = [l[0] for l in set(homologous_gn_ids)] - - homologous_agr_refs = ( - db.query(Gene.gn_ref_id).filter(Gene.gn_id.in_(homologous_gn_ids)).all() - ) - homologous_refs = [ - convert_agr_ref_to_ode(l[0]) for l in set(homologous_agr_refs) - ] - - ode_gene_ids = ( - db.query(GeneweaverGene.ode_gene_id) - .filter(GeneweaverGene.ode_ref_id.in_(homologous_refs)) - .all() - ) - ode_gene_ids = [l[0] for l in set(ode_gene_ids)] - - return ode_gene_ids - - -@NS.route("/get_homology_by_ode_gene_ids", methods=["GET", "POST"]) -class get_homology_by_ode_gene_ids(Resource): - """:param ode_gene_ids - list of ode_gene_id used to search for homology - :return: list of hom_ids that contain the gene given as input - """ - - @NS.expect(parser) - def get(self): - # gets list of genes for each geneset - parser.add_argument("ode_gene_ids", type=str, action="append") - data = parser.parse_args() - ode_gene_ids = data["ode_gene_ids"] - - # find the gn_ids for any gene with the given ode_gene_id - result = ( - db.query(GeneweaverGene.ode_ref_id) - .filter(GeneweaverGene.ode_gene_id.in_(ode_gene_ids)) - .all() - ) - ode_refs = [] - for r in result: - ode_refs.append(convert_ode_ref_to_agr(r[0])) - - gn_ids = db.query(Gene.gn_id).filter(Gene.gn_ref_id.in_(ode_refs)).all() - if not gn_ids: - return [] - gn_ids = list(set(list(zip(*gn_ids))[0])) - - hom_ids = [] - if len(gn_ids) != 0: - hom_ids = db.query(Homology.hom_id).filter(Homology.gn_id.in_(gn_ids)).all() - else: - return [] - - # changes the output from the query to a list without repeats, rather than a list - # of lists - if len(hom_ids) != 0: - hom_ids = [l[0] for l in set(hom_ids)] - - return hom_ids - - -@NS.route("/get_ode_genes_from_hom_id//") -class get_ode_genes_from_hom_id(Resource): - """:param hom_id - hom_id that is searched for genes with the gdb_id - target_gdb_id - gdb_id that is used to filter genes in the hom_id - :return: list of ode_ref_ids of the genes in the hom_id group that are have the given gdb_id - """ - - def get(self, hom_id, target_gdb_id): - # these gdb_ids have 0 for their sp_id, so we cannot find homologs for a specific species - if target_gdb_id in [1, 2, 3, 4, 5, 6, 7, 8, 17, 21]: - return [] - - # find the sp_id from the given gdb_id - target_sp_id = ( - db.query(GeneweaverGeneDB.sp_id) - .filter(GeneweaverGeneDB.gdb_id == target_gdb_id) - .first() - ) - target_sp_id = convert_species_ode_to_agr(target_sp_id[0]) - - homologous_gn_ids = ( - db.query(Homology.gn_id) - .filter(Homology.hom_id == hom_id, Homology.sp_id == target_sp_id) - .all() - ) - - if not homologous_gn_ids: - return [] - # create unique list of gn_ids - homologous_gn_ids = list(set(list(zip(*homologous_gn_ids))[0])) - - refs = [] - for id in homologous_gn_ids: - ref = convert_agr_ref_to_ode( - (db.query(Gene.gn_ref_id).filter(Gene.gn_id == id).first())[0] - ) - if not ref: - break - refs.append(ref) - - return refs - - -@NS.route("/get_ortholog_by_from_gene_and_gdb//") -class get_ortholog_by_from_gene_and_gdb(Resource): - """:param ode_gene_id - ode_gene_id of from gene - gdb_id - gdb_id of specified gene, this is the gdb_id we are filtering by - :return: to gene info (ode_ref_id) for any ortholog that has the given from - gene. the goal is to find info about the orthologous gene from the given gene. - """ - - @NS.doc( - "returns to gene ode_gene_id and ode_ref_id of any ortholog with the from gene matching" - "the given ode_gene_id and to gene matching the gdb_id" - ) - def get(self, from_ode_gene_id, gdb_id): - # any gene with a gdb_id that is not in the agr_compatible_gdb_ids will not be found in the agr - # database, so it is filtered out in the search. - agr_compatible_gdb_ids = [10, 11, 12, 13, 14, 15, 16] - genes = ( - db.query(GeneweaverGene.ode_ref_id) - .filter( - GeneweaverGene.ode_gene_id == from_ode_gene_id, - GeneweaverGene.gdb_id.in_(agr_compatible_gdb_ids), - ) - .all() - ) - if not genes: - abort(404, message="no genes found under that gene id") - - # the species from the geneweaver database is translated into the agr_species id so it can - # be used to filter by species (gdb_id) within the agr database. - gdb_id = int(gdb_id) - gdb_to_agr_species = {10: 1, 11: 7, 12: 2, 13: 6, 14: 5, 15: 11, 16: 3} - agr_species = gdb_to_agr_species[gdb_id] - - # ode_ref_ids are translated into the agr format found in the ref_id column of the - # agr gene table - from_gene_refs = [] - for g in genes: - ref = convert_ode_ref_to_agr(str(g[0])) - from_gene_refs.append(ref) - - # a list of agr gene ids that match the ref ids - from_gene_ids = ( - db.query(Gene.gn_id).filter(Gene.gn_ref_id.in_(from_gene_refs)).all() - ) - - # a list of to_gene ids where the from_gene value is in the from_genes list of ids - to_gene_ids = ( - db.query(Ortholog.to_gene) - .filter(Ortholog.from_gene.in_(from_gene_ids)) - .all() - ) - - # agr ref ids of to_genes - to_gene_refs = ( - db.query(Gene.gn_ref_id) - .filter(Gene.gn_id.in_(to_gene_ids), Gene.sp_id == agr_species) - .all() - ) - - # translate the agr format ref_ids back to ode format - to_gene_ode_refs = [] - for r in to_gene_refs: - to_gene_ode_refs.append(convert_agr_ref_to_ode(r)) - - return to_gene_ode_refs - - -@NS.route("/get_intersect_by_homology", methods=["GET", "POST"]) -class get_intersect_by_homology(Resource): - """:param: gs1 - taken from parser, list of gene info in geneset 1 - gs2 - taken from parser, list of gene info in geneset 2 - :return: gene info (gi_symbol, ode_gene_id, and ort_id) of - genes that intersect both genesets using the hom_homology table - """ - - @NS.expect(parser) - def get(self): - # gets list of genes for each geneset - parser.add_argument("gs1", type=str, action="append") - parser.add_argument("gs2", type=str, action="append") - data = parser.parse_args() - gs1 = data["gs1"] - gs2 = data["gs2"] - - # maps gn_id to ode_ref_id - gs1_gn_ids = {} - gs2_gn_ids = {} - for i in range(0, len(gs1)): - ref1 = convert_ode_ref_to_agr(gs1[i]) - gn_id1 = db.query(Gene.gn_id).filter(Gene.gn_ref_id == ref1).first() - # map the gn_id to the ode_ref_id that has not been converted to agr form - if gn_id1 is not None: - gs1_gn_ids[gn_id1[0]] = gs1[i] - - for i in range(0, len(gs2)): - ref2 = convert_ode_ref_to_agr(gs2[i]) - gn_id2 = db.query(Gene.gn_id).filter(Gene.gn_ref_id == ref2).first() - if gn_id2 is not None: - gs2_gn_ids[gn_id2[0]] = gs2[i] - - # get the list of gn_ids for each geneset - gn_ids1 = list(set(gs1_gn_ids.keys())) - gn_ids2 = list(set(gs2_gn_ids.keys())) - - # get both the hom_id and the gn_id for any homolog group each gn_id is in so we can map it to - # ode_gene_id and gene symbol - hom_and_gn_id1 = ( - db.query(Homology.hom_id, Homology.gn_id) - .filter(Homology.gn_id.in_(gn_ids1)) - .all() - ) - hom_and_gn_id2 = ( - db.query(Homology.hom_id, Homology.gn_id) - .filter(Homology.gn_id.in_(gn_ids2)) - .all() - ) - - # create unique lists of the hom_ids for each geneset - hom_ids1 = [l[0] for l in set(hom_and_gn_id1)] - hom_ids2 = [l[0] for l in set(hom_and_gn_id2)] - # find the hom_ids that are in both lists - common_hom_ids = list(set(hom_ids1) & set(hom_ids2)) - - hom_and_gn_id1 = dict(hom_and_gn_id1) - - result = [] - for h in common_hom_ids: - hom_id = h - gn_id = hom_and_gn_id1[h] - ode_ref_id = gs1_gn_ids[gn_id] - - ode_gene_id = ( - db.query(GeneweaverGene.ode_gene_id) - .filter(GeneweaverGene.ode_ref_id == ode_ref_id) - .first() - )[0] - gene_symbol = ( - db.query(GeneweaverGene.ode_ref_id) - .filter( - GeneweaverGene.ode_gene_id == ode_gene_id, - GeneweaverGene.gdb_id == 7, - ) - .first() - )[0] - gene_info = (gene_symbol, ode_gene_id, hom_id) - result.append(gene_info) - - return result - - -@NS.route("/transpose_genes_by_species", methods=["GET", "POST"]) -class transpose_genes_by_species(Resource): - """:params: genes - taken through parser, list of ode_ref_ids to be tranposed - species - newSpecies that genes will be transposed to through orthology - :return: list of ode_ref_ids of transposed genes, genes that are orthologs to the - original genes but are of the specified newSpecies - """ - - @NS.expect(parser) - def get(self): - parser.add_argument("genes", type=str, action="append") - parser.add_argument("species", type=int) - data = parser.parse_args() - # sp contains the new species that the genes will be transposed to - sp = data["species"] - - # checking if sp is in the available agr species - if sp not in [1, 2, 3, 4, 5, 8, 9]: - abort(404, message="No matching genes with that species") - else: - sp = convert_species_ode_to_agr(sp) - - # store all converted refs in a parallel list to genes - refs = [] - for g in data["genes"]: - agr_ref = convert_ode_ref_to_agr(g) - refs.append(agr_ref) - - # get the gn_ids because these are used to identify genes in the hom_homolg table - all_gn_ids = db.query(Gene.gn_id).filter(Gene.gn_ref_id.in_(refs)).all() - all_gn_ids = list(set(list(zip(*all_gn_ids))[0])) - - # get all hom_ids that contain the genes we have now - hom_ids = db.query(Homology.hom_id).filter(Homology.gn_id.in_(all_gn_ids)).all() - hom_ids = list(set(list(zip(*hom_ids))[0])) - - # any gene in each of these hom_ids is homologous to at least one of our original genes. now - # we can search through all the genes associated with these hom_ids for genes that are also - # the species we are looking for - homologous_new_species_gn_ids = ( - db.query(Homology.gn_id) - .filter(Homology.hom_id.in_(hom_ids), Homology.sp_id == sp) - .all() - ) - homologous_new_species_gn_ids = list( - set(list(zip(*homologous_new_species_gn_ids))[0]) - ) - - # convert gn_ids back to ode_ref_ids - homologous_new_species_agr_refs = ( - db.query(Gene.gn_ref_id) - .filter(Gene.gn_id.in_(homologous_new_species_gn_ids)) - .all() - ) - homologous_new_species_agr_refs = list( - set(list(zip(*homologous_new_species_agr_refs))[0]) - ) - - homologous_new_species_gw_refs = [] - for r in homologous_new_species_agr_refs: - homologous_new_species_gw_refs.append(convert_agr_ref_to_ode(r)) - - ode_gene_ids = ( - db.query(GeneweaverGene.ode_gene_id) - .filter(GeneweaverGene.ode_ref_id.in_(homologous_new_species_gw_refs)) - .all() - ) - gene_symbols = ( - db.query(GeneweaverGene.ode_ref_id) - .filter( - GeneweaverGene.ode_gene_id.in_(ode_gene_ids), - GeneweaverGene.gdb_id == 7, - GeneweaverGene.ode_pref is True, - ) - .all() - ) - gene_symbols = list(set(list(zip(*gene_symbols))[0])) - return gene_symbols - - -@NS.route("/if_gene_has_homolog/") -class if_gene_has_homolog(Resource): - """:params: ode_gene_id - :return: 1 if the gene has any homologous relationships, 0 if not - """ - - def get(self, ode_gene_id): - ref = ( - db.query(GeneweaverGene.ode_ref_id) - .filter(GeneweaverGene.ode_gene_id == ode_gene_id) - .all() - ) - result = 0 - - for r in ref: - agr_ref = convert_ode_ref_to_agr(r[0]) - gn_id = db.query(Gene.gn_id).filter(Gene.gn_ref_id == agr_ref).first() - if gn_id is not None: - homs = db.query(Homology).filter(Homology.gn_id == gn_id[0]).first() - if homs is not None: - result = 1 - break - - return result - - -@NS.route("/get_orthologs_by_symbol///") -class get_orthologs_by_symbol(Resource): - """:params: sym - list of gene symbols, in csv format (ex: Nptx2,Tnfrsf12a,Elk1) - orig_species - species that all the gene symbols come from - homologous species - species the user wants to map to - :return: data - dictionary with keys being the original symbols provide in the - sym list. the values are a list of genes that are the homologs of the key - gene and are the correct species. The genes are in the format [ode_ref_id, symbol]. - """ - - def get(self, sym, orig_species, homologous_species): - # create a list of provided symbols - symbols = sym.split(",") - - # get the sp_id from the species name - orig_sp_id = ( - db.query(GeneweaverSpecies.sp_id) - .filter(GeneweaverSpecies.sp_name == orig_species) - .first() - ) - # get the target homologous species sp_id - gdb_id = ( - db.query(GeneweaverGeneDB.gdb_id) - .filter(GeneweaverGeneDB.sp_id == orig_sp_id) - .first() - ) - homologous_sp_id = ( - db.query(Species.sp_id) - .filter(Species.sp_name == homologous_species) - .first() - ) - - data = {} - for s in symbols: - gene_id = ( - db.query(GeneweaverGene.ode_gene_id) - .filter( - GeneweaverGene.ode_ref_id == s, - GeneweaverGene.sp_id.in_(orig_sp_id), - ) - .first() - ) - - # if the symbol is in the Geneweaver database, find its ode_ref_id - if gene_id is None: - continue - else: - ref = ( - db.query(GeneweaverGene.ode_ref_id) - .filter( - GeneweaverGene.ode_gene_id == gene_id, - GeneweaverGene.gdb_id.in_(gdb_id), - ) - .first() - ) - if ref is None: - continue - - # get the gn_id from the ode_ref_id - ref = convert_ode_ref_to_agr(ref) - agr_id = db.query(Gene.gn_id).filter(Gene.gn_ref_id == ref).first() - - # move on to next symbol if the ode_ref_id is not in the AGR database - if agr_id is None: - continue - - # find all the gn_ids orthologous to the given genes using both directions of the pairwise - # relationships in the Ortholog table - orthos = ( - db.query(Ortholog.to_gene).filter(Ortholog.from_gene == agr_id).all() - ) - orthos.extend( - db.query(Ortholog.from_gene).filter(Ortholog.to_gene == agr_id).all() - ) - orthos = list(list(zip(*orthos))[0]) - - # get the ref ids from the gn_ids of the orthologous genes - agr_ortho_refs = ( - db.query(Gene.gn_ref_id) - .filter(Gene.gn_id.in_(orthos), Gene.sp_id == homologous_sp_id) - .all() - ) - # format the list - ortho_refs = [] - for o in agr_ortho_refs: - ortho_refs.append(convert_agr_ref_to_ode(o[0])) - - ortho_syms = [] - ortho_data = [] - for o in ortho_refs: - # get the ode_gene_id from the ode_ref_id - ortho_id = ( - db.query(GeneweaverGene.ode_gene_id) - .filter(GeneweaverGene.ode_ref_id == o) - .first() - ) - # convert the ode_gene_id to the gene symbol - ortho_sym = ( - db.query(GeneweaverGene.ode_ref_id) - .filter( - GeneweaverGene.ode_gene_id == ortho_id, - GeneweaverGene.gdb_id == 7, - GeneweaverGene.ode_pref is True, - ) - .first() - ) - if ortho_sym is None: - continue - ortho_syms.append(ortho_sym[0]) - ortho_data.append([o, ortho_sym[0]]) - data[s] = ortho_data - - return data diff --git a/src/geneweaver/aon/controller/flask/healthcheck.py b/src/geneweaver/aon/controller/flask/healthcheck.py deleted file mode 100644 index 697906b..0000000 --- a/src/geneweaver/aon/controller/flask/healthcheck.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Endpoints to check on the services health.""" - -from datetime import datetime - -from flask_restx import Namespace, Resource - -NS = Namespace( - "healthcheck", description="Check on the health of Geneweaver Ortholog Normalizer" -) - - -@NS.route("") -class Healthcheck(Resource): - """Checks whether the server is alive.""" - - @NS.doc(security=None) - def get(self): - """Get the current application health.""" - return {"status": "Available", "timestamp": datetime.now().isoformat()} diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..74ff692 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Root of the testing module for Geneweaver AON API.""" From efb1520b2b8e5f9c18e2a2f0cbc44a57149c67d9 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 13:14:23 -0500 Subject: [PATCH 14/21] Fixing everything except the load module --- .../versions/4297df5638d7_first_commit.py | 13 +++++-- .../aon/alembic/versions/666568a64356_.py | 14 ++++++-- .../aon/alembic/versions/__init__.py | 12 +++++++ src/geneweaver/aon/cli/load.py | 20 ++++++++--- src/geneweaver/aon/cli/main.py | 1 + src/geneweaver/aon/cli/temporal.py | 23 ++++++------ .../aon/load/geneweaver/homologs.py | 36 ++++++++++++------- 7 files changed, 85 insertions(+), 34 deletions(-) diff --git a/src/geneweaver/aon/alembic/versions/4297df5638d7_first_commit.py b/src/geneweaver/aon/alembic/versions/4297df5638d7_first_commit.py index c192cec..e0b80fa 100644 --- a/src/geneweaver/aon/alembic/versions/4297df5638d7_first_commit.py +++ b/src/geneweaver/aon/alembic/versions/4297df5638d7_first_commit.py @@ -16,7 +16,12 @@ depends_on = None -def _has_table(table_name): +def _has_table(table_name: str) -> bool: + """Check if table exists. + + :param table_name: The table name. + :return: True if table exists, False otherwise. + """ config = op.get_context().config engine = sa.engine_from_config( config.get_section(config.config_ini_section), prefix="sqlalchemy." @@ -26,7 +31,8 @@ def _has_table(table_name): return table_name in tables -def upgrade(): +def upgrade() -> None: + """Upgrade to v1, create the schema_version table.""" if not _has_table("schema_version"): op.create_table( "schema_version", @@ -46,5 +52,6 @@ def upgrade(): ) -def downgrade(): +def downgrade() -> None: + """Downgrade, remove the schema_version table.""" op.drop_table("schema_version", schema="versions") diff --git a/src/geneweaver/aon/alembic/versions/666568a64356_.py b/src/geneweaver/aon/alembic/versions/666568a64356_.py index 995ee0a..f7cecaa 100644 --- a/src/geneweaver/aon/alembic/versions/666568a64356_.py +++ b/src/geneweaver/aon/alembic/versions/666568a64356_.py @@ -16,7 +16,12 @@ depends_on = None -def upgrade(): +def upgrade() -> None: + """Create the AGR tables. + + When using this function, you should specify a schema with: + `-x tenant=$SCHEMA_NAME` + """ op.create_table( "alg_algorithm", sa.Column("alg_id", sa.Integer(), nullable=False), @@ -96,7 +101,12 @@ def upgrade(): ) -def downgrade(): +def downgrade() -> None: + """Remove the AGR tables. + + When using this function, you should specify a schema with: + `-x tenant=$SCHEMA_NAME` + """ op.drop_table("ora_ortholog_algorithms") op.drop_table("ort_ortholog") op.drop_table("gn_gene") diff --git a/src/geneweaver/aon/alembic/versions/__init__.py b/src/geneweaver/aon/alembic/versions/__init__.py index e69de29..5f42a91 100644 --- a/src/geneweaver/aon/alembic/versions/__init__.py +++ b/src/geneweaver/aon/alembic/versions/__init__.py @@ -0,0 +1,12 @@ +"""Alembic version definitions. + +We're using versions sligtly differently than the default Alembic versioning. + +The first version creates a table to be shared across schemas (data loads) +and subsequent versions create the per-data-load tables, and must be run individually +against a specified schema. + +This can be done by using the builtin gwaon commands, or by using the -x option, e.g + +`-x tenant=$SCHEMA_NAME` +""" diff --git a/src/geneweaver/aon/cli/load.py b/src/geneweaver/aon/cli/load.py index 30d4f37..b4a06ce 100644 --- a/src/geneweaver/aon/cli/load.py +++ b/src/geneweaver/aon/cli/load.py @@ -78,7 +78,7 @@ def agr_release_exists(release: str) -> bool: @cli.command() -def create_schema(release) -> Tuple[str, int]: +def create_schema(release: str) -> Tuple[str, int]: """Create the database schema.""" with Progress() as progress: db_creation_msg = "Creating database schema" @@ -118,6 +118,11 @@ def create_schema(release) -> Tuple[str, int]: def load_agr(orthology_file: str, schema_id: int) -> bool: + """Load the Alliance of Genome Resources data. + + :param orthology_file: The path to the orthology file. + :param schema_id: The schema id. + """ version = get_schema_version(schema_id) schema_name = version.schema_name session, _ = set_up_sessionmanager(version) @@ -157,7 +162,7 @@ def complete( release: Optional[str] = None, orthology_file: Optional[Path] = None, schema_id: Optional[int] = None, -): +) -> None: """Load the Alliance of Genome Resources data.""" if orthology_file is None: orthology_file, release = get_data(release) @@ -202,10 +207,10 @@ def gw(schema_id: int) -> bool: # Homologs with psycopg.connect( config.GW_DB.URI.replace("postgresql+psycopg", "postgresql") - ) as gw_connection, psycopg.connect( + ) as gw_conn, psycopg.connect( config.DB.URI.replace("postgresql+psycopg", "postgresql") - ) as aon_connection: - with gw_connection.cursor() as gw_cursor, aon_connection.cursor() as aon_cursor: + ) as aon_conn: + with gw_conn.cursor() as gw_cursor, aon_conn.cursor() as aon_cursor: homologs = geneweaver.homologs.get_homolog_information( aon_cursor, gw_cursor, aon_schema_name=schema_name ) @@ -221,6 +226,11 @@ def gw(schema_id: int) -> bool: @cli.command() def homology(schema_id: int) -> bool: + """Load homology data into the AON database. + + :param schema_id: The schema id. + :return: True if successful. + """ version = get_schema_version(schema_id) session, _ = set_up_sessionmanager(version) diff --git a/src/geneweaver/aon/cli/main.py b/src/geneweaver/aon/cli/main.py index b475f09..0fbaaa9 100644 --- a/src/geneweaver/aon/cli/main.py +++ b/src/geneweaver/aon/cli/main.py @@ -1,3 +1,4 @@ +"""GeneWeaver AON CLI client.""" import typer from geneweaver.aon import __version__ from geneweaver.aon.cli import load, setup, temporal diff --git a/src/geneweaver/aon/cli/temporal.py b/src/geneweaver/aon/cli/temporal.py index 84c180b..30cac2e 100644 --- a/src/geneweaver/aon/cli/temporal.py +++ b/src/geneweaver/aon/cli/temporal.py @@ -19,7 +19,7 @@ cli = typer.Typer(no_args_is_help=True, rich_markup_mode="rich") -async def _clear_schedules(): +async def _clear_schedules() -> None: """Clear all schedules.""" try: client = await Client.connect( @@ -36,12 +36,12 @@ async def _clear_schedules(): @cli.command() -def clear_schedules(): +def clear_schedules() -> None: """Clear all schedules.""" aiorun(_clear_schedules()) -async def _schedule_load(hour_frequency: int = 24): +async def _schedule_load(hour_frequency: int = 24) -> None: """Schedule a load job.""" await _clear_schedules() client = await Client.connect( @@ -63,12 +63,12 @@ async def _schedule_load(hour_frequency: int = 24): @cli.command() -def schedule_load(hour_frequency: int = 24): +def schedule_load(hour_frequency: int = 24) -> None: """Schedule a load job.""" aiorun(_schedule_load(hour_frequency)) -async def _start_load(): +async def _start_load() -> None: """Start a load job.""" client = await Client.connect( config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE @@ -81,12 +81,12 @@ async def _start_load(): @cli.command() -def start_load(): +def start_load() -> None: """Start a load job.""" aiorun(_start_load()) -async def _cancel_load(): +async def _cancel_load() -> None: """Cancel a load job.""" client = await Client.connect( config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE @@ -95,12 +95,12 @@ async def _cancel_load(): @cli.command() -def cancel_load(): +def cancel_load() -> None: """Cancel a load job.""" aiorun(_cancel_load()) -async def _terminate_load(): +async def _terminate_load() -> None: """Cancel a load job.""" client = await Client.connect( config.TEMPORAL_URI, namespace=config.TEMPORAL_NAMESPACE @@ -109,11 +109,12 @@ async def _terminate_load(): @cli.command() -def terminate_load(): +def terminate_load() -> None: """Cancel a load job.""" aiorun(_terminate_load()) @cli.command() -def start_worker(): +def start_worker() -> None: + """Start the worker.""" aiorun(worker.main()) diff --git a/src/geneweaver/aon/load/geneweaver/homologs.py b/src/geneweaver/aon/load/geneweaver/homologs.py index 443765d..4149bf9 100644 --- a/src/geneweaver/aon/load/geneweaver/homologs.py +++ b/src/geneweaver/aon/load/geneweaver/homologs.py @@ -1,17 +1,25 @@ """Code for adding homolog/ortholog information from the geneweaver database.""" import itertools -from typing import Optional +from typing import Optional, List -from geneweaver.aon.controller.flask.controller import convert_ode_ref_to_agr +from geneweaver.aon.service.convert import ode_ref_to_agr from geneweaver.aon.models import Gene, GeneweaverGene, Ortholog from psycopg import Cursor, sql +from psycopg.rows import Row from sqlalchemy.orm import Session def get_homolog_information( aon_cursor: Cursor, geneweaver_cursor: Cursor, aon_schema_name: Optional[str] = None -): +) -> List[Row]: + """Get homolog information from the geneweaver database. + + :param aon_cursor: The cursor for the aon database. + :param geneweaver_cursor: The cursor for the geneweaver database. + :param aon_schema_name: The name of the aon schema in the aon database. + :return: A list of rows containing homolog information. + """ # get all ode_gene_ids of the genes in the gn_gene table for the 3 missing species aon_schema_name = "public" if aon_schema_name is None else aon_schema_name aon_cursor.execute( @@ -34,15 +42,17 @@ def get_homolog_information( gw_genes = [str(i[0]) for i in ode_gene_ids] - # get hom_id, ode_gene_id, and sp_id from the geneweaver homology table for any homolog - # that is a member of a cluster that contains a gene in agr and of the 3 missing species, - # also orders by hom_id to make it easier to parse + # get hom_id, ode_gene_id, and sp_id from the geneweaver homology table for any + # homolog that is a member of a cluster that contains a gene in agr and of the + # 3 missing species, also orders by hom_id to make it easier to parse geneweaver_cursor.execute( """ - select hom_id, ode_gene_id, sp_id from extsrc.homology h1 where h1.hom_id in ( - select hom_id from extsrc.homology h2 where ode_gene_id = ANY(%(ode_ids)s)) - order by hom_id; - """, + SELECT hom_id, ode_gene_id, sp_id FROM extsrc.homology h1 + WHERE h1.hom_id IN ( + SELECT hom_id FROM extsrc.homology h2 + WHERE ode_gene_id = ANY(%(ode_ids)s)) + ORDER BY hom_id; + """, {"ode_ids": gw_genes}, ) homologs = geneweaver_cursor.fetchall() @@ -76,7 +86,7 @@ def add_missing_orthologs_2(db: Session, homologs): db.query(Gene) .filter( Gene.gn_ref_id.in_( - [convert_ode_ref_to_agr(g.ode_ref_id) for g in gw_gene_query] + [ode_ref_to_agr(db, g.ode_ref_id) for g in gw_gene_query] ) ) .all() @@ -114,7 +124,7 @@ def add_missing_orthologs(db: Session, homologs): .first() ) if gw_gene_ref: - agr_ref = convert_ode_ref_to_agr(gw_gene_ref.ode_ref_id) + agr_ref = ode_ref_to_agr(db, gw_gene_ref.ode_ref_id) gene_in_agr = ( db.query(Gene).filter(Gene.gn_ref_id == agr_ref).first() ) @@ -163,7 +173,7 @@ def add_missing_orthologs(db: Session, homologs): ) .first() ) - t_agr_ref = convert_ode_ref_to_agr(t_gene_ref.ode_ref_id) + t_agr_ref = ode_ref_to_agr(db, t_gene_ref.ode_ref_id) # check that to gene is in agr and geneweaver database if not t_gene_ref or not t_agr_ref: From 66e43e3789801ab13736e8f6d6fc05f2103538ff Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 13:22:12 -0500 Subject: [PATCH 15/21] Fixing load module formatting --- src/geneweaver/aon/load/agr/load.py | 8 ++-- src/geneweaver/aon/load/agr/sources.py | 2 +- src/geneweaver/aon/load/geneweaver/genes.py | 16 +++++-- .../aon/load/geneweaver/homologs.py | 47 +++++-------------- 4 files changed, 28 insertions(+), 45 deletions(-) diff --git a/src/geneweaver/aon/load/agr/load.py b/src/geneweaver/aon/load/agr/load.py index d9df2c0..63550f5 100644 --- a/src/geneweaver/aon/load/agr/load.py +++ b/src/geneweaver/aon/load/agr/load.py @@ -1,4 +1,5 @@ """Functions used to load the database.""" +# ruff: noqa: ANN001, ANN201 from itertools import chain, islice @@ -122,7 +123,8 @@ def init_species(db: Session, ortho_file: str, schema_name: str) -> None: max_id = max([int(species) for species in enum.Species]) db.execute( text( - f"ALTER SEQUENCE {schema_name}.sp_species_sp_id_seq RESTART WITH {max_id + 1}" + f"ALTER SEQUENCE {schema_name}.sp_species_sp_id_seq " + f"RESTART WITH {max_id + 1}" ) ) db.commit() @@ -277,7 +279,7 @@ def add_orthologs(db: Session, ortho_file, batch_size, batches_to_process=-1) -> heading_size = 15 with open(ortho_file, "r") as f: - for i in range(heading_size): + for _ in range(heading_size): f.readline() f.readline() @@ -301,7 +303,7 @@ def get_ortholog_batches(ortho_file, batch_size, batches_to_process=-1): heading_size = 15 with open(ortho_file, "r") as f: - for i in range(heading_size): + for _ in range(heading_size): f.readline() f.readline() diff --git a/src/geneweaver/aon/load/agr/sources.py b/src/geneweaver/aon/load/agr/sources.py index 87e125c..68ff83f 100644 --- a/src/geneweaver/aon/load/agr/sources.py +++ b/src/geneweaver/aon/load/agr/sources.py @@ -48,7 +48,7 @@ def download_agr_orthology_data( """Download the latest AGR orthology data. :param download_url: The AGR orthology download URL. - :param download_location: The location to download the AGR orthology data (optional). + :param download_location: The location to download the AGR orthology data. :return: The path to the downloaded AGR orthology data. """ filename = Path(download_url.split("/")[-1]) diff --git a/src/geneweaver/aon/load/geneweaver/genes.py b/src/geneweaver/aon/load/geneweaver/genes.py index a5e9428..b9d4306 100644 --- a/src/geneweaver/aon/load/geneweaver/genes.py +++ b/src/geneweaver/aon/load/geneweaver/genes.py @@ -25,8 +25,10 @@ } -def convert_gdb_to_prefix(gdb_id): - """:param: gdb_id - gdb_id from genedb table in geneweaver database, used as key for +def convert_gdb_to_prefix(gdb_id: int) -> str: + """Convert gdb_id to gn_prefix. + + :param: gdb_id - gdb_id from genedb table in geneweaver database, used as key for gn_prefix in agr gn_gene table because in agr, each prefix corresponds to one genedb :return: gn_prefix from gdb_dict corresponding to param gdb_id @@ -51,10 +53,15 @@ def get_species_to_taxon_id_map(db: Session) -> dict: return {s.name: s.sp_id for s in species} -def add_missing_genes(db: Session, geneweaver_cursor: Cursor): - """:description: adds genes from geneweaver gene table for the three missing species. +def add_missing_genes(db: Session, geneweaver_cursor: Cursor) -> None: + """Add genes from geneweaver that weren't loaded from AGR. + + Adds genes from geneweaver gene table for the three missing species. parses information from this table to create Gene objects to go into gn_gene table in agr. + + :param db: database session + :param geneweaver_cursor: cursor for geneweaver database """ # query for a list of geneweaver genes from Gallus gallus (sp_id=10, gdb_id=20), # Canis familiaris(sp_id=11, gdb_id=2), and Macaca mulatta (sp_id=6, gdb_id=1) @@ -71,7 +78,6 @@ def add_missing_genes(db: Session, geneweaver_cursor: Cursor): for g in gw_genes: gn_ref_id = g[0] gn_prefix = convert_gdb_to_prefix(g[1]) - # sp_id = convert_species_ode_to_agr(int(g[2])) sp_id = int(g[2]) gene = Gene(gn_ref_id=gn_ref_id, gn_prefix=gn_prefix, sp_id=sp_id) diff --git a/src/geneweaver/aon/load/geneweaver/homologs.py b/src/geneweaver/aon/load/geneweaver/homologs.py index 4149bf9..be3b072 100644 --- a/src/geneweaver/aon/load/geneweaver/homologs.py +++ b/src/geneweaver/aon/load/geneweaver/homologs.py @@ -1,10 +1,13 @@ """Code for adding homolog/ortholog information from the geneweaver database.""" +# ruff: noqa: C901 +# TODO: The above noqa is for the complexity of the `add_missing_orthologs` function, +# which should be refactored to be more readable and maintainable. import itertools -from typing import Optional, List +from typing import List, Optional -from geneweaver.aon.service.convert import ode_ref_to_agr from geneweaver.aon.models import Gene, GeneweaverGene, Ortholog +from geneweaver.aon.service.convert import ode_ref_to_agr from psycopg import Cursor, sql from psycopg.rows import Row from sqlalchemy.orm import Session @@ -47,9 +50,9 @@ def get_homolog_information( # 3 missing species, also orders by hom_id to make it easier to parse geneweaver_cursor.execute( """ - SELECT hom_id, ode_gene_id, sp_id FROM extsrc.homology h1 + SELECT hom_id, ode_gene_id, sp_id FROM extsrc.homology h1 WHERE h1.hom_id IN ( - SELECT hom_id FROM extsrc.homology h2 + SELECT hom_id FROM extsrc.homology h2 WHERE ode_gene_id = ANY(%(ode_ids)s)) ORDER BY hom_id; """, @@ -60,40 +63,12 @@ def get_homolog_information( return homologs -def add_missing_orthologs_2(db: Session, homologs): - """A refactor of the add_missing_orthologs function. +def add_missing_orthologs(db: Session, homologs: tuple) -> None: + """Add missing orthologs to the database. - This refactor stores more information in memory so that it doesn't need to make as - many database calls, which can be slow when using the cloud-sql-proxy. - - Homologs should be a list of tuples, where each tuple contains the hom_id, - ode_gene_id, and sp_id of a gene in the geneweaver database. + :param db: The database session. + :param homologs: The homologs to add to the database. """ - geneweaver_genes = {} - agr_genes = {} - - gw_gene_query = ( - db.query(GeneweaverGene) - .filter( - GeneweaverGene.ode_gene_id.in_([h[1] for h in homologs]), - GeneweaverGene.sp_id.in_([6, 10, 11]), - ) - .all() - ) - gw_genes = {g.ode_gene_id: g.sp_id for g in gw_gene_query} - - agr_gene_query = ( - db.query(Gene) - .filter( - Gene.gn_ref_id.in_( - [ode_ref_to_agr(db, g.ode_ref_id) for g in gw_gene_query] - ) - ) - .all() - ) - - -def add_missing_orthologs(db: Session, homologs): # starting hom_id is the first hom_id of the first homolog, # keeps track of current cluster curr_hom_id = homologs[0][0] From 2a4574ed14320a9b41cd21d76994fddccad63992 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 13:23:08 -0500 Subject: [PATCH 16/21] Fixing remaining black formatting changes --- src/geneweaver/aon/cli/main.py | 1 + src/geneweaver/aon/load/agr/load.py | 1 + src/geneweaver/aon/load/geneweaver/homologs.py | 1 + 3 files changed, 3 insertions(+) diff --git a/src/geneweaver/aon/cli/main.py b/src/geneweaver/aon/cli/main.py index 0fbaaa9..da24df0 100644 --- a/src/geneweaver/aon/cli/main.py +++ b/src/geneweaver/aon/cli/main.py @@ -1,4 +1,5 @@ """GeneWeaver AON CLI client.""" + import typer from geneweaver.aon import __version__ from geneweaver.aon.cli import load, setup, temporal diff --git a/src/geneweaver/aon/load/agr/load.py b/src/geneweaver/aon/load/agr/load.py index 63550f5..4d19c3d 100644 --- a/src/geneweaver/aon/load/agr/load.py +++ b/src/geneweaver/aon/load/agr/load.py @@ -1,4 +1,5 @@ """Functions used to load the database.""" + # ruff: noqa: ANN001, ANN201 from itertools import chain, islice diff --git a/src/geneweaver/aon/load/geneweaver/homologs.py b/src/geneweaver/aon/load/geneweaver/homologs.py index be3b072..3fc9756 100644 --- a/src/geneweaver/aon/load/geneweaver/homologs.py +++ b/src/geneweaver/aon/load/geneweaver/homologs.py @@ -1,4 +1,5 @@ """Code for adding homolog/ortholog information from the geneweaver database.""" + # ruff: noqa: C901 # TODO: The above noqa is for the complexity of the `add_missing_orthologs` function, # which should be refactored to be more readable and maintainable. From d7cb47f829a4b8ce3a92501137c61634df2f31ce Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 13:37:44 -0500 Subject: [PATCH 17/21] Adding contribution file --- CONTRIBUTING.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2c8b5a7 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,17 @@ +This package is developed and maintained by the Computational Sciences department at +the Jackson Laboratory. + +A full contribution guide is available at the: +[Geneweaver Documentation: Contributing Guide](https://thejacksonlaboratory.github.io/geneweaver-docs/reference/contributing-guide/) + +For more information, please visit +[our website](https://www.jax.org/research-and-faculty/resources/scientific-research-services/computational-sciences). + +For questions, comments, bug reports and security vulnerability reporting, please reach +out to us as cssc@jax.org. + +Other Geneweaver packages are community developed and can be found at on GitHub: +- [Geneweaver Testing](https://github.com/TheJacksonLaboratory/geneweaver-testing) +- [Geneweaver Tools](https://github.com/TheJacksonLaboratory/geneweaver-tools) +- [Geneweaver Core](https://github.com/TheJacksonLaboratory/geneweaver-core) +- [Geneweaver API](https://github.com/TheJacksonLaboratory/geneweaver-api) \ No newline at end of file From c5c0f42675bc6f5d6652f0861073f8f65ae31c2d Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 14:31:37 -0500 Subject: [PATCH 18/21] Adding compass metric reporting --- .github/workflows/_check-coverage-action.yml | 28 +++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_check-coverage-action.yml b/.github/workflows/_check-coverage-action.yml index 0990df5..0e4f958 100644 --- a/.github/workflows/_check-coverage-action.yml +++ b/.github/workflows/_check-coverage-action.yml @@ -99,4 +99,30 @@ jobs: ### Test Coverage Report ``` ${{ env.COVERAGE_REPORT }} - ``` \ No newline at end of file + ``` + - name: Upload coverage to Compass + run: | + METRIC_VALUE=$(cat coverage_report.txt | grep 'Total coverage:' | awk '{print $NF}' | sed 's/%//') + curl --request POST \ + --url https://jacksonlaboratory.atlassian.net/gateway/api/compass/v1/metrics \ + --user "${{ vars.ATLASSIAN_COMPASS_EMAIL }}:${{ secrets.ATLASSIAN_COMPASS_KEY }}" \ + --header "Accept: application/json" \ + --header "Content-Type: application/json" \ + --data "{ + \"metricSourceId\": \"ari:cloud:compass:44257862-6c68-4d47-8211-da38d2bb001b:metric-source/90bb0329-f6c5-429a-abbc-8d174535ad21/1c2a22de-708c-4a73-bebc-c84669a9d32b\", + \"value\": $METRIC_VALUE, + \"timestamp\": \"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\" + }" + - name: Upload complexity to Compass + run: | + METRIC_VALUE=$(poetry run radon cc src --total-average | grep 'Average complexity:' | awk '{print $NF}' | sed 's/[\(\)]//g') + curl --request POST \ + --url https://jacksonlaboratory.atlassian.net/gateway/api/compass/v1/metrics \ + --user "${{ vars.ATLASSIAN_COMPASS_EMAIL }}:${{ secrets.ATLASSIAN_COMPASS_KEY }}" \ + --header "Accept: application/json" \ + --header "Content-Type: application/json" \ + --data "{ + \"metricSourceId\": \"ari:cloud:compass:44257862-6c68-4d47-8211-da38d2bb001b:metric-source/90bb0329-f6c5-429a-abbc-8d174535ad21/6cc79ae7-47b1-474d-a1d0-4f78242fc89e\", + \"value\": $METRIC_VALUE, + \"timestamp\": \"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\" + }" \ No newline at end of file From 20a0c818be3c1564d12469924690315fb40770f0 Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 14:42:11 -0500 Subject: [PATCH 19/21] Moving reporting to main branch only --- .github/workflows/_check-coverage-action.yml | 58 +++++++++++--------- .github/workflows/coverage.yml | 2 + 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/.github/workflows/_check-coverage-action.yml b/.github/workflows/_check-coverage-action.yml index 0e4f958..a813d2e 100644 --- a/.github/workflows/_check-coverage-action.yml +++ b/.github/workflows/_check-coverage-action.yml @@ -26,6 +26,10 @@ on: description: "Show traceback for failed tests" type: string default: "no" + report-to-compass: + description: "Report coverage to Compass" + type: boolean + default: false jobs: check_coverage: runs-on: ${{ inputs.runner-os }} @@ -65,6 +69,34 @@ jobs: with: name: coverage-report-html path: htmlcov + - name: Upload coverage to Compass + if: ${{ inputs.report-to-compass }} + run: | + METRIC_VALUE=$(cat coverage_report.txt | grep 'Total coverage:' | awk '{print $NF}' | sed 's/%//') + curl --request POST \ + --url https://jacksonlaboratory.atlassian.net/gateway/api/compass/v1/metrics \ + --user "${{ vars.ATLASSIAN_COMPASS_EMAIL }}:${{ secrets.ATLASSIAN_COMPASS_KEY }}" \ + --header "Accept: application/json" \ + --header "Content-Type: application/json" \ + --data "{ + \"metricSourceId\": \"ari:cloud:compass:44257862-6c68-4d47-8211-da38d2bb001b:metric-source/90bb0329-f6c5-429a-abbc-8d174535ad21/1c2a22de-708c-4a73-bebc-c84669a9d32b\", + \"value\": $METRIC_VALUE, + \"timestamp\": \"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\" + }" + - name: Upload complexity to Compass + if: ${{ inputs.report-to-compass }} + run: | + METRIC_VALUE=$(poetry run radon cc src --total-average | grep 'Average complexity:' | awk '{print $NF}' | sed 's/[\(\)]//g') + curl --request POST \ + --url https://jacksonlaboratory.atlassian.net/gateway/api/compass/v1/metrics \ + --user "${{ vars.ATLASSIAN_COMPASS_EMAIL }}:${{ secrets.ATLASSIAN_COMPASS_KEY }}" \ + --header "Accept: application/json" \ + --header "Content-Type: application/json" \ + --data "{ + \"metricSourceId\": \"ari:cloud:compass:44257862-6c68-4d47-8211-da38d2bb001b:metric-source/90bb0329-f6c5-429a-abbc-8d174535ad21/6cc79ae7-47b1-474d-a1d0-4f78242fc89e\", + \"value\": $METRIC_VALUE, + \"timestamp\": \"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\" + }" comment-coverage-report: needs: [ check_coverage ] runs-on: ubuntu-latest @@ -100,29 +132,3 @@ jobs: ``` ${{ env.COVERAGE_REPORT }} ``` - - name: Upload coverage to Compass - run: | - METRIC_VALUE=$(cat coverage_report.txt | grep 'Total coverage:' | awk '{print $NF}' | sed 's/%//') - curl --request POST \ - --url https://jacksonlaboratory.atlassian.net/gateway/api/compass/v1/metrics \ - --user "${{ vars.ATLASSIAN_COMPASS_EMAIL }}:${{ secrets.ATLASSIAN_COMPASS_KEY }}" \ - --header "Accept: application/json" \ - --header "Content-Type: application/json" \ - --data "{ - \"metricSourceId\": \"ari:cloud:compass:44257862-6c68-4d47-8211-da38d2bb001b:metric-source/90bb0329-f6c5-429a-abbc-8d174535ad21/1c2a22de-708c-4a73-bebc-c84669a9d32b\", - \"value\": $METRIC_VALUE, - \"timestamp\": \"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\" - }" - - name: Upload complexity to Compass - run: | - METRIC_VALUE=$(poetry run radon cc src --total-average | grep 'Average complexity:' | awk '{print $NF}' | sed 's/[\(\)]//g') - curl --request POST \ - --url https://jacksonlaboratory.atlassian.net/gateway/api/compass/v1/metrics \ - --user "${{ vars.ATLASSIAN_COMPASS_EMAIL }}:${{ secrets.ATLASSIAN_COMPASS_KEY }}" \ - --header "Accept: application/json" \ - --header "Content-Type: application/json" \ - --data "{ - \"metricSourceId\": \"ari:cloud:compass:44257862-6c68-4d47-8211-da38d2bb001b:metric-source/90bb0329-f6c5-429a-abbc-8d174535ad21/6cc79ae7-47b1-474d-a1d0-4f78242fc89e\", - \"value\": $METRIC_VALUE, - \"timestamp\": \"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\" - }" \ No newline at end of file diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 70b8341..d6af374 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -6,8 +6,10 @@ on: jobs: check-coverage: uses: ./.github/workflows/_check-coverage-action.yml + secrets: inherit permissions: pull-requests: write with: required-coverage: ${{ vars.REQUIRED_COVERAGE }} coverage-module: "geneweaver.aon" + report-to-compass: true From f60171413e772d08a3921297854e5f2f8e5c628d Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 14:50:09 -0500 Subject: [PATCH 20/21] Adding version table bootstrap as init container. --- deploy/k8s/base/deployment.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/deploy/k8s/base/deployment.yaml b/deploy/k8s/base/deployment.yaml index 2113a3e..3d09c15 100644 --- a/deploy/k8s/base/deployment.yaml +++ b/deploy/k8s/base/deployment.yaml @@ -15,17 +15,17 @@ spec: app: geneweaver-aon-api spec: serviceAccountName: workload-identity-geneweaver -# initContainers: -# - name: geneweaver-aon-api-init -# image: geneweaver-aon-api -# imagePullPolicy: Always -# command: ["/bin/bash", "-c", "--" ] -# args: ["poetry run gwaon setup version-table && poetry run gwaon temporal start-load"] -# envFrom: -# - configMapRef: -# name: geneweaver-aon-config -# - secretRef: -# name: geneweaver-aon-db + initContainers: + - name: geneweaver-aon-api-init + image: geneweaver-aon-api + imagePullPolicy: Always + command: ["/bin/bash", "-c", "--" ] + args: ["poetry run gwaon setup version-table"] + envFrom: + - configMapRef: + name: geneweaver-aon-config + - secretRef: + name: geneweaver-aon-db containers: - name: geneweaver-aon-api image: geneweaver-aon-api From 567e830118dba903dc3e260bc15ab7b4b914834f Mon Sep 17 00:00:00 2001 From: Alexander Berger Date: Wed, 21 Feb 2024 15:31:02 -0500 Subject: [PATCH 21/21] Updating alembic.ini and env.py --- src/geneweaver/aon/alembic/alembic.ini | 85 ++++++++++++++++++++++ src/geneweaver/aon/alembic/env.py | 97 ++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 src/geneweaver/aon/alembic/alembic.ini create mode 100644 src/geneweaver/aon/alembic/env.py diff --git a/src/geneweaver/aon/alembic/alembic.ini b/src/geneweaver/aon/alembic/alembic.ini new file mode 100644 index 0000000..cc17373 --- /dev/null +++ b/src/geneweaver/aon/alembic/alembic.ini @@ -0,0 +1,85 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = . + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# TODO: Update me to point at the relevant db +sqlalchemy.url = postgresql+psycopg://127.0.0.1:5432/aon + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks=black +# black.type=console_scripts +# black.entrypoint=black +# black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S \ No newline at end of file diff --git a/src/geneweaver/aon/alembic/env.py b/src/geneweaver/aon/alembic/env.py new file mode 100644 index 0000000..ba14c56 --- /dev/null +++ b/src/geneweaver/aon/alembic/env.py @@ -0,0 +1,97 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config, text +from sqlalchemy import pool + +from alembic import context + +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = [Base.metadata] + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + _config_section = config.get_section(config.config_ini_section) + x_arguments = context.get_x_argument(as_dictionary=True) + + current_tenant = x_arguments.get("tenant") + dburi = x_arguments.get('dburi') + if dburi: + _config_section['sqlalchemy.url'] = dburi + + connectable = engine_from_config( + _config_section, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + connection.execute(text('CREATE SCHEMA IF NOT EXISTS "%s"' % 'versions')) + connection.execute(text('CREATE SCHEMA IF NOT EXISTS "%s"' % current_tenant)) + connection.execute(text('set search_path to "%s"' % current_tenant)) + connection.commit() + connection.dialect.default_schema_name = current_tenant + + context.configure( + connection=connection, + target_metadata=target_metadata, + # include_schemas=False, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online()