From 6146b550fd3147c838a398dd3598cd41d5dbb340 Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 22 Feb 2024 14:23:59 -0500 Subject: [PATCH] Set up image deployment for seqcolapi --- README_seqcolapi.md | 31 ++++++---- deployment/dockerhub/Dockerfile | 11 ++++ .../localhost/dev_local.env | 0 .../localhost/seqcolapi.yaml | 0 deployment/seqcolapi.databio.org/Dockerfile | 3 + .../primary_task_def.json | 0 .../seqcolapi.databio.org/production.env | 2 +- .../seqcolapi.databio.org/seqcolapi.yaml | 0 refget/seqcol.py | 21 ------- seqcolapi/__init__.py | 3 - seqcolapi/cli.py | 59 ------------------- seqcolapi/main.py | 17 +++--- .../requirements}/requirements-seqcolapi.txt | 2 +- servers/seqcolapi.databio.org/Dockerfile | 5 -- 14 files changed, 46 insertions(+), 108 deletions(-) create mode 100644 deployment/dockerhub/Dockerfile rename {servers => deployment}/localhost/dev_local.env (100%) rename {servers => deployment}/localhost/seqcolapi.yaml (100%) create mode 100644 deployment/seqcolapi.databio.org/Dockerfile rename {servers => deployment}/seqcolapi.databio.org/primary_task_def.json (100%) rename {servers => deployment}/seqcolapi.databio.org/production.env (81%) rename {servers => deployment}/seqcolapi.databio.org/seqcolapi.yaml (100%) delete mode 100644 seqcolapi/cli.py rename {requirements => seqcolapi/requirements}/requirements-seqcolapi.txt (88%) delete mode 100644 servers/seqcolapi.databio.org/Dockerfile diff --git a/README_seqcolapi.md b/README_seqcolapi.md index dc2ae8c..dc2178e 100644 --- a/README_seqcolapi.md +++ b/README_seqcolapi.md @@ -9,35 +9,44 @@ This repository contains: ### Run locally for development -To run a local server with a **local database**: -``` -source servers/localhost/dev_local.env -uvicorn seqcolapi.main:app --reload --port 8100 -``` +First, configure env vars: +- To run a local server with a **local database**:`source servers/localhost/dev_local.env` +- To run a local server with **the production database**:`source servers/seqcolapi.databio.org/production.env` + +Then, run service: -To run a local server with **the production database**: ``` -source servers/seqcolapi.databio.org/production.env uvicorn seqcolapi.main:app --reload --port 8100 ``` ### Running with docker -To build the docker file: +To build the docker file, from the root of this repository: +First you build the general-purpose image ``` -docker build --no-cache -t scim . +docker build -f deployment/dockerhub/Dockerfile -t databio/seqcolapi seqcolapi +``` + +Next you build the wrapped image (this just wraps the config into the app): + +``` +docker build -f deployment/seqcolapi.databio.org/Dockerfile -t seqcolapi.databio.org deployment/seqcolapi.databio.org ``` To run in a container: ``` -export POSTGRES_PASSWORD=`pass aws/rds_postgres` +source deployment/seqcolapi.databio.org/production.env +docker run --rm -p 8000:80 --name seqcolapi seqcolapi.databio.org +``` + + docker run --rm -p 8000:8000 --name sccon \ --env "POSTGRES_PASSWORD" \ --volume $CODE/seqcolapi.databio.org/config/seqcolapi.yaml:/config.yaml \ - scim seqcolapi serve -c /config.yaml -p 8000 + scim ``` To deploy container to dockerhub: diff --git a/deployment/dockerhub/Dockerfile b/deployment/dockerhub/Dockerfile new file mode 100644 index 0000000..6b6e8b4 --- /dev/null +++ b/deployment/dockerhub/Dockerfile @@ -0,0 +1,11 @@ +FROM tiangolo/uvicorn-gunicorn:python3.11-slim +LABEL authors="Nathan Sheffield" +RUN pip install https://github.com/databio/yacman/archive/dev.zip +RUN pip install https://github.com/refgenie/refget/archive/seqcolapi.zip + +COPY requirements/requirements-seqcolapi.txt requirements/requirements-seqcolapi.txt +RUN pip install -r requirements/requirements-seqcolapi.txt --no-cache-dir + +COPY . /app/seqcolapi + +CMD ["uvicorn", "seqcolapi.main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/servers/localhost/dev_local.env b/deployment/localhost/dev_local.env similarity index 100% rename from servers/localhost/dev_local.env rename to deployment/localhost/dev_local.env diff --git a/servers/localhost/seqcolapi.yaml b/deployment/localhost/seqcolapi.yaml similarity index 100% rename from servers/localhost/seqcolapi.yaml rename to deployment/localhost/seqcolapi.yaml diff --git a/deployment/seqcolapi.databio.org/Dockerfile b/deployment/seqcolapi.databio.org/Dockerfile new file mode 100644 index 0000000..7fcc959 --- /dev/null +++ b/deployment/seqcolapi.databio.org/Dockerfile @@ -0,0 +1,3 @@ +FROM databio/seqcolapi +COPY seqcolapi.yaml /seqcolapi_config.yaml +ENV SEQCOLAPI_CONFIG /seqcolapi_config.yaml diff --git a/servers/seqcolapi.databio.org/primary_task_def.json b/deployment/seqcolapi.databio.org/primary_task_def.json similarity index 100% rename from servers/seqcolapi.databio.org/primary_task_def.json rename to deployment/seqcolapi.databio.org/primary_task_def.json diff --git a/servers/seqcolapi.databio.org/production.env b/deployment/seqcolapi.databio.org/production.env similarity index 81% rename from servers/seqcolapi.databio.org/production.env rename to deployment/seqcolapi.databio.org/production.env index 7300d2f..5ddb740 100644 --- a/servers/seqcolapi.databio.org/production.env +++ b/deployment/seqcolapi.databio.org/production.env @@ -6,4 +6,4 @@ export POSTGRES_TABLE="seqcol" export SEQCOLAPI_PORT="5432" export SERVER_ENV="production" -export SEQCOLAPI_CONFIG="servers/seqcolapi.databio.org/seqcolapi.yaml" +export SEQCOLAPI_CONFIG="deployment/seqcolapi.databio.org/seqcolapi.yaml" diff --git a/servers/seqcolapi.databio.org/seqcolapi.yaml b/deployment/seqcolapi.databio.org/seqcolapi.yaml similarity index 100% rename from servers/seqcolapi.databio.org/seqcolapi.yaml rename to deployment/seqcolapi.databio.org/seqcolapi.yaml diff --git a/refget/seqcol.py b/refget/seqcol.py index ba6ff56..5b81f72 100644 --- a/refget/seqcol.py +++ b/refget/seqcol.py @@ -1,6 +1,5 @@ import henge import logging -import yacman from itertools import compress @@ -12,26 +11,6 @@ henge.ITEM_TYPE = "_item_type" -class SeqColConf(yacman.YAMLConfigManager): - """ - Simple configuration manager object for SeqColHenge. - """ - - def __init__( - self, - entries={}, - filepath=None, - yamldata=None, - writable=False, - wait_max=60, - skip_read_lock=False, - ): - filepath = yacman.select_config( - config_filepath=filepath, config_env_vars=["SEQCOLAPI_CONFIG"], config_name="seqcol" - ) - super(SeqColConf, self).__init__(entries, filepath, yamldata, writable) - - class SeqColHenge(henge.Henge): """ Extension of henge that accommodates collections of sequences. diff --git a/seqcolapi/__init__.py b/seqcolapi/__init__.py index c2a5a43..e69de29 100644 --- a/seqcolapi/__init__.py +++ b/seqcolapi/__init__.py @@ -1,3 +0,0 @@ -import logmuse - -logmuse.init_logger("seqcolapi") diff --git a/seqcolapi/cli.py b/seqcolapi/cli.py deleted file mode 100644 index f91d06c..0000000 --- a/seqcolapi/cli.py +++ /dev/null @@ -1,59 +0,0 @@ -from ubiquerg import VersionInHelpParser - -from .const import PKG_NAME, seqcolapi_version - - -def build_parser(): - """ - Build argument parser - - :return argparse.ArgumentParser - """ - banner = "%(prog)s - API for sequence collections" - additional_description = "For subcommand-specific options, type: '%(prog)s -h'" - additional_description += "\nhttps://github.com/regenie/seqcolapi" - - parser = VersionInHelpParser(prog=PKG_NAME, description=banner, epilog=additional_description) - - parser.add_argument( - "-V", - "--version", - action="version", - version="%(prog)s {v}".format(v=seqcolapi_version), - ) - - msg_by_cmd = {"serve": "run the server"} - - subparsers = parser.add_subparsers(dest="command") - - def add_subparser(cmd, description): - return subparsers.add_parser(cmd, description=description, help=description) - - sps = {} - # add arguments that are common for all subparsers - for cmd, desc in msg_by_cmd.items(): - sps[cmd] = add_subparser(cmd, desc) - sps[cmd].add_argument( - "-c", - "--config", - required=True, - dest="config", - help="Path to the seqcolapi config file (YAML).", - ) - sps[cmd].add_argument( - "-d", - "--dbg", - action="store_true", - dest="debug", - help="Set logger verbosity to debug", - ) - # add subparser-specific arguments - sps["serve"].add_argument( - "-p", - "--port", - dest="port", - type=int, - help="The port the webserver should be run on.", - default=None, - ) - return parser diff --git a/seqcolapi/main.py b/seqcolapi/main.py index e10ea69..18005e3 100644 --- a/seqcolapi/main.py +++ b/seqcolapi/main.py @@ -1,7 +1,6 @@ import henge import json import logging -import logmuse import os import sys import uvicorn @@ -17,12 +16,15 @@ from starlette.staticfiles import StaticFiles from typing import Union -from .cli import build_parser from .const import * from .scconf import RDBDict from .examples import * -from refget import SeqColConf, SeqColHenge, format_itemwise +from refget import SeqColHenge, format_itemwise +from yacman import select_config, FutureYAMLConfigManager as YAMLConfigManager + +class SeqColConf(YAMLConfigManager): + pass global _LOGGER @@ -124,7 +126,7 @@ def create_globals(scconf: yacman.YAMLConfigManager): def main(injected_args=None): # Entry point for running from console_scripts, installed package parser = build_parser() - parser = logmuse.add_logging_options(parser) + # parser = logmuse.add_logging_options(parser) args = parser.parse_args() if injected_args: args.__dict__.update(injected_args) @@ -133,10 +135,10 @@ def main(injected_args=None): print("No subcommand given") sys.exit(1) - _LOGGER = logmuse.logger_via_cli(args, make_root=True) + # _LOGGER = logmuse.logger_via_cli(args, make_root=True) _LOGGER.info(f"args: {args}") if "config" in args and args.config is not None: - scconf = SeqColConf(filepath=args.config) + scconf = SeqColConf.from_yaml_file(args.config) create_globals(scconf) app.state.schenge = schenge port = args.port or scconf.exp["server"]["port"] @@ -149,7 +151,8 @@ def main(injected_args=None): if __name__ != "__main__": # Entrypoint for running through uvicorn CLI (dev) if os.environ.get("SEQCOLAPI_CONFIG") is not None: - scconf = SeqColConf() + _LOGGER.info(f"Loading config from SEQCOLAPI_CONFIG: {os.environ.get('SEQCOLAPI_CONFIG')}") + scconf = SeqColConf.from_yaml_file(os.environ.get('SEQCOLAPI_CONFIG')) create_globals(scconf) app.state.schenge = schenge else: diff --git a/requirements/requirements-seqcolapi.txt b/seqcolapi/requirements/requirements-seqcolapi.txt similarity index 88% rename from requirements/requirements-seqcolapi.txt rename to seqcolapi/requirements/requirements-seqcolapi.txt index e658e82..d9997e6 100644 --- a/requirements/requirements-seqcolapi.txt +++ b/seqcolapi/requirements/requirements-seqcolapi.txt @@ -2,7 +2,7 @@ aiofiles fastapi henge>=0.2.1 jinja2 -logmuse>=0.2 +pipestat psycopg2-binary refget uvicorn>=0.7.1 diff --git a/servers/seqcolapi.databio.org/Dockerfile b/servers/seqcolapi.databio.org/Dockerfile deleted file mode 100644 index fdbeb88..0000000 --- a/servers/seqcolapi.databio.org/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM databio/seqcolapi - -COPY seqcolapi.yaml /seqcolapi_config.yaml - -ENTRYPOINT ["seqcolapi", "serve", "-c", "/seqcolapi_config.yaml"]