Merge pull request #8 from b97pla/update_deployment

switched to pyproject.toml, update rq and dependencies

.python-version

@@ -1 +1 @@
-3.6.12
+3.9.10

.travis.yml

@@ -1,16 +1,17 @@
 language: python
 
 python:
-        - "3.6.12"
+        - "3.9.10"
 
 before_install:
-        - sudo python -m pip install pipenv
+        - python -m venv --upgrade-deps .venv
 
 install:
-        - pipenv install --dev
+        - source .venv/bin/activate
+        - pip install -r requirements/dev .
 
 script:
-        - pipenv run nosetests tests/
+        - nosetests tests/
 
 notifications:
         email: false

README.md

@@ -1,65 +1,80 @@
 SNPSEQ Archive Verify
 ==================
 
-A self contained (aiohttp) REST service that helps verify uploaded SNP&SEQ archives by first downloading the archive from PDC, and then compare the MD5 sums for all associated files. 
+A self contained (aiohttp) REST service that helps verify uploaded SNP&SEQ archives by first downloading the archive 
+from PDC, and then compare the MD5 sums for all associated files. The downloaded files are deleted on successful 
+verification, and retained if any error occurs.
 
-The web service enqueues certain job functions in the RQ/Redis queue, where they get picked up by the separate RQ worker process. 
+The service is composed of 3 components which must all be running and the system must be set up to allow the services 
+to communicate over the configured protocols and ports (refer to the redis documentation for details):
 
-The downloaded files are deleted on successful verification, and retained if any error occurs.
+- archive-verify-ws REST service
+- Redis queue server
+- RQ worker
+
+The web service enqueues certain job functions in the RQ/Redis queue, where they get picked up by the separate RQ 
+worker process.  
 
 Pre-requisites
 --------------
-You will need python 3.6 and redis.
+You will need python >=3.9 and redis.
 
-Install redis with:
+[Download and install redis](https://redis.io/docs/getting-started/installation/install-redis-on-linux/)
 
-    apt-get install redis-server
 
-And start it with:
+Install
+-------
+It is recommended to set up the service in a virtual environment. [venv](https://docs.python.org/3/library/venv.html) 
+is used below with bash on a Linux system. 
 
-    redis-server
+    python3 -m venv --upgrade-deps .venv
+    source .venv/bin/activate
+    pip install .
 
+Running the service
+-------------------
 
-Trying it out
--------------
+Start the Redis server and RQ worker:
 
-    python3 -m pip install pipenv
-    pipenv install --deploy
+    redis-server
+    rq worker
 
-Try running it:
+Start the REST service
 
-     pipenv run ./archive-verify-ws -c=config/
-     pipenv run rq worker
+    archive-verify-ws -c=config/
 
 
 Mock Downloading
 ----------------
 
-If you are running this service locally and don't have IBM's dsmc client installed, you can skip the downloading step and verify an archive that is already on your machine.
+If you are running this service locally and don't have IBM's dsmc client installed, you can skip the downloading step 
+and verify an archive that is already on your machine.
 
 To use this method:
-- copy* an archive that has been pre-downloaded from PDC into the verify_root_dir set in app.yaml
+- copy an archive that has been pre-downloaded from PDC into the verify_root_dir set in app.yaml
 - Delete or edit some files from the archive if you wish to trigger a validation error.
 - in app.yml, set:
 
-
     pdc_client: "MockPdcClient"
 
-\*Note that the archive will be deleted from verify_root_dir on successful verification
+*Note that the archive will be deleted from verify_root_dir on successful verification*
 
 #### Naming Conventions for Mock Download ####
-Note that when an archive is downloaded from PDC using snpseq-archive-verify, the downloaded directory is formatted with the name of the archive plus the RQ job id, like so:
+Note that when an archive is downloaded from PDC using snpseq-archive-verify, the downloaded directory is formatted 
+with the name of the archive plus the RQ job id, like so:
 
     {verify_root_dir}/{archive_name}_{rq_job_id}
 
-When mocking downloading, we search verify_root_dir for archive_name and use the first directory found, ignoring the rq_job_id.
+When mocking downloading, we search verify_root_dir for archive_name and use the first directory found, ignoring the 
+rq_job_id.
 
 
 Running tests
 -------------
 
-    pipenv install --dev
-    pipenv run nosetests tests/
+    source .venv/bin/activate
+    pip install -e .[test]
+    nosetests tests/
 
 REST endpoints
 --------------
@@ -68,8 +83,10 @@ Enqueue a verification job of a specific archive:
 
     curl -i -X "POST" -d '{"host": "my-host", "description": "my-descr", "archive": "my_001XBC_archive"}' http://localhost:8989/api/1.0/verify
 
-Check the current status of an enqueued job: 
+Enqueue a download job of a specific archive:
 
-    curl -i -X "GET" http://localhost:8989/api/1.0/status/<job-uuid-returned-from-verify-endpoint>
+    curl -i -X "POST" -d '{"host": "my-host", "description": "my-descr", "archive": "my_001XBC_archive"}' http://localhost:8989/api/1.0/download
 
+Check the current status of an enqueued job: 
 
+    curl -i -X "GET" http://localhost:8989/api/1.0/status/<job-uuid-returned-from-verify-endpoint>

archive_verify/__init__.py

@@ -1 +0,0 @@
-__version__ = "1.0.5"

archive_verify/app.py

@@ -11,10 +11,12 @@
 
 log = logging.getLogger(__name__)
 
+
 def setup_routes(app):
     app.router.add_post(app["config"]["base_url"] + "/verify", handlers.verify)
     app.router.add_get(app["config"]["base_url"] + "/status/{job_id}", handlers.status)
 
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("-c", "--configroot", help="Path to config root dir", type=str, default="/etc/arteria/archive-verify/")
@@ -27,25 +29,28 @@ def parse_args():
 
     return args
 
+
 def load_config(args):
     config_file = os.path.join(args.configroot, "app.yaml")
     logger_file = os.path.join(args.configroot, "logger.yaml")
 
     try:
         with open(logger_file) as logger:
-            logger_conf = yaml.load(logger)
+            logger_conf = yaml.safe_load(logger)
             logging.config.dictConfig(logger_conf)
 
         with open(config_file) as config:
-            return yaml.load(config)
+            return yaml.safe_load(config)
     except Exception as e:
         log.error("Could not parse config file {}".format(e))
         sys.exit(1)
 
+
 def init_config():
     args = parse_args()
     return load_config(args)
 
+
 def start():
     conf = init_config()
     log.info("Starting archive-verify-ws on {}...".format(conf["port"]))

pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+packages = ["archive_verify"]
+
+[project]
+name = "archive-verify"
+version = "1.1.0"
+authors = [
+    {name = "SNP&SEQ Technology Platform, Uppsala University", email = "seq@medsci.uu.se" },
+]
+keywords = ["bioinformatics"]
+description = "Micro-service for downloading and verifying archives with DSMC"
+readme = "README.md"
+license = {file = "LICENSE"}
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: POSIX :: Linux",
+    "Topic :: Scientific/Engineering :: Bio-Informatics",
+    "Topic :: System :: Archiving"
+]
+dependencies = [
+    "aiohttp",
+    "pyyaml",
+    "redis",
+    "rq"
+]
+
+[project.optional-dependencies]
+test = [
+    "nose",
+    "fakeredis"]
+
+[project.scripts]
+archive-verify-ws = "archive_verify.app:start"
+
+[project.urls]
+homepage = "https://github.com/Molmed/snpseq-archive-verify"

tests/mock_redis_client.py

@@ -1,4 +1,5 @@
 import fakeredis
 
+
 def get_redis_instance():
     return fakeredis.FakeStrictRedis()

tests/test_handlers.py

@@ -1,6 +1,6 @@
 import yaml
 
-from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop
+from aiohttp.test_utils import AioHTTPTestCase
 from aiohttp import web
 
 import archive_verify.app as app_setup
@@ -23,14 +23,12 @@ async def get_application(self):
         app_setup.setup_routes(app)
         return app
 
-    @unittest_run_loop
-    async def test_root(self): 
+    async def test_root(self):
         request = await self.client.request("GET", "/")
         assert request.status == 404
         text = await request.text()
         assert "not found"  in text.lower()
 
-    @unittest_run_loop
     async def test_basic_verify(self):
         url = self.BASE_URL + "/verify"
         payload = {"host": "testbox", "archive": "test_archive", "description": "test-description"}
@@ -40,7 +38,6 @@ async def test_basic_verify(self):
         assert resp["status"] == "pending"
         assert resp["job_id"] != ""
 
-    @unittest_run_loop
     async def test_basic_status_wrong_id(self):
         url = self.BASE_URL + "/status/foobar"
         request = await self.client.request("GET", url)

tests/test_pdc_client.py

@@ -1,5 +1,5 @@
-import mock
 import unittest
+import unittest.mock as mock
 import yaml
 
 from archive_verify.pdc_client import PdcClient

tests/test_workers.py

@@ -1,6 +1,6 @@
 import copy
-import mock
 import unittest
+import unittest.mock as mock
 import yaml
 
 from archive_verify.workers import compare_md5sum, pdc_client_factory, verify_archive