Merge pull request #6 from Molmed/DEVELOP-1339-fix-security-advisory

WIP: Develop 1339 - update app to run on Python 3.6

.python-version

@@ -0,0 +1 @@
+3.6.12

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 
 python:
-        - "3.5"
+        - "3.6.12"
 
 before_install:
         - sudo python -m pip install pipenv

Pipfile

@@ -9,16 +9,17 @@ python_version = "3.6"
 
 [packages]
 
-aiohttp = "==2.3.7"
+aiohttp = "==3.7.4"
 rq = "==0.10.0"
 redis = "==2.10.6"
 aiodns = "==1.1.1"
 cchardet = "==2.1.1"
-pyyaml = "==3.12"
-yarl = "==0.18.0"
+pyyaml = "==5.4"
+yarl = "==1.6.3"
 
 
 [dev-packages]
 
 mock = "*"
 nose = "*"
+fakeredis = "*"

README.md

@@ -7,12 +7,24 @@ The web service enqueues certain job functions in the RQ/Redis queue, where they
 
 The downloaded files are deleted on successful verification, and retained if any error occurs.
 
+Pre-requisites
+--------------
+You will need python 3.6 and redis.
+
+Install redis with:
+
+    apt-get install redis-server
+
+And start it with:
+
+    redis-server
+
+
 Trying it out
 -------------
 
     python3 -m pip install pipenv
     pipenv install --deploy
-    apt-get install redis-server
 
 Try running it:
 

archive_verify/__init__.py

@@ -1 +1 @@
-__version__ = "1.0.0"
+__version__ = "1.0.5"

archive_verify/handlers.py

@@ -1,9 +1,8 @@
-import json
 import logging
 import os
 
 from aiohttp import web
-from redis import Redis
+import archive_verify.redis_client as redis_client
 from rq import Queue
 
 from archive_verify.workers import verify_archive
@@ -31,7 +30,7 @@ async def verify(request):
     # use a supplied path if available, otherwise construct it from the src_root and archive
     archive_path = path or os.path.join(src_root, archive)
 
-    redis_conn = Redis()
+    redis_conn = redis_client.get_redis_instance()
     q = Queue(connection=redis_conn)
 
     # Enqueue the verify_archive function with the user supplied input parameters.
@@ -63,7 +62,7 @@ async def status(request):
     """
     job_id = str(request.match_info['job_id'])
 
-    redis_conn = Redis()
+    redis_conn = redis_client.get_redis_instance()
     q = Queue(connection=redis_conn)
     job = q.fetch_job(job_id)
 

archive_verify/redis_client.py

@@ -0,0 +1,5 @@
+from redis import Redis
+
+
+def get_redis_instance():
+    return Redis()

requirements.txt

@@ -1,6 +1,28 @@
-aiohttp==2.3.7
 aiodns==1.1.1
-pyyaml==3.12
-rq==0.10.0
+aiohttp==3.7.4
+async-timeout==3.0.1
+attrs==21.2.0
 cchardet==2.1.1
-yarl==0.18.0
+cffi==1.14.6
+chardet==3.0.4
+click==8.0.1
+fakeredis==1.6.1
+idna==3.2
+idna-ssl==1.1.0
+importlib-metadata==4.8.1
+mock==4.0.3
+multidict==5.2.0
+nose==1.3.7
+packaging==21.0
+pbr==5.1.1
+pycares==4.0.0
+pycparser==2.20
+pyparsing==2.4.7
+PyYAML==5.4
+redis==2.10.6
+rq==0.10.0
+six==1.16.0
+sortedcontainers==2.4.0
+typing-extensions==3.10.0.2
+yarl==1.6.3
+zipp==3.6.0

tests/mock_redis_client.py

@@ -0,0 +1,4 @@
+import fakeredis
+
+def get_redis_instance():
+    return fakeredis.FakeStrictRedis()

tests/test_handlers.py

@@ -3,21 +3,24 @@
 from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop
 from aiohttp import web
 
-from archive_verify.app import setup_routes
+import archive_verify.app as app_setup
+import mock_redis_client
+
 
 class HandlerTestCase(AioHTTPTestCase): 
 
     BASE_URL = ""
 
     def _load_config(self):
         with open("tests/test_config.yaml") as config:
-            return yaml.load(config)
+            return yaml.safe_load(config)
 
     async def get_application(self):
         app = web.Application()
         app["config"] = self._load_config()
         self.BASE_URL = app["config"]["base_url"]
-        setup_routes(app)
+        app_setup.handlers.redis_client = mock_redis_client
+        app_setup.setup_routes(app)
         return app
 
     @unittest_run_loop
@@ -28,7 +31,7 @@ async def test_root(self):
         assert "not found"  in text.lower()
 
     @unittest_run_loop
-    async def test_basic_verify(self): 
+    async def test_basic_verify(self):
         url = self.BASE_URL + "/verify"
         payload = {"host": "testbox", "archive": "test_archive", "description": "test-description"}
         request = await self.client.request("POST", url, json=payload)

tests/test_pdc_client.py

@@ -9,7 +9,7 @@ class TestPdcClient(unittest.TestCase):
 
     def setUp(self):
         with open("tests/test_config.yaml") as config:
-            self.config = yaml.load(config)
+            self.config = yaml.safe_load(config)
 
     def getPdcClient(self):
         return PdcClient("archive", "path", "descr", "1234", self.config)

tests/test_workers.py

@@ -9,7 +9,7 @@
 class TestWorkers(unittest.TestCase):
     def setUp(self):
         with open("tests/test_config.yaml") as config:
-            self.config = yaml.load(config)
+            self.config = yaml.safe_load(config)
 
     def test_pdc_client_is_default(self):
         pdc_client_class = pdc_client_factory(self.config)