Merge pull request #12 from ustudio/connect-to-redis-via-uri

Change Client API to connect to Redis via URI

.circleci/config.yml

@@ -0,0 +1,141 @@
+version: 2
+jobs:
+  test-python27:
+    docker:
+      - image: circleci/python:2.7
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-python27-{{ checksum "setup.py" }}-{{ checksum "requirements.txt" }}
+            - v1-dependencies-python27-{{ checksum "setup.py" }}-
+            - v1-dependencies-python27-
+
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m virtualenv ~/venv
+            . ~/venv/bin/activate
+            pip install -e .
+            pip install -r requirements.txt
+            mkdir -p test-reports
+
+      - save_cache:
+          paths:
+            - ~/venv
+          key: v1-dependencies-python27-{{ checksum "setup.py" }}-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: Run Tests
+          command: |
+            . ~/venv/bin/activate
+            nosetests --verbose --with-xunit --xunit-file=test-reports/nosetests-python27.xml
+
+      - run:
+          name: Run Linter
+          command: |
+            . ~/venv/bin/activate
+            flake8
+
+      - store_artifacts:
+          path: test-reports
+
+      - store_test_results:
+          path: test-reports
+
+  test-python37:
+    docker:
+      - image: circleci/python:3.7
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-python37-{{ checksum "setup.py" }}-{{ checksum "requirements.txt" }}
+            - v1-dependencies-python37-{{ checksum "setup.py" }}-
+            - v1-dependencies-python37-
+
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m virtualenv ~/venv
+            . ~/venv/bin/activate
+            pip install -e .
+            pip install -r requirements.txt
+            mkdir -p test-reports
+
+      - save_cache:
+          paths:
+            - ~/venv
+          key: v1-dependencies-python37-{{ checksum "setup.py" }}-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: Run Tests
+          command: |
+            . ~/venv/bin/activate
+            nosetests --verbose --with-xunit --xunit-file=test-reports/nosetests-python37.xml
+
+      - run:
+          name: Run Linter
+          command: |
+            . ~/venv/bin/activate
+            flake8
+
+      - store_artifacts:
+          path: test-reports
+
+      - store_test_results:
+          path: test-reports
+
+  publish:
+    docker:
+      - image: circleci/python:3.7
+    working_directory: ~/repo
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+          - v1-publish-dependencies-
+
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m virtualenv ~/venv
+            . ~/venv/bin/activate
+            pip install twine
+
+      - save_cache:
+          paths:
+            - ~/venv
+          key: v1-publish-dependencies-
+
+      - run:
+          name: Publish to PyPI
+          command: |
+            . ~/venv/bin/activate
+            ./publish_to_pypi.sh
+
+workflows:
+  version: 2
+  test-and-publish:
+    jobs:
+      - test-python27
+      - test-python37
+      - publish:
+          requires:
+            - test-python27
+            - test-python37
+          filters:
+            tags:
+              only: /^v[0-9]+(\.[0-9]+)*.*/
+            branches:
+              ignore: /.*/
+          context: org-global

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 100
+ignore=

README.md

@@ -39,7 +39,7 @@ system, and constructs the other objects necessary for interacting
 with the system
 
 ```python
-client = Blueque.Client(hostname, port, db)
+client = Blueque.Client("redis://hostname:port/db")
 ```
 
 ### Queue ###

blueque/__init__.py

@@ -1 +1 @@
-from blueque.client import Client
+from blueque.client import Client  # noqa: F401

blueque/client.py

@@ -9,10 +9,10 @@
 
 
 class Client(object):
-    def __init__(self, hostname, port, db, **kwargs):
+    def __init__(self, *args, **kwargs):
         super(Client, self).__init__()
 
-        self._redis = redis.StrictRedis(host=hostname, port=port, db=db, **kwargs)
+        self._redis = redis.StrictRedis.from_url(*args, **kwargs)
 
     def get_queue(self, name):
         redis_queue = RedisQueue(name, self._redis)

examples/forking_worker.py

@@ -1,21 +1,22 @@
 # This hasn't actually been run; it's just a scratch-pad, for now, to
 # figure out what API we want
 
+from __future__ import print_function
 from blueque import Client
 from blueque import forking_runner
 
 import time
 
 
 def do_work(task):
-    print task.id, task.parameters
+    print(task.id, task.parameters)
 
     time.sleep(1000)
 
     return "result"
 
 
 if __name__ == "__main__":
-    client = Client(hostname="localhost", port=6379, db=0)
+    client = Client("redis://localhost")
 
     forking_runner.run(client, "some.queue", do_work, 4)

requirements.txt

@@ -1,3 +1,3 @@
-coverage==3.6
+flake8==3.7.8
 mock==1.0.1
 nose==1.3.0

setup.py

@@ -10,7 +10,7 @@
 ]
 
 setup(name="blueque",
-      version="0.2.5",
+      version="0.3.0",
       description="Simple job queuing for very long tasks",
       url="https://github.com/ustudio/Blueque",
       packages=["blueque"],

tests/test_client.py

@@ -13,14 +13,13 @@ class TestClient(unittest.TestCase):
     # classes they create.
 
     @mock.patch("redis.StrictRedis", autospec=True)
-    def test_client_connects_with_requested_information(self, mock_redis):
-        self.client = Client(hostname="foo", port=1234, db=0)
+    def test_client_connects_with_requested_information(self, mock_redis_class):
+        self.client = Client("redis://url")
 
-        mock_redis.assert_called_with(host="foo", port=1234, db=0)
+        mock_redis_class.from_url.assert_called_with("redis://url")
 
     @mock.patch("redis.StrictRedis", autospec=True)
-    def test_client_passes_additional_args_to_client(self, mock_redis):
-        self.client = Client(hostname="foo", port=1234, db=0, password="password", charset="utf8")
+    def test_client_passes_additional_args_to_client(self, mock_redis_class):
+        self.client = Client("redis://url", 4, socket_timeout=5)
 
-        mock_redis.assert_called_with(
-            host="foo", port=1234, db=0, password="password", charset="utf8")
+        mock_redis_class.from_url.assert_called_with("redis://url", 4, socket_timeout=5)

tests/test_forking_runner.py

@@ -18,12 +18,12 @@ class BreakLoop(RuntimeError):
 @mock.patch("blueque.client.RedisQueue", autospec=True)
 class TestForkingRunner(unittest.TestCase):
     @mock.patch("redis.StrictRedis", autospec=True)
-    def setUp(self, strict_redis_class):
-        self.mock_strict_redis = strict_redis_class.return_value
+    def setUp(self, mock_redis_class):
+        self.mock_strict_redis = mock_redis_class.from_url.return_value
 
         self.task_callback = mock.Mock()
 
-        self.client = Client(hostname="asdf", port=1234, db=0)
+        self.client = Client("redis://asdf:1234")
         self.runner = forking_runner.ForkingRunner(self.client, "some.queue", self.task_callback)
 
     def _get_task(self, **kwargs):

tests/test_listener.py

@@ -14,19 +14,19 @@ class TestListener(unittest.TestCase):
     @mock.patch("redis.StrictRedis", autospec=True)
     @mock.patch("blueque.client.RedisQueue", autospec=True)
     def setUp(self, mock_redis_queue_class, mock_strict_redis, _, __):
-        self.mock_strict_redis = mock_strict_redis
+        self.mock_strict_redis = mock_strict_redis.from_url.return_value
 
         self.mock_redis_queue_class = mock_redis_queue_class
         self.mock_redis_queue = mock_redis_queue_class.return_value
 
-        self.client = Client("asdf", 1234, 0)
+        self.client = Client("redis://asdf:1234")
         self.listener = self.client.get_listener("some.queue")
 
     def test_listener_adds_itself(self):
         self.mock_redis_queue.add_listener.assert_called_with("somehost.example.com_2314")
 
     def test_listener_calls_callback_when_task_in_queue(self):
-        self.mock_strict_redis.return_value.hgetall.return_value = {
+        self.mock_strict_redis.hgetall.return_value = {
             "parameters": "some parameters"
         }
         self.mock_redis_queue.dequeue.side_effect = ["some_task"]
@@ -127,7 +127,7 @@ def test_claim_orphan_returns_task_when_reclaimed(self, mock_kill):
         self.mock_redis_queue.remove_listener.return_value = 1
         self.mock_redis_queue.reclaim_task.return_value = "some_task"
 
-        self.mock_strict_redis.return_value.hgetall.return_value = {
+        self.mock_strict_redis.hgetall.return_value = {
             "parameters": "some parameters"
         }
 
@@ -138,7 +138,7 @@ def test_claim_orphan_returns_task_when_reclaimed(self, mock_kill):
         mock_kill.assert_called_with(4321, 0)
         self.mock_redis_queue.reclaim_task.assert_called_with(
             "somehost.example.com_4321", "somehost.example.com_2314")
-        self.mock_strict_redis.return_value.hgetall.assert_called_with("blueque_task_some_task")
+        self.mock_strict_redis.hgetall.assert_called_with("blueque_task_some_task")
 
         self.assertIsNotNone(claimed)
         self.assertEqual("some parameters", claimed.parameters)

tests/test_processor.py

@@ -13,7 +13,7 @@ class TestProcessor(unittest.TestCase):
     @mock.patch("blueque.client.RedisQueue", autospec=True)
     def setUp(self, mock_redis_queue_class, mock_strict_redis):
         self.mock_redis_queue = mock_redis_queue_class.return_value
-        self.mock_strict_redis = mock_strict_redis.return_value
+        self.mock_strict_redis = mock_strict_redis.from_url.return_value
 
         self.mock_strict_redis.hgetall.return_value = {
             "queue": "some.queue",
@@ -22,7 +22,7 @@ def setUp(self, mock_redis_queue_class, mock_strict_redis):
             "parameters": "some parameters"
         }
 
-        self.client = Client(hostname="asdf", port=1234, db=0)
+        self.client = Client("redis://asdf:1234")
         self.task = self.client.get_task("some_task")
         self.processor = self.client.get_processor(self.task)
 
@@ -61,7 +61,7 @@ class TestProcessorWithStartedTask(unittest.TestCase):
     @mock.patch("blueque.client.RedisQueue", autospec=True)
     def setUp(self, mock_redis_queue_class, mock_strict_redis):
         self.mock_redis_queue = mock_redis_queue_class.return_value
-        self.mock_strict_redis = mock_strict_redis.return_value
+        self.mock_strict_redis = mock_strict_redis.from_url.return_value
 
         self.mock_strict_redis.hgetall.return_value = {
             "queue": "some.queue",
@@ -71,7 +71,7 @@ def setUp(self, mock_redis_queue_class, mock_strict_redis):
             "parameters": "some parameters"
         }
 
-        self.client = Client(hostname="asdf", port=1234, db=0)
+        self.client = Client("redis://asdf:1234")
         self.task = self.client.get_task("some_task")
         self.processor = self.client.get_processor(self.task)
 

tests/test_queue.py

@@ -12,17 +12,16 @@ class TestQueue(unittest.TestCase):
     @mock.patch("redis.StrictRedis", autospec=True)
     @mock.patch("blueque.client.RedisQueue", autospec=True)
     def setUp(self, mock_redis_queue_class, mock_strict_redis):
-        self.mock_strict_redis = mock_strict_redis
+        self.mock_strict_redis = mock_strict_redis.from_url.return_value
 
         self.mock_redis_queue_class = mock_redis_queue_class
         self.mock_redis_queue = mock_redis_queue_class.return_value
 
-        self.client = Client("asdf", 1234, 0)
+        self.client = Client("redis://asdf:1234")
         self.queue = self.client.get_queue("some.queue")
 
     def test_name_passed_to_redis_queue(self):
-        self.mock_redis_queue_class.assert_called_with(
-            "some.queue", self.mock_strict_redis.return_value)
+        self.mock_redis_queue_class.assert_called_with("some.queue", self.mock_strict_redis)
 
     def test_enqueue_enqueues_task(self):
         self.mock_redis_queue.enqueue.return_value = "task_id"
@@ -46,7 +45,7 @@ def test_enqueue_due_tasks_enqueues_due_tasks(self):
         self.mock_redis_queue.enqueue_due_tasks.assert_called_with()
 
     def test_delete_deletes_task(self):
-        self.mock_strict_redis.return_value.hgetall.return_value = {
+        self.mock_strict_redis.hgetall.return_value = {
             "status": "complete",
             "queue": "some.queue"
         }
@@ -58,7 +57,7 @@ def test_delete_deletes_task(self):
         self.mock_redis_queue.delete_task.assert_called_with("some_task", "complete")
 
     def test_delete_errors_on_wrong_queue(self):
-        self.mock_strict_redis.return_value.hgetall.return_value = {
+        self.mock_strict_redis.hgetall.return_value = {
             "status": "complete",
             "queue": "other.queue"
         }

tests/test_redis_queue.py

@@ -308,7 +308,7 @@ def test_schedule_with_past_eta_just_enqueues(self):
         self.assertFalse(self._get_pipeline().zadd.called)
 
     def test_enqueue_due_enqueues_all_due_tasks(self):
-        pipeline = mock.MagicMock(spec=redis.client.StrictPipeline)
+        pipeline = mock.MagicMock(spec=redis.client.Pipeline)
 
         pipeline.zrangebyscore.return_value = ["some_task", "other_task"]
 
@@ -344,7 +344,7 @@ def test_enqueue_due_enqueues_all_due_tasks(self):
             "Blueque queue some.queue: enqueuing due tasks: ['some_task', 'other_task']")
 
     def test_enqueue_due_does_nothing_when_nothing_is_due(self):
-        pipeline = mock.MagicMock(spec=redis.client.StrictPipeline)
+        pipeline = mock.MagicMock(spec=redis.client.Pipeline)
 
         pipeline.zrangebyscore.return_value = []