Merge pull request #275 from arXiv/ARXIVCE-1763-add-purge-by-key

adds utility functions to purge fastly cache of surrogate keys

arxiv/config/__init__.py

@@ -179,4 +179,9 @@ class Settings(BaseSettings):
     """ How many requests do we handle at once -> How many db connections should we be able to open at once """
     POOL_PRE_PING: bool = True
     """ Liveness check of sqlalchemy connections before checking out of pool """
+
+
+    FASTLY_SERVICE_IDS:str='{"arxiv.org":"umpGzwE2hXfa2aRXsOQXZ4", "browse.dev.arxiv.org":"5eZxUHBG78xXKNrnWcdDO7", "export.arxiv.org": "hCz5jlkWV241zvUN0aWxg2", "rss.arxiv.org": "yPg50VJsPLwZQ5lFsD7rA1"}'
+    """a dictionary of the various fastly services and their ids"""
+    FASTLY_PURGE_TOKEN:str= "FASTLY_PURGE_TOKEN"
 settings = Settings()

arxiv/integration/fastly/purge.py

@@ -0,0 +1,58 @@
+from typing import List, Optional, Union, Any
+import logging
+import json
+
+import fastly 
+from fastly.api.purge_api import PurgeApi
+
+from arxiv.config import settings
+
+logger = logging.getLogger(__name__)
+
+SERVICE_IDS=json.loads(settings.FASTLY_SERVICE_IDS)
+MAX_PURGE_KEYS=256
+
+def purge_fastly_keys(key:Union[str, List[str]], service_name: Optional[str]="arxiv.org"):
+    """purges requested fastly surrogate keys for the service.
+    If no service is specified default is the arxiv.org service
+    """
+    configuration = fastly.Configuration()
+    configuration.api_token = settings.FASTLY_PURGE_TOKEN
+
+    with fastly.ApiClient(configuration) as api_client:
+        api_instance = PurgeApi(api_client)
+        try:
+            if isinstance(key, str):
+                api_response=_purge_single_key(key, SERVICE_IDS[service_name], api_instance)
+                logger.info(f"Fastly Purge service: {service_name}, key: {key}, status: {api_response.get('status')}, id: {api_response.get('id')}")
+            else:
+                _purge_multiple_keys(key, SERVICE_IDS[service_name], api_instance)
+                logger.info(f"Fastly bulk purge complete service: {service_name}, keys: {key}")
+        except fastly.ApiException as e:
+            logger.error(f"Exception purging fastly key(s): {e} service: {service_name}, key: {key}")
+
+def _purge_single_key(key:str, service_id: str, api_instance: PurgeApi)->Any:
+    """purge all pages with a specific key from fastly, fastly will not indicate if the key does not exist"""
+    options = {
+        'service_id': service_id,
+        'surrogate_key': key,
+        'fastly_soft_purge':1
+    }
+    return api_instance.purge_tag(**options)
+
+def _purge_multiple_keys(keys: List[str], service_id:str, api_instance: PurgeApi):
+    """purge all pages with any of the requested keys from fastly
+        calls itself recursively to stay within fastly maximum key amount
+    """
+    if len(keys)> MAX_PURGE_KEYS:
+        _purge_multiple_keys(keys[0:MAX_PURGE_KEYS], service_id, api_instance)
+        _purge_multiple_keys(keys[MAX_PURGE_KEYS:], service_id, api_instance)
+
+    options = {
+        'service_id': service_id,
+        'purge_response': {'surrogate_keys':keys,},
+        'fastly_soft_purge':1
+    }
+    api_response=api_instance.bulk_purge_tag(**options)
+    logger.debug(f"Bulk purge keys response: {api_response}")
+    return

arxiv/integration/tests/test_fastly.py

@@ -0,0 +1,67 @@
+import unittest
+from unittest.mock import patch, MagicMock
+from fastly.api.purge_api import PurgeApi
+
+from arxiv.integration.fastly.purge import purge_fastly_keys
+
+class TestPurgeFastlyKeys(unittest.TestCase):
+    @patch('arxiv.integration.fastly.purge.PurgeApi')
+    @patch('arxiv.integration.fastly.purge.fastly.ApiClient')
+    def test_purge_single_key(self, MockApiClient, MockPurgeApi: PurgeApi):
+        mock_api_instance:PurgeApi = MockPurgeApi.return_value
+        mock_api_instance.purge_tag = MagicMock()
+
+        purge_fastly_keys('test', "export.arxiv.org")
+
+        mock_api_instance.purge_tag.assert_called_once_with(
+            service_id="hCz5jlkWV241zvUN0aWxg2",
+            surrogate_key='test',
+            fastly_soft_purge=1
+        )
+
+
+    @patch('arxiv.integration.fastly.purge.PurgeApi')
+    @patch('arxiv.integration.fastly.purge.fastly.ApiClient')
+    def test_purge_multiple_keys(self, MockApiClient, MockPurgeApi: PurgeApi):
+        mock_api_instance:PurgeApi = MockPurgeApi.return_value
+        mock_api_instance.bulk_purge_tag = MagicMock()
+
+        keys = ['key1', 'key2']
+        purge_fastly_keys(keys)
+
+        mock_api_instance.bulk_purge_tag.assert_called_once_with(
+            service_id="umpGzwE2hXfa2aRXsOQXZ4",
+            purge_response= {'surrogate_keys':keys},
+            fastly_soft_purge=1
+        )
+
+
+    @patch('arxiv.integration.fastly.purge.PurgeApi')
+    @patch('arxiv.integration.fastly.purge.fastly.ApiClient')
+    @patch('arxiv.integration.fastly.purge.MAX_PURGE_KEYS', 3)
+    def test_purge_over_max_keys(self, MockApiClient, MockPurgeApi: PurgeApi):
+        mock_api_instance:PurgeApi = MockPurgeApi.return_value
+        mock_api_instance.bulk_purge_tag = MagicMock()
+
+        keys = ['1', '2', '3', '4','5','6','7']
+        purge_fastly_keys(keys)
+        calls = [
+            unittest.mock.call(
+                service_id="umpGzwE2hXfa2aRXsOQXZ4",
+                purge_response= {'surrogate_keys':['1','2','3']},
+                fastly_soft_purge=1
+            ),
+                        unittest.mock.call(
+                service_id="umpGzwE2hXfa2aRXsOQXZ4",
+                purge_response= {'surrogate_keys':['4','5','6']},
+                fastly_soft_purge=1
+            ),
+                        unittest.mock.call(
+                service_id="umpGzwE2hXfa2aRXsOQXZ4",
+                purge_response= {'surrogate_keys':['7']},
+                fastly_soft_purge=1
+            ),
+        ]
+
+        mock_api_instance.bulk_purge_tag.assert_has_calls(calls, any_order=True)
+