From 7389c650e3380069f02d19e800c74a8610e58a16 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Fri, 5 Jul 2024 08:59:07 -0700 Subject: [PATCH 1/3] bugfix, dont call with too many items --- arxiv/integration/fastly/purge.py | 18 +++++++++--------- arxiv/integration/tests/test_fastly.py | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/arxiv/integration/fastly/purge.py b/arxiv/integration/fastly/purge.py index e31304ff..b7588ef2 100644 --- a/arxiv/integration/fastly/purge.py +++ b/arxiv/integration/fastly/purge.py @@ -143,13 +143,13 @@ def _purge_multiple_keys(keys: List[str], service_id:str, api_instance: PurgeApi if len(keys)> MAX_PURGE_KEYS: _purge_multiple_keys(keys[0:MAX_PURGE_KEYS], service_id, api_instance, soft_purge) _purge_multiple_keys(keys[MAX_PURGE_KEYS:], service_id, api_instance,soft_purge) - - options = { - 'service_id': service_id, - 'purge_response': {'surrogate_keys':keys,} - } - if soft_purge: - options['fastly_soft_purge']=1 - api_response=api_instance.bulk_purge_tag(**options) - logger.debug(f"Bulk purge keys response: {api_response}") + else: + options = { + 'service_id': service_id, + 'purge_response': {'surrogate_keys':keys,} + } + if soft_purge: + options['fastly_soft_purge']=1 + api_response=api_instance.bulk_purge_tag(**options) + logger.debug(f"Bulk purge keys response: {api_response}") return diff --git a/arxiv/integration/tests/test_fastly.py b/arxiv/integration/tests/test_fastly.py index 116608a5..c3486cc5 100644 --- a/arxiv/integration/tests/test_fastly.py +++ b/arxiv/integration/tests/test_fastly.py @@ -77,6 +77,7 @@ def test_purge_over_max_keys(self, MockApiClient, MockPurgeApi: PurgeApi): ] mock_api_instance.bulk_purge_tag.assert_has_calls(calls, any_order=True) + self.assertEqual(mock_api_instance.bulk_purge_tag.call_count, len(calls)) #tests for adding surrogate keys helper function From b5929368145d1a925a17c59d8ca788d990e04696 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Fri, 5 Jul 2024 09:25:27 -0700 Subject: [PATCH 2/3] adds number of keys to log --- arxiv/integration/fastly/purge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arxiv/integration/fastly/purge.py b/arxiv/integration/fastly/purge.py index b7588ef2..7e2ee1b8 100644 --- a/arxiv/integration/fastly/purge.py +++ b/arxiv/integration/fastly/purge.py @@ -122,7 +122,7 @@ def purge_fastly_keys(key:Union[str, List[str]], service_name: Optional[str]="ar logger.info(f"Fastly Purge service: {service_name}, key: {key}, status: {api_response.get('status')}, id: {api_response.get('id')}") else: _purge_multiple_keys(key, SERVICE_IDS[service_name], api_instance, soft_purge) - logger.info(f"Fastly bulk purge complete service: {service_name}, keys: {key}") + logger.info(f"Fastly bulk purge complete service: {service_name}, keys ({len(key)}): {key}") except fastly.ApiException as e: logger.error(f"Exception purging fastly key(s): {e} service: {service_name}, key: {key}") From 125ffc690d4dd7b70cc9a8a19e4b20cd3e66c787 Mon Sep 17 00:00:00 2001 From: kyokukou Date: Fri, 5 Jul 2024 09:36:28 -0700 Subject: [PATCH 3/3] better logging for bulk keys --- arxiv/integration/fastly/purge.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/arxiv/integration/fastly/purge.py b/arxiv/integration/fastly/purge.py index 7e2ee1b8..9281600e 100644 --- a/arxiv/integration/fastly/purge.py +++ b/arxiv/integration/fastly/purge.py @@ -116,15 +116,18 @@ def purge_fastly_keys(key:Union[str, List[str]], service_name: Optional[str]="ar with fastly.ApiClient(configuration) as api_client: api_instance = PurgeApi(api_client) - try: - if isinstance(key, str): + if isinstance(key, str): + try: api_response=_purge_single_key(key, SERVICE_IDS[service_name], api_instance, soft_purge) logger.info(f"Fastly Purge service: {service_name}, key: {key}, status: {api_response.get('status')}, id: {api_response.get('id')}") - else: + except fastly.ApiException as e: + logger.error(f"Exception purging fastly key(s): {e} service: {service_name}, key: {key}") + else: + try: _purge_multiple_keys(key, SERVICE_IDS[service_name], api_instance, soft_purge) logger.info(f"Fastly bulk purge complete service: {service_name}, keys ({len(key)}): {key}") - except fastly.ApiException as e: - logger.error(f"Exception purging fastly key(s): {e} service: {service_name}, key: {key}") + except fastly.ApiException as e: + logger.error(f"Exception purging fastly key(s): {e} service: {service_name}, for {len(key)} keys") def _purge_single_key(key:str, service_id: str, api_instance: PurgeApi, soft_purge: bool=False)->Any: """purge all pages with a specific key from fastly, fastly will not indicate if the key does not exist""" @@ -151,5 +154,5 @@ def _purge_multiple_keys(keys: List[str], service_id:str, api_instance: PurgeApi if soft_purge: options['fastly_soft_purge']=1 api_response=api_instance.bulk_purge_tag(**options) - logger.debug(f"Bulk purge keys response: {api_response}") + #logger.debug(f"Bulk purge keys response: {api_response}") return