From 71f1282080171e96a522684f46be081e3c813f22 Mon Sep 17 00:00:00 2001 From: Moshe Galitzky <112559840+moishce@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:41:52 +0300 Subject: [PATCH] Qualys VMDR: handle timeout error (#36388) * handle timeout * added rn * fixed cr comments * added test --- .../qualys/Integrations/Qualysv2/Qualysv2.py | 73 +++++++++++++------ .../Integrations/Qualysv2/Qualysv2_test.py | 35 ++++++++- Packs/qualys/ReleaseNotes/3_0_6.md | 4 + Packs/qualys/pack_metadata.json | 2 +- 4 files changed, 90 insertions(+), 24 deletions(-) create mode 100644 Packs/qualys/ReleaseNotes/3_0_6.md diff --git a/Packs/qualys/Integrations/Qualysv2/Qualysv2.py b/Packs/qualys/Integrations/Qualysv2/Qualysv2.py index 1c597345409..c34e2b970c5 100644 --- a/Packs/qualys/Integrations/Qualysv2/Qualysv2.py +++ b/Packs/qualys/Integrations/Qualysv2/Qualysv2.py @@ -30,6 +30,7 @@ ASSETS_FETCH_FROM = '90 days' HOST_LIMIT = 2000 TEST_FROM_DATE = 'one day' +FETCH_ASSETS_COMMAND_TIME_OUT = 180 ASSETS_DATE_FORMAT = '%Y-%m-%d' DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # ISO8601 format with UTC, default in XSOAR @@ -1679,17 +1680,23 @@ def get_host_list_detection(self, since_datetime, next_page=None, limit=HOST_LIM "truncation_limit": limit, "vm_scan_date_after": since_datetime, } + timeout = 150 if next_page: params["id_min"] = next_page - - response = self._http_request( - method='GET', - url_suffix=urljoin(API_SUFFIX, 'asset/host/vm/detection/?action=list'), - resp_type='text', - params=params, - timeout=200, - error_handler=self.error_handler, - ) + try: + response = self._http_request( + method='GET', + url_suffix=urljoin(API_SUFFIX, 'asset/host/vm/detection/?action=list'), + resp_type='text', + params=params, + timeout=timeout, + error_handler=self.error_handler, + ) + except requests.exceptions.ReadTimeout: + new_limit = set_last_run_with_new_limit(limit) + raise TimeoutError(f"Request to get host_list_detection exceeded the defined timeout ({timeout} secs). " + f"The integration will automatically reduce the request host limit from {limit} " + f"to {new_limit} in the next iteration") return response def get_vulnerabilities(self, since_datetime) -> Union[str, bytes]: @@ -2883,13 +2890,14 @@ def get_activity_logs_events(client, since_datetime, max_fetch, next_page=None) return activity_logs_events, next_run_dict -def get_host_list_detections_events(client, since_datetime, next_page='', limit=HOST_LIMIT) -> tuple: +def get_host_list_detections_events(client, since_datetime, next_page='', limit=HOST_LIMIT, is_test=False) -> tuple: """ Get host list detections from qualys Args: client: Qualys client next_page: pagination marking since_datetime: The start fetch date. limit: The limit of the host list detections + is_test: Indicates whether it's test-module run or regular run. Returns: Host list detections assets """ @@ -2899,7 +2907,7 @@ def get_host_list_detections_events(client, since_datetime, next_page='', limit= next_page = get_next_page_from_url(next_url, 'id_min') - assets = get_detections_from_hosts(host_list_assets) if host_list_assets else [] + assets = get_detections_from_hosts(host_list_assets) if host_list_assets and not is_test else [] demisto.debug(f'Parsed detections from hosts, got {len(assets)=} assets.') add_fields_to_events(assets, ['DETECTION', 'FIRST_FOUND_DATETIME'], 'host_list_detection') @@ -2932,19 +2940,20 @@ def fetch_assets(client, assets_last_run): assets: assets to push to xsiam vulnerabilities: vulnerabilities to push to xsiam """ - demisto.debug('Starting fetch for assets') since_datetime = assets_last_run.get('since_datetime', '') next_page = assets_last_run.get('next_page', '') total_assets = assets_last_run.get('total_assets', 0) - snapshot_id = assets_last_run.get('snapshot_id', round(time.time() * 1000)) + snapshot_id = assets_last_run.get('snapshot_id', str(round(time.time() * 1000))) + limit = assets_last_run.get('limit', HOST_LIMIT) if not since_datetime: since_datetime = arg_to_datetime(ASSETS_FETCH_FROM).strftime(ASSETS_DATE_FORMAT) # type: ignore[union-attr] - assets, next_run_page = get_host_list_detections_events(client, since_datetime, next_page) + assets, next_run_page = get_host_list_detections_events(client, since_datetime, next_page, limit) + total_assets += len(assets) stage = 'assets' if next_run_page else 'vulnerabilities' - amount_to_send = 0 if next_run_page else str(total_assets) + amount_to_send = 1 if next_run_page else total_assets new_last_run = {'stage': stage, 'next_page': next_run_page, 'total_assets': total_assets, 'since_datetime': since_datetime, 'snapshot_id': snapshot_id, @@ -2953,6 +2962,23 @@ def fetch_assets(client, assets_last_run): return assets, new_last_run, amount_to_send, snapshot_id +def check_fetch_duration_time(start_time, limit): + if (time.time() - start_time) > FETCH_ASSETS_COMMAND_TIME_OUT: + demisto.debug('We passed the defined timeout, so we will not send the results to XSIAM,' + 'because there is not enough time left, and we will lower the limit for the next time') + new_limit = set_last_run_with_new_limit(limit) + raise TimeoutError(f"passed the defined timeout, we will lower the limit {limit=} for the next run {new_limit}") + + +def set_last_run_with_new_limit(limit): + new_limit = int(limit / 2) if limit > 1 else 1 + demisto.debug(f'Setting host limit to: {new_limit}') + last_run = demisto.getAssetsLastRun() + last_run['limit'] = new_limit + demisto.setAssetsLastRun(last_run) + return new_limit + + def fetch_vulnerabilities(client, last_run): """ Fetches vulnerabilities Args: @@ -3076,7 +3102,7 @@ def test_module(client: Client, params: dict[str, Any], first_fetch_time: str) - ) if is_fetch_assets: since_datetime = arg_to_datetime('1 hour').strftime(ASSETS_DATE_FORMAT) # type: ignore[union-attr] - get_host_list_detections_events(client=client, since_datetime=since_datetime, limit=1) + get_host_list_detections_events(client=client, since_datetime=since_datetime, limit=1, is_test=True) else: build_args_dict({'launched_after_datetime': TEST_FROM_DATE}, COMMANDS_ARGS_DATA["test-module"], False) client.command_http_request(COMMANDS_API_DATA["test-module"]) @@ -3457,19 +3483,22 @@ def main(): # pragma: no cover fetch_stage = last_run.get('stage', 'assets') if fetch_stage == 'assets': - assets, last_run, total_assets, snapshot_id = fetch_assets(client=client, assets_last_run=last_run) + start_time = time.time() + demisto.debug(f'Starting fetch for assets, {start_time=}') + assets, new_last_run, total_assets, snapshot_id = fetch_assets(client=client, assets_last_run=last_run) + check_fetch_duration_time(start_time, last_run.get('limit')) demisto.debug('sending assets to XSIAM.') - send_data_to_xsiam(data=assets, vendor=VENDOR, product='assets', data_type='assets', snapshot_id=str(snapshot_id), - items_count=total_assets, should_update_health_module=False) - demisto.setAssetsLastRun(last_run) + send_data_to_xsiam(data=assets, vendor=VENDOR, product='assets', data_type='assets', + snapshot_id=snapshot_id, items_count=total_assets, should_update_health_module=False) + demisto.setAssetsLastRun(new_last_run) demisto.updateModuleHealth({'{data_type}Pulled'.format(data_type='assets'): total_assets}) elif fetch_stage == 'vulnerabilities': - vulnerabilities, last_run = fetch_vulnerabilities(client=client, last_run=last_run) + vulnerabilities, new_last_run = fetch_vulnerabilities(client=client, last_run=last_run) demisto.debug('sending vulnerabilities to XSIAM.') send_data_to_xsiam(data=vulnerabilities, vendor=VENDOR, product='vulnerabilities', data_type='assets') - demisto.setAssetsLastRun(last_run) + demisto.setAssetsLastRun(new_last_run) demisto.debug('finished fetch assets run') else: diff --git a/Packs/qualys/Integrations/Qualysv2/Qualysv2_test.py b/Packs/qualys/Integrations/Qualysv2/Qualysv2_test.py index 0e7e00d59de..bd52e74e748 100644 --- a/Packs/qualys/Integrations/Qualysv2/Qualysv2_test.py +++ b/Packs/qualys/Integrations/Qualysv2/Qualysv2_test.py @@ -164,11 +164,44 @@ def test_fetch_assets_command(requests_mock): ) assets, last_run, total_assets, snapshot_id = fetch_assets(client=client, assets_last_run={}) assert len(assets) == 8 - assert total_assets == '8' + assert total_assets == 8 assert snapshot_id assert last_run['stage'] == 'vulnerabilities' +def test_fetch_assets_command_time_out(requests_mock, mocker): + """ + Given: + - fetch_assets_command + When: + - Want to list all existing incidents and got a timeout + Then: + - Ensure the limit was reduced. + """ + time_out_error = False + base_url = 'https://server_url/' + with open('./test_data/host_list_detections_raw.xml') as f: + assets = f.read() + requests_mock.get(f'{base_url}api/2.0/fo/asset/host/vm/detection/' + f'?action=list&truncation_limit={HOST_LIMIT}&vm_scan_date_after=' + f'{arg_to_datetime(ASSETS_FETCH_FROM).strftime(ASSETS_DATE_FORMAT)}', exc=requests.exceptions.ReadTimeout) + set_last_run_call = mocker.patch.object(demisto, 'setAssetsLastRun') + + client = Client(base_url=base_url, + verify=True, + headers={}, + proxy=False, + username='demisto', + password='demisto', + ) + try: + assets, last_run, total_assets, snapshot_id = fetch_assets(client=client, assets_last_run={}) + except TimeoutError: + time_out_error = True + assert time_out_error + assert set_last_run_call.call_args.args[0].get('limit') == 1000 + + def test_fetch_vulnerabilities_command(requests_mock): """ Given: diff --git a/Packs/qualys/ReleaseNotes/3_0_6.md b/Packs/qualys/ReleaseNotes/3_0_6.md new file mode 100644 index 00000000000..563c7651b65 --- /dev/null +++ b/Packs/qualys/ReleaseNotes/3_0_6.md @@ -0,0 +1,4 @@ + +#### Integrations +##### Qualys VMDR +- Fixed an issue where the ***fetch-assets*** process occasionally failed on timeout. diff --git a/Packs/qualys/pack_metadata.json b/Packs/qualys/pack_metadata.json index 79e145dad9a..e720ee3593a 100644 --- a/Packs/qualys/pack_metadata.json +++ b/Packs/qualys/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Qualys", "description": "Qualys Vulnerability Management let's you create, run, fetch and manage reports, launch and manage vulnerability and compliance scans, and manage the host assets you want to scan for vulnerabilities and compliance", "support": "xsoar", - "currentVersion": "3.0.5", + "currentVersion": "3.0.6", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",