Qualys VMDR: handle timeout error (#36388)

* handle timeout

* added rn

* fixed cr comments

* added test

Packs/qualys/Integrations/Qualysv2/Qualysv2.py

@@ -30,6 +30,7 @@
 ASSETS_FETCH_FROM = '90 days'
 HOST_LIMIT = 2000
 TEST_FROM_DATE = 'one day'
+FETCH_ASSETS_COMMAND_TIME_OUT = 180
 
 ASSETS_DATE_FORMAT = '%Y-%m-%d'
 DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"  # ISO8601 format with UTC, default in XSOAR
@@ -1679,17 +1680,23 @@ def get_host_list_detection(self, since_datetime, next_page=None, limit=HOST_LIM
             "truncation_limit": limit,
             "vm_scan_date_after": since_datetime,
         }
+        timeout = 150
         if next_page:
             params["id_min"] = next_page
-
-        response = self._http_request(
-            method='GET',
-            url_suffix=urljoin(API_SUFFIX, 'asset/host/vm/detection/?action=list'),
-            resp_type='text',
-            params=params,
-            timeout=200,
-            error_handler=self.error_handler,
-        )
+        try:
+            response = self._http_request(
+                method='GET',
+                url_suffix=urljoin(API_SUFFIX, 'asset/host/vm/detection/?action=list'),
+                resp_type='text',
+                params=params,
+                timeout=timeout,
+                error_handler=self.error_handler,
+            )
+        except requests.exceptions.ReadTimeout:
+            new_limit = set_last_run_with_new_limit(limit)
+            raise TimeoutError(f"Request to get host_list_detection exceeded the defined timeout ({timeout} secs). "
+                               f"The integration will automatically reduce the request host limit from {limit} "
+                               f"to {new_limit} in the next iteration")
         return response
 
     def get_vulnerabilities(self, since_datetime) -> Union[str, bytes]:
@@ -2883,13 +2890,14 @@ def get_activity_logs_events(client, since_datetime, max_fetch, next_page=None)
     return activity_logs_events, next_run_dict
 
 
-def get_host_list_detections_events(client, since_datetime, next_page='', limit=HOST_LIMIT) -> tuple:
+def get_host_list_detections_events(client, since_datetime, next_page='', limit=HOST_LIMIT, is_test=False) -> tuple:
     """ Get host list detections from qualys
     Args:
         client: Qualys client
         next_page: pagination marking
         since_datetime: The start fetch date.
         limit: The limit of the host list detections
+        is_test: Indicates whether it's test-module run or regular run.
     Returns:
         Host list detections assets
     """
@@ -2899,7 +2907,7 @@ def get_host_list_detections_events(client, since_datetime, next_page='', limit=
 
     next_page = get_next_page_from_url(next_url, 'id_min')
 
-    assets = get_detections_from_hosts(host_list_assets) if host_list_assets else []
+    assets = get_detections_from_hosts(host_list_assets) if host_list_assets and not is_test else []
     demisto.debug(f'Parsed detections from hosts, got {len(assets)=} assets.')
 
     add_fields_to_events(assets, ['DETECTION', 'FIRST_FOUND_DATETIME'], 'host_list_detection')
@@ -2932,19 +2940,20 @@ def fetch_assets(client, assets_last_run):
         assets: assets to push to xsiam
         vulnerabilities: vulnerabilities to push to xsiam
     """
-    demisto.debug('Starting fetch for assets')
     since_datetime = assets_last_run.get('since_datetime', '')
     next_page = assets_last_run.get('next_page', '')
     total_assets = assets_last_run.get('total_assets', 0)
-    snapshot_id = assets_last_run.get('snapshot_id', round(time.time() * 1000))
+    snapshot_id = assets_last_run.get('snapshot_id', str(round(time.time() * 1000)))
+    limit = assets_last_run.get('limit', HOST_LIMIT)
 
     if not since_datetime:
         since_datetime = arg_to_datetime(ASSETS_FETCH_FROM).strftime(ASSETS_DATE_FORMAT)  # type: ignore[union-attr]
 
-    assets, next_run_page = get_host_list_detections_events(client, since_datetime, next_page)
+    assets, next_run_page = get_host_list_detections_events(client, since_datetime, next_page, limit)
+
     total_assets += len(assets)
     stage = 'assets' if next_run_page else 'vulnerabilities'
-    amount_to_send = 0 if next_run_page else str(total_assets)
+    amount_to_send = 1 if next_run_page else total_assets
 
     new_last_run = {'stage': stage, 'next_page': next_run_page, 'total_assets': total_assets,
                     'since_datetime': since_datetime, 'snapshot_id': snapshot_id,
@@ -2953,6 +2962,23 @@ def fetch_assets(client, assets_last_run):
     return assets, new_last_run, amount_to_send, snapshot_id
 
 
+def check_fetch_duration_time(start_time, limit):
+    if (time.time() - start_time) > FETCH_ASSETS_COMMAND_TIME_OUT:
+        demisto.debug('We passed the defined timeout, so we will not send the results to XSIAM,'
+                      'because there is not enough time left, and we will lower the limit for the next time')
+        new_limit = set_last_run_with_new_limit(limit)
+        raise TimeoutError(f"passed the defined timeout, we will lower the limit {limit=} for the next run {new_limit}")
+
+
+def set_last_run_with_new_limit(limit):
+    new_limit = int(limit / 2) if limit > 1 else 1
+    demisto.debug(f'Setting host limit to: {new_limit}')
+    last_run = demisto.getAssetsLastRun()
+    last_run['limit'] = new_limit
+    demisto.setAssetsLastRun(last_run)
+    return new_limit
+
+
 def fetch_vulnerabilities(client, last_run):
     """ Fetches vulnerabilities
     Args:
@@ -3076,7 +3102,7 @@ def test_module(client: Client, params: dict[str, Any], first_fetch_time: str) -
             )
         if is_fetch_assets:
             since_datetime = arg_to_datetime('1 hour').strftime(ASSETS_DATE_FORMAT)  # type: ignore[union-attr]
-            get_host_list_detections_events(client=client, since_datetime=since_datetime, limit=1)
+            get_host_list_detections_events(client=client, since_datetime=since_datetime, limit=1, is_test=True)
     else:
         build_args_dict({'launched_after_datetime': TEST_FROM_DATE}, COMMANDS_ARGS_DATA["test-module"], False)
         client.command_http_request(COMMANDS_API_DATA["test-module"])
@@ -3457,19 +3483,22 @@ def main():  # pragma: no cover
             fetch_stage = last_run.get('stage', 'assets')
 
             if fetch_stage == 'assets':
-                assets, last_run, total_assets, snapshot_id = fetch_assets(client=client, assets_last_run=last_run)
+                start_time = time.time()
+                demisto.debug(f'Starting fetch for assets, {start_time=}')
+                assets, new_last_run, total_assets, snapshot_id = fetch_assets(client=client, assets_last_run=last_run)
+                check_fetch_duration_time(start_time, last_run.get('limit'))
 
                 demisto.debug('sending assets to XSIAM.')
-                send_data_to_xsiam(data=assets, vendor=VENDOR, product='assets', data_type='assets', snapshot_id=str(snapshot_id),
-                                   items_count=total_assets, should_update_health_module=False)
-                demisto.setAssetsLastRun(last_run)
+                send_data_to_xsiam(data=assets, vendor=VENDOR, product='assets', data_type='assets',
+                                   snapshot_id=snapshot_id, items_count=total_assets, should_update_health_module=False)
+                demisto.setAssetsLastRun(new_last_run)
                 demisto.updateModuleHealth({'{data_type}Pulled'.format(data_type='assets'): total_assets})
 
             elif fetch_stage == 'vulnerabilities':
-                vulnerabilities, last_run = fetch_vulnerabilities(client=client, last_run=last_run)
+                vulnerabilities, new_last_run = fetch_vulnerabilities(client=client, last_run=last_run)
                 demisto.debug('sending vulnerabilities to XSIAM.')
                 send_data_to_xsiam(data=vulnerabilities, vendor=VENDOR, product='vulnerabilities', data_type='assets')
-                demisto.setAssetsLastRun(last_run)
+                demisto.setAssetsLastRun(new_last_run)
 
             demisto.debug('finished fetch assets run')
         else:

Packs/qualys/Integrations/Qualysv2/Qualysv2_test.py

@@ -164,11 +164,44 @@ def test_fetch_assets_command(requests_mock):
                     )
     assets, last_run, total_assets, snapshot_id = fetch_assets(client=client, assets_last_run={})
     assert len(assets) == 8
-    assert total_assets == '8'
+    assert total_assets == 8
     assert snapshot_id
     assert last_run['stage'] == 'vulnerabilities'
 
 
+def test_fetch_assets_command_time_out(requests_mock, mocker):
+    """
+    Given:
+    - fetch_assets_command
+    When:
+    - Want to list all existing incidents and got a timeout
+    Then:
+    - Ensure the limit was reduced.
+    """
+    time_out_error = False
+    base_url = 'https://server_url/'
+    with open('./test_data/host_list_detections_raw.xml') as f:
+        assets = f.read()
+    requests_mock.get(f'{base_url}api/2.0/fo/asset/host/vm/detection/'
+                      f'?action=list&truncation_limit={HOST_LIMIT}&vm_scan_date_after='
+                      f'{arg_to_datetime(ASSETS_FETCH_FROM).strftime(ASSETS_DATE_FORMAT)}', exc=requests.exceptions.ReadTimeout)
+    set_last_run_call = mocker.patch.object(demisto, 'setAssetsLastRun')
+
+    client = Client(base_url=base_url,
+                    verify=True,
+                    headers={},
+                    proxy=False,
+                    username='demisto',
+                    password='demisto',
+                    )
+    try:
+        assets, last_run, total_assets, snapshot_id = fetch_assets(client=client, assets_last_run={})
+    except TimeoutError:
+        time_out_error = True
+    assert time_out_error
+    assert set_last_run_call.call_args.args[0].get('limit') == 1000
+
+
 def test_fetch_vulnerabilities_command(requests_mock):
     """
     Given:

Packs/qualys/ReleaseNotes/3_0_6.md

@@ -0,0 +1,4 @@
+
+#### Integrations
+##### Qualys VMDR
+- Fixed an issue where the ***fetch-assets*** process occasionally failed on timeout.

Packs/qualys/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Qualys",
     "description": "Qualys Vulnerability Management let's you create, run, fetch and manage reports, launch and manage vulnerability and compliance scans, and manage the host assets you want to scan for vulnerabilities and compliance",
     "support": "xsoar",
-    "currentVersion": "3.0.5",
+    "currentVersion": "3.0.6",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",