Merge branch 'contrib/PaloAltoNetworks_EXPANDR-9174' into EXPANDR-9174

.github/CODEOWNERS

@@ -80,9 +80,14 @@ Packs/ApiModules/Scripts/DemistoClassApiModule/DemistoClassApiModule.py @dantavo
 
 # ML scripts
 /Packs/ML/ @jlevypaloalto
-/Packs/PhishingURL/Scripts/DBotPredictURLPhishing/ @jlevypaloalto
+/Packs/PhishingURL/ @jlevypaloalto
+/Packs/Phishing/Scripts/PhishingDedupPreprocessingRule @jlevypaloalto
+/Packs/Phishing/Scripts/FindDuplicateEmailIncidents @jlevypaloalto
+/Packs/Base/Scripts/DrawRelatedIncidentsCanvas@jlevypaloalto
+/Packs/Campaign/Scripts/FindEmailCampaign/ @jlevypaloalto
 /Packs/CortexXDR/Scripts/DBotGroupXDRIncidents/ @jlevypaloalto
 /Packs/CommonScripts/Scripts/DBotUpdateLogoURLPhishing/ @jlevypaloalto
+/Packs/Base/Scripts/FindSimilarIncidentsByText/ @jlevypaloalto
 /Packs/Base/Scripts/DBotTrainTextClassifierV2/ @jlevypaloalto
 /Packs/Base/Scripts/DBotShowClusteringModelInfo/ @jlevypaloalto
 /Packs/Base/Scripts/DBotPredictPhishingWords/ @jlevypaloalto

.github/content_roles.json

@@ -4,8 +4,8 @@
         "samuelFain",
         "israelpoli"
     ],
-    "CONTRIBUTION_TL": "thefrieddan1",
-    "CONTRIBUTION_SECURITY_REVIEWER": "ssokolovich",
+    "CONTRIBUTION_TL": "DeanArbel",
+    "CONTRIBUTION_SECURITY_REVIEWER": ["efelmandar", "melamedbn"],
     "ON_CALL_DEVS": [
         "skidorball",
         "ayousef"

.github/github_workflow_scripts/handle_external_pr.py

@@ -124,6 +124,9 @@ def determine_random_reviewer(potential_reviewers: list[str], repo: Repository)
     Returns:
         str: The github username to assign to a PR
     """
+    if len(potential_reviewers) == 1:
+        print(f'There is only 1 potential reviewer {potential_reviewers}')
+        return potential_reviewers[0]
     label_to_consider = 'contribution'
     pulls = repo.get_pulls(state='OPEN')
     assigned_prs_per_potential_reviewer = {reviewer: 0 for reviewer in potential_reviewers}
@@ -434,6 +437,7 @@ def find_reviewer_to_assign(content_repo: Repository, pr: PullRequest, pr_number
     - content_repo - the content repository
     - pr - current new PR
     - pr_number - number of current_pr
+    - content_reviewers - the list of content reviewers
 
     Returns:
     - Reviewer to assign
@@ -549,6 +553,10 @@ def main():
 
     # Add a security architect reviewer if the PR contains security content items
     if is_requires_security_reviewer(pr_files):
+        if isinstance(security_reviewer, list):
+            security_reviewer = determine_random_reviewer(security_reviewer, content_repo)
+        # else security_reviewer is a string of a single reviewer, just add it to the list of reviewers
+        print(f'The selected security reviewer {security_reviewer}')
         reviewers.append(security_reviewer)
         pr.add_to_assignees(security_reviewer)
         pr.add_to_labels(SECURITY_LABEL)

.github/github_workflow_scripts/utils.py

@@ -202,8 +202,8 @@ def get_content_reviewers(content_roles: dict[str, Any]) -> tuple[list[str], str
             print(f"'{CONTRIBUTION_REVIEWERS_KEY}' is not an array. Terminating...")  # noqa: T201
             sys.exit(1)
 
-        if not isinstance(security_reviewer, str) or not security_reviewer:
-            print(f"'{CONTRIBUTION_SECURITY_REVIEWER_KEY}' is not a string. Terminating...")  # noqa: T201
+        if not isinstance(security_reviewer, list) or not security_reviewer:
+            print(f"'{CONTRIBUTION_SECURITY_REVIEWER_KEY}' is not a list. Terminating...")  # noqa: T201
             sys.exit(1)
 
         if not isinstance(tim_reviewer, str) or not tim_reviewer:

.github/github_workflow_scripts/utils_test.py

@@ -137,11 +137,11 @@ def test_existing_env_var_with_default(self, monkeypatch):
     [
         ({
             CONTRIBUTION_REVIEWERS_KEY: ["cr1", "cr2", "cr3", "cr4"],
-            CONTRIBUTION_SECURITY_REVIEWER_KEY: "sr1",
+            CONTRIBUTION_SECURITY_REVIEWER_KEY: ["sr1"],
             TIM_REVIEWER_KEY: "tr1",
             "CONTRIBUTION_TL": "tl1",
             "ON_CALL_DEVS": ["ocd1", "ocd2"]
-        }, ["cr1", "cr2", "cr3", "cr4"], "sr1", "tr1")
+        }, ["cr1", "cr2", "cr3", "cr4"], ["sr1"], "tr1")
     ]
 )
 def test_get_content_reviewers(
@@ -169,6 +169,43 @@ def test_get_content_reviewers(
     assert actual_tim_reviewer == expected_tim_reviewer
 
 
+@pytest.mark.parametrize(
+    'content_roles,expected_content_reviewers,expected_security_reviewer, expected_tim_reviewer',
+    [
+        ({
+            CONTRIBUTION_REVIEWERS_KEY: ["cr1", "cr2", "cr3", "cr4"],
+            CONTRIBUTION_SECURITY_REVIEWER_KEY: ["sr1", "sr2"],
+            TIM_REVIEWER_KEY: "tr1",
+            "CONTRIBUTION_TL": "tl1",
+            "ON_CALL_DEVS": ["ocd1", "ocd2"]
+        }, ["cr1", "cr2", "cr3", "cr4"], ["sr1", "sr2"], "tr1")
+    ]
+)
+def test_get_content_reviewers_multiple_security(
+    content_roles: dict[str, Any],
+    expected_content_reviewers: list[str],
+    expected_security_reviewer: str,
+    expected_tim_reviewer: str
+):
+    """
+    Test retrieval of content and security reviewers.
+
+    Given:
+        - A ``dict[str, Any]``
+
+    When:
+        - 4 content reviewers and 1 security reviewers provided
+
+    Then:
+        - 4 content reviewers and 1 security reviewer added
+    """
+
+    actual_content_reviewers, actual_security_reviewer, actual_tim_reviewer = get_content_reviewers(content_roles)
+    assert actual_content_reviewers == expected_content_reviewers
+    assert actual_security_reviewer == expected_security_reviewer
+    assert actual_tim_reviewer == expected_tim_reviewer
+
+
 @pytest.mark.parametrize(
     'content_roles',
     [

.github/workflows/update-demisto-sdk-version.yml

@@ -10,10 +10,6 @@ on:
         description: The SDK release version
         required: true
         type: string
-      release_changes:
-        description: The SDK release changes
-        required: true
-        type: string
       is_draft:
         description: Is draft pull request
         required: false
@@ -42,7 +38,7 @@ jobs:
       with:
         python-version: '3.10'
         cache: 'poetry'
-        
+
     - name: Create branch
       run: |
         git config --global user.email "bot@demisto.com"
@@ -65,10 +61,10 @@ jobs:
         git push origin ${{ inputs.release_version }}
         if ${{ inputs.is_draft == true }}; then
           echo "creating draft release pull request"
-          gh pr create -B master -H ${{ inputs.release_version }} --title "demisto-sdk-release ${{ inputs.release_version }}" --body "${{ inputs.release_changes }}" --reviewer ${{ inputs.reviewer }} -l "docs-approved" --draft
+          gh pr create -B master -H ${{ inputs.release_version }} --title "demisto-sdk-release ${{ inputs.release_version }}" --body "Initial body, will be replaced." --reviewer ${{ inputs.reviewer }} -l "docs-approved" --draft
         else
           echo "creating release pull request"
-          gh pr create -B master -H ${{ inputs.release_version }} --title "demisto-sdk-release ${{ inputs.release_version }}" --body "${{ inputs.release_changes }}" --reviewer ${{ inputs.reviewer }} -l "docs-approved"
+          gh pr create -B master -H ${{ inputs.release_version }} --title "demisto-sdk-release ${{ inputs.release_version }}" --body "Initial body, will be replaced." --reviewer ${{ inputs.reviewer }} -l "docs-approved"
         fi
       env:
         GH_TOKEN: ${{ secrets.CONTENTBOT_GH_ADMIN_TOKEN }}

Packs/Base/.secrets-ignore

@@ -3,8 +3,10 @@ FE80:0000:0000:0000:0202:B3FF:FE1E:8329
 2001:0db8:85a3:1337:0000:8a2e:0370:7334
 2001:db8:a0b:12f0::1
 1.1.1.1
+1.1.1.2
+1.1.1.3
+1.1.1.4
 user@example.com
-8.8.8.8
 192.168.1.1
 192.168.1.12
 6.6.6.6
@@ -156,4 +158,5 @@ ff02::1:ff00:1
 2001:db8:1:0:0:0:0:1
 2001:0db8:0000:0000:0000:0000:0000:0001
 ff02:1::ff00:1
-192.168.1.1:8080
+192.168.1.1:8080
+45

Packs/Base/ReleaseNotes/1_34_31.md

@@ -0,0 +1,21 @@
+
+#### Scripts
+
+##### DBotTrainTextClassifierV2
+
+- Updated the Docker image to: *demisto/ml:1.0.0.105874*.
+##### DBotFindSimilarIncidentsByIndicators
+
+- Updated the Docker image to: *demisto/ml:1.0.0.105874*.
+##### GetMLModelEvaluation
+
+- Updated the Docker image to: *demisto/ml:1.0.0.105874*.
+##### DBotPredictPhishingWords
+
+- Updated the Docker image to: *demisto/ml:1.0.0.105874*.
+##### DBotFindSimilarIncidents
+
+- Updated the Docker image to: *demisto/ml:1.0.0.105874*.
+##### DBotPreProcessTextData
+
+- Updated the Docker image to: *demisto/ml:1.0.0.105874*.

Packs/Base/ReleaseNotes/1_34_32.md

@@ -0,0 +1,6 @@
+
+#### Scripts
+
+##### CommonServerPython
+
+- Enhance ***send_data_to_xsiam*** to support custom snapshot IDs in chunks.

Packs/Base/ReleaseNotes/1_34_33.md

@@ -0,0 +1,7 @@
+
+#### Scripts
+
+##### DBotTrainClustering
+
+- Code refactor and general improvements.
+- Updated the Docker image to: *demisto/mlclustering:1.0.0.105775*.

Packs/Base/Scripts/CommonServerPython/CommonServerPython.py

@@ -11811,7 +11811,7 @@ def has_passed_time_threshold(timestamp_str, seconds_threshold):
 
 def send_data_to_xsiam(data, vendor, product, data_format=None, url_key='url', num_of_attempts=3,
                        chunk_size=XSIAM_EVENT_CHUNK_SIZE, data_type=EVENTS, should_update_health_module=True,
-                       add_proxy_to_request=False):
+                       add_proxy_to_request=False, snapshot_id='', items_count=None):
     """
     Send the supported fetched data types into the XDR data-collector private api.
 
@@ -11849,6 +11849,12 @@ def send_data_to_xsiam(data, vendor, product, data_format=None, url_key='url', n
     :type add_proxy_to_request: ``bool``
     :param add_proxy_to_request: whether to add proxy to the send evnets request.
 
+    :type snapshot_id: ``str``
+    :param snapshot_id: the snapshot id.
+
+    :type items_count: ``str``
+    :param items_count: the asset snapshot items count.
+
     :return: None
     :rtype: ``None``
     """
@@ -11858,7 +11864,8 @@ def send_data_to_xsiam(data, vendor, product, data_format=None, url_key='url', n
     calling_context = demisto.callingContext.get('context', {})
     instance_name = calling_context.get('IntegrationInstance', '')
     collector_name = calling_context.get('IntegrationBrand', '')
-    items_count = len(data) if isinstance(data, list) else 1
+    if not items_count:
+        items_count = len(data) if isinstance(data, list) else 1
     if data_type not in DATA_TYPES:
         demisto.debug("data type must be one of these values: {types}".format(types=DATA_TYPES))
         return
@@ -11899,7 +11906,9 @@ def send_data_to_xsiam(data, vendor, product, data_format=None, url_key='url', n
         'collector-type': ASSETS if data_type == ASSETS else EVENTS
     }
     if data_type == ASSETS:
-        headers['snapshot-id'] = str(round(time.time() * 1000))
+        if not snapshot_id:
+            snapshot_id = str(round(time.time() * 1000))
+        headers['snapshot-id'] = instance_name + snapshot_id
         headers['total-items-count'] = str(items_count)
 
     header_msg = 'Error sending new {data_type} into XSIAM.\n'.format(data_type=data_type)
@@ -12034,7 +12043,7 @@ def parse_json_string(json_string):
     try:
         data = json.loads(json_string)
         return data
-    except json.JSONDecodeError as error: # type: ignore[attr-defined]
+    except json.JSONDecodeError as error:  # type: ignore[attr-defined]
         demisto.error("Error decoding JSON: {error}".format(error=error))
         return {}
 
@@ -12051,6 +12060,7 @@ def get_server_config():
     server_config = body.get('sysConf', {})
     return server_config
 
+
 from DemistoClassApiModule import *     # type:ignore [no-redef]  # noqa:E402
 
 

Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py

@@ -68,7 +68,7 @@
         "instance-name": "test_integration_instance",
         "final-reporting-device": "www.test_url.com",
         "collector-type": "assets",
-        "snapshot-id": "123000",
+        "snapshot-id": "test_integration_instance123000",
         "total-items-count": "2"
 }}
 
@@ -8961,6 +8961,49 @@ def test_send_data_to_xsiam_positive(self, mocker, data_use_case, data_type):
             assert arguments_called['headers']['snapshot-id'] == '123000'
             assert arguments_called['headers']['total-items-count'] == '2'
 
+    @pytest.mark.parametrize('data_type, snapshot_id, items_count, expected', [
+        ('assets', None, None, {'snapshot_id': '123000', 'items_count': '2'}),
+        ('assets', '12345', 25, {'snapshot_id': '12345', 'items_count': '25'})
+    ])
+    def test_send_data_to_xsiam_custom_snapshot_id_and_items_count(self, mocker, data_type, snapshot_id, items_count, expected):
+        """
+        Test the send_data_to_xsiam with and without custom snapshot_id and items_count
+        Given:
+            Case a: no custom snapshot_id and items_count.
+            Case b: custom snapshot_id and items_count.
+
+        When:
+            Case a: Calling the send_assets_to_xsiam function without custom snapshot_id and items_count.
+            Case b: Calling the send_assets_to_xsiam function with custom snapshot_id and items_count.
+
+        Then ensure that:
+            Case a: The headers was set with the default data.
+            Case b: The headers was set with the custom data
+        """
+        if not IS_PY3:
+            return
+
+        from CommonServerPython import BaseClient
+        from requests import Response
+        mocker.patch.object(demisto, 'getLicenseCustomField', side_effect=self.get_license_custom_field_mock)
+        mocker.patch.object(demisto, 'updateModuleHealth')
+        mocker.patch('time.time', return_value=123)
+
+        api_response = Response()
+        api_response.status_code = 200
+        api_response._content = json.dumps({'error': 'false'}).encode('utf-8')
+
+        _http_request_mock = mocker.patch.object(BaseClient, '_http_request', return_value=api_response)
+
+        items = self.test_data['json_assets'][data_type]
+        send_data_to_xsiam(data=items, vendor='some vendor', product='some product', data_type=data_type, snapshot_id=snapshot_id,
+                           items_count=items_count)
+
+        arguments_called = _http_request_mock.call_args[1]
+        assert arguments_called['headers']['collector-type'] == data_type
+        assert arguments_called['headers']['snapshot-id'] == expected['snapshot_id']
+        assert arguments_called['headers']['total-items-count'] == expected['items_count']
+
     @pytest.mark.parametrize('error_msg, data_type', [(None, "events"), ({'error': 'error'}, "events"), ('', "events"),
                                                       ({'error': 'error'}, "assets")])
     def test_send_data_to_xsiam_error_handling(self, mocker, requests_mock, error_msg, data_type):

Packs/Base/Scripts/DBotFindSimilarIncidents/DBotFindSimilarIncidents.yml

@@ -86,7 +86,7 @@ script: '-'
 subtype: python3
 timeout: '0'
 type: python
-dockerimage: demisto/ml:1.0.0.103517
+dockerimage: demisto/ml:1.0.0.105874
 runas: DBotWeakRole
 tests:
 - DBotFindSimilarIncidents-test

Packs/Base/Scripts/DBotFindSimilarIncidentsByIndicators/DBotFindSimilarIncidentsByIndicators.yml

@@ -42,7 +42,7 @@ script: '-'
 subtype: python3
 timeout: '0'
 type: python
-dockerimage: demisto/ml:1.0.0.103517
+dockerimage: demisto/ml:1.0.0.105874
 runas: DBotWeakRole
 tests:
 - DBotFindSimilarIncidentsByIndicators - Test

Packs/Base/Scripts/DBotPredictPhishingWords/DBotPredictPhishingWords.yml

@@ -98,7 +98,7 @@ tags:
 - phishing
 timeout: 60µs
 type: python
-dockerimage: demisto/ml:1.0.0.103517
+dockerimage: demisto/ml:1.0.0.105874
 tests:
 - Create Phishing Classifier V2 ML Test
 fromversion: 5.0.0

Packs/Base/Scripts/DBotPreprocessTextData/DBotPreprocessTextData.yml

@@ -104,7 +104,7 @@ tags:
 - ml
 timeout: 120µs
 type: python
-dockerimage: demisto/ml:1.0.0.103517
+dockerimage: demisto/ml:1.0.0.105874
 tests:
 - Create Phishing Classifier V2 ML Test
 fromversion: 5.0.0