Merge remote-tracking branch 'upstream/master'

.circleci/config.yml

@@ -56,6 +56,31 @@ references:
       NIGHTLY_PARAMETER: << pipeline.parameters.nightly >>
       GCS_MARKET_BUCKET: << pipeline.parameters.gcs_market_bucket >>
 
+  install_neo4j: &install_neo4j
+    run:
+      name: Install Neo4j
+      command: |
+        # workaround for 402 Payment required issue: https://github.com/actions/runner-images/issues/1983
+        sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list
+        sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list.save
+        wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
+        echo 'deb https://debian.neo4j.com stable 4.4' | sudo tee /etc/apt/sources.list.d/neo4j.list
+        sudo apt-get update
+        apt list -a neo4j
+        sudo apt-get install neo4j
+        sudo chown -R circleci /var/log/neo4j
+        sudo chown -R circleci /var/lib/neo4j
+        sudo chown -R circleci /etc/neo4j
+        mkdir -p /var/lib/neo4j/plugins
+        wget -O /var/lib/neo4j/plugins/apoc-4.4.0.8-all.jar https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/4.4.0.8/apoc-4.4.0.8-all.jar
+        neo4j_conf_file="/etc/neo4j/neo4j.conf"
+        sudo echo "dbms.security.procedures.unrestricted=apoc.*" >> $neo4j_conf_file
+        sudo echo "dbms.security.procedures.allowlist=apoc.*" >> $neo4j_conf_file
+        apoc_conf_file="/etc/neo4j/apoc.conf"
+        sudo echo "apoc.export.file.enabled=true" > $apoc_conf_file
+        sudo echo "apoc.import.file.enabled=true" >> $apoc_conf_file
+        sudo echo "apoc.import.file.use_neo4j_config=true" >> $apoc_conf_file
+        neo4j-admin set-initial-password test
 
   container_config: &container_config
     docker:
@@ -168,6 +193,10 @@ references:
           exit 0
         fi
 
+        neo4j start
+        # poll for neo4j status until available
+        while ! neo4j status; do sleep 1; done
+
         ./Tests/scripts/linters_runner.sh
         ./Tests/scripts/validate.sh
 
@@ -228,32 +257,6 @@ references:
           ./Tests/scripts/sdk_pylint_check.sh
         fi
 
-  create_id_set: &create_id_set
-    run:
-      name: Create ID Set
-      when: always
-      command: |
-        demisto-sdk create-id-set -o ./Tests/id_set.json --fail-duplicates
-        cp ./Tests/id_set.json $CIRCLE_ARTIFACTS
-
-  merge_public_and_private_id_sets: &merge_public_and_private_id_sets
-    run:
-      name: Merge public and private ID sets
-      when: always
-      command: |
-        if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then
-            echo "Skipping, Should not run on contributor's branch."
-            exit 0
-        fi
-
-        # Download private ID set
-        gsutil cp gs://marketplace-dist/content/private_id_set.json $CIRCLE_ARTIFACTS/unified_id_set.json
-        echo "successfully downloaded private ID set"
-
-        # Merge public and private ID sets
-        demisto-sdk merge-id-sets -i1 ./Tests/id_set.json -i2 $CIRCLE_ARTIFACTS/unified_id_set.json -o $CIRCLE_ARTIFACTS/unified_id_set.json
-        echo "successfully merged public and private ID sets"
-
   get_contribution_pack: &get_contribution_pack
     when:
       condition: << pipeline.parameters.contrib_branch >>
@@ -322,6 +325,7 @@ jobs:
       - *remote_docker
       - *restore_cache
       - *prepare_environment
+      - *install_neo4j
       - *infrastructure_testing
       - *run_unit_testing_and_lint
       - *generate_coverage_reports
@@ -331,15 +335,14 @@ jobs:
 
   Run Validations:
     <<: *container_config
-    resource_class: medium
+    resource_class: large
     <<: *environment
     steps:
       - *attach_workspace
       - *restore_cache
       - *prepare_environment
+      - *install_neo4j
       - *secrets
-      - *create_id_set
-      - *merge_public_and_private_id_sets
       - *validate_files_and_yaml
       - run:
           name: Spell Checks
@@ -406,5 +409,4 @@ workflows:
     # will initiate when using the trigger script.
     when: << pipeline.parameters.nightly >>
     jobs:
-      *nightly_jobs
-
+      *nightly_jobs

.github/workflows/auto-merge-dockers.yml

@@ -1,7 +1,8 @@
 name: Docker Update Auto Merge
 on:
-  schedule:
-    - cron: '30 10 * * *'
+  pull_request:
+    types: [opened, edited, synchronize, assigned]
+
 
 permissions:
   pull-requests: write
@@ -10,7 +11,7 @@ permissions:
 jobs:
   auto_approve_and_merge:
     runs-on: ubuntu-latest
-    if: startsWith(github.head_ref, 'demisto/') && github.repository == 'demisto/content' && contains(github.event.pull_request.labels.*.name, 'docker') && contains(github.event.pull_request.labels.*.name, 'docker-image-auto-update')
+    if: startsWith(github.head_ref, 'demisto/') && github.repository == 'demisto/content' && contains(github.event.pull_request.labels.*.name, 'docker-image-auto-update')
     steps:
       - name: Print out context
         run: |
@@ -22,8 +23,11 @@ jobs:
           PR_URL: ${{github.event.pull_request.html_url}}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          echo "Approving and merging"
+          echo "Approving the pull request"
           gh pr review --approve "$PR_URL"
+          echo "Waiting tii all the checks are done"
+          gh pr checks --required --interval 60 --watch "$PR_URL"
+          echo "Merging the pull request"
           gh pr merge --squash "$PR_URL"
       - name: Post to a Slack channel
         if: ${{ failure() }}

.github/workflows/check-devcontainer.yml

@@ -5,11 +5,15 @@ on:
       - .devcontainer/**
       - pyproject.toml
       - poetry.lock
+      - .github/workflows/check-devcontainer.yml
   push:
     branches:
       - master
     paths:
       - .devcontainer/**
+      - pyproject.toml
+      - poetry.lock
+      - .github/workflows/check-devcontainer.yml
 jobs:
   Build-Devcontainer:
     permissions:
@@ -20,21 +24,13 @@ jobs:
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - name: Check if pyproject.toml or poetry.lock has been changed
-      id: changed-files-poetry
-      uses: tj-actions/changed-files@v35
-      with:
-        files: |
-          pyproject.toml
-          poetry.lock
     - name: Update .devcontainer.json description
-      if: steps.changed-files-poetry.outputs.any_changed == 'true'
       run: |
         # replace the first line of .devcontainer/devcontainer.json with the new description
         POETRY_LOCK_HASH=${{ hashFiles('poetry.lock') }}
         sed -i "1s/^.*$/\/\/ Development container for Content. Poetry hash $POETRY_LOCK_HASH/" .devcontainer/devcontainer.json 
     - name: commit file
-      if: steps.changed-files-poetry.outputs.any_changed == 'true'
+      if: github.ref == 'refs/heads/master'
       uses: stefanzweifel/git-auto-commit-action@v4
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitlab/ci/global.yml

@@ -142,6 +142,7 @@
   - nvm use default
   - echo "Installing Node Modules" | tee --append $ARTIFACTS_FOLDER/logs/installations.log
   - npm ci --cache .npm --prefer-offline | tee --append $ARTIFACTS_FOLDER/logs/installations.log
+  - npm list --json
   - npm link jsdoc-to-markdown@5.0.3 | tee --append $ARTIFACTS_FOLDER/logs/installations.log  # disable-secrets-detection
   - section_end "Installing node modules"
 
@@ -291,27 +292,6 @@
     - section_start "Look For Secrets"
     - demisto-sdk secrets --post-commit --ignore-entropy
     - section_end "Look For Secrets"
-    - section_start "Create id set"
-    - *create-id-set
-    - section_end "Create id set"
-    - section_start "Merge public and private id sets"
-    - |
-      if [[ $CI_COMMIT_BRANCH =~ pull/[0-9]+ ]]; then
-          echo "Skipping, Should not run on contributor's branch."
-      else
-        gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> $ARTIFACTS_FOLDER/logs/auth.out
-        echo "successfully activated google cloud service account"
-
-        echo "Download private ID set"
-        gsutil cp "gs://marketplace-dist/content/private_id_set.json" $ARTIFACTS_FOLDER/unified_id_set.json
-        echo "successfully downloaded private ID set"
-        gcloud auth revoke $GCS_ARTIFACTS_ACCOUNT_NAME
-
-        echo "Merge public and private ID sets"
-        demisto-sdk merge-id-sets -i1 ./Tests/id_set.json -i2 $ARTIFACTS_FOLDER/unified_id_set.json -o $ARTIFACTS_FOLDER/unified_id_set.json
-        echo "successfully merged public and private ID sets"
-      fi
-    - section_end "Merge public and private id sets"
     - section_start "Copy Tests To Artifact Folder"
     - cp "./Tests/conf.json" "$ARTIFACTS_FOLDER/conf.json"
     - section_end "Copy Tests To Artifact Folder"

.gitlab/ci/on-push.yml

@@ -37,6 +37,23 @@ trigger-private-build:
   - ./Documentation/commonServerDocs.sh
   - section_end "Create Release Notes and Common Server Documentation"
 
+stop-running-pipelines:
+  tags:
+    - gke
+  stage: unittests-and-validations
+  needs: []
+  extends:
+    - .default-job-settings
+  variables:
+    master_branch_name: master
+  rules:
+    - if: '$CI_PIPELINE_SOURCE =~ /^(push)$/ && $CI_COMMIT_BRANCH != $master_branch_name'
+  script:
+    - section_start "Stop running piplines on current branch"
+    - Utils/gitlab/stop_running_pipelines.sh $CI_COMMIT_BRANCH $CI_PIPELINE_ID
+    - section_end "Stop running piplines on current branch"
+
+
 # runs in circle for the on-push flow (because we need to run it there for contributors anyways)
 run-unittests-and-lint:
   extends:
@@ -106,7 +123,7 @@ validate-content-conf:
 
     - echo "Updating the content graph"
     - mkdir $ARTIFACTS_FOLDER/content_graph
-    - demisto-sdk create-content-graph --marketplace $MARKETPLACE_VERSION -o $ARTIFACTS_FOLDER/content_graph  # TODO: return to update with -g when fixed issues
+    - demisto-sdk create-content-graph --marketplace $MARKETPLACE_VERSION -o $ARTIFACTS_FOLDER/content_graph
     - echo "Successfully updated content graph"
 
     - section_end "Create or update content graph"
@@ -146,7 +163,7 @@ validate-content-conf:
         echo "Skipping - not running in INSTANCE_TESTS build"
       else
         [ -n "${NIGHTLY}" ] && IS_NIGHTLY=true || IS_NIGHTLY=false
-        python3 ./Tests/scripts/collect_tests/collect_tests.py -n $IS_NIGHTLY --marketplace "$MARKETPLACE_VERSION" --service_account $GCS_MARKET_KEY --graph true
+        python3 ./Tests/scripts/collect_tests/collect_tests.py -n $IS_NIGHTLY --marketplace "$MARKETPLACE_VERSION" --service_account $GCS_MARKET_KEY --graph true --override_all_packs $OVERRIDE_ALL_PACKS
       fi
     - section_end "Collect Tests"
 
@@ -398,7 +415,7 @@ slack-notify-nightly-build:
     - section_start "Test Modeling Rules"
     - |
       if [[ -s "$ARTIFACTS_FOLDER/modeling_rules_to_test.txt" ]]; then
-        ./Tests/scripts/test_modeling_rules.sh || echo "failed (known issue)"
+        ./Tests/scripts/test_modeling_rules.sh || EXIT_CODE=$?
       else
         echo "No modeling rules were marked for testing during test collection"
       fi
@@ -488,16 +505,20 @@ test-upload-flow:
   rules:
     - if: '$CI_PIPELINE_SOURCE =~ /^(push|contrib)$/'
   variables:
-    CURRENT_BRANCH: "$CI_COMMIT_BRANCH"
     ALL_BUCKETS: "$GCS_MARKET_BUCKET_DEV,$GCS_MARKET_V2_BUCKET_DEV"
   stage: unittests-and-validations
   allow_failure: true
   script:
     - |
-      if ! $(git diff origin/master $CI_COMMIT_BRANCH --name-only | grep -q "Tests/\|Utils/") ; then
-        echo "No upload related files were modified, skipping upload test "
-        exit 0
-      fi
+    - section_start "Checks Whether to Trigger a Test Upload"
+    - SHOULD_SKIP_TEST_UPLOAD=$(./Utils/should_trigger_test_upload.sh)
+    - if [ -z "$SHOULD_SKIP_TEST_UPLOAD" ]; then
+    -   echo "No upload-flow related files were modified, skipping upload test"
+    -   exit 0
+    - fi
+    - echo "Found modified files that should be tested in upload-flow"
+    - section_end "Checks Whether to Trigger a Test Upload"
+
     - section_start "Create Testing Branch"
     - export BRANCH="${CI_COMMIT_BRANCH}-upload_test_branch-$(date +%s)"
     - python3 ./Utils/test_upload_flow/create_test_branch.py -tb $BRANCH -a $ARTIFACTS_FOLDER -g $GITLAB_PUSH_TOKEN

.pre-commit-config.yaml

@@ -9,6 +9,7 @@ repos:
     -   id: check-ast
     -   id: check-toml
     -   id: check-xml
+    -   id: check-merge-conflict
 
 -   repo: https://github.com/abravalheri/validate-pyproject
     rev: v0.10.1

Packs/AHA/Integrations/AHA/AHA.yml

@@ -199,6 +199,6 @@ script:
   script: "-"
   type: python
   subtype: python3
-  dockerimage: demisto/python3:3.10.9.45313
+  dockerimage: demisto/python3:3.10.10.47713
   feed: false
 fromversion: 6.5.0

Packs/AHA/ReleaseNotes/1_0_7.md

@@ -0,0 +1,3 @@
+#### Integrations
+##### Aha
+- Updated the Docker image to: *demisto/python3:3.10.10.47713*.

Packs/AHA/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "AHA",
     "description": "Use the Aha! integration to edit name/title description and status of features in Aha! according to their status in Jira",
     "support": "xsoar",
-    "currentVersion": "1.0.6",
+    "currentVersion": "1.0.7",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/ANYRUN/Integrations/ANYRUN/ANYRUN.yml

@@ -358,6 +358,15 @@ script:
       - '64'
       required: false
       secret: false
+    - name: obj_ext_browser
+      auto: PREDEFINED
+      predefined:
+      - Internet Explorer
+      - Google Chrome
+      - Mozilla Firefox
+      - Opera
+      - Microsoft Edge
+      description: Browser name, used only for "url" type.
     - auto: PREDEFINED
       default: false
       defaultValue: Windows 7
@@ -425,7 +434,7 @@ script:
     - contextPath: ANYRUN.Task.ID
       description: ID of the task created to analyze the submission.
       type: String
-  dockerimage: demisto/python3:3.10.9.40422
+  dockerimage: demisto/python3:3.10.10.47713
   subtype: python3
   isfetch: false
   runonce: false

Packs/ANYRUN/Integrations/ANYRUN/README.md

@@ -345,6 +345,7 @@ Submit a file or url for analysis.
 | opt_network_connect | Network connection state. Possible values are: true, false. Default is true. | Optional | 
 | opt_kernel_heavyevasion | Heavy evasion option. Possible values are: true, false. Default is false. | Optional | 
 | opt_privacy_type | Privacy settings for generated task. Possible values are: owner, bylink, public. Default is owner. | Optional | 
+| obj_ext_browser | Browser name, used only for "url" type. Possible values are: Internet Explorer, Google Chrome, Mozilla Firefox, Opera, Microsoft Edge. | Optional |
 
 
 #### Context Output

Packs/ANYRUN/ReleaseNotes/1_0_12.md

@@ -0,0 +1,4 @@
+#### Integrations
+##### ANY.RUN
+- Added the *obj_ext_browser* argument to the ***anyrun-run-analysis*** command.
+- Updated the Docker image to: *demisto/python3:3.10.10.47713*.

Packs/ANYRUN/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "ANY.RUN",
     "description": "ANY.RUN is a cloud-based sandbox with interactive access.",
     "support": "xsoar",
-    "currentVersion": "1.0.11",
+    "currentVersion": "1.0.12",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",