ci: attempt to divide up longtests (#4631)

The longtests are taking long enough to cause unexpected timeouts. This
attempts to break them up into 4 groups of tests to help make them
more manageable: synchronous tests, scanner/binary checker tests, language 
parser tests and everything else.

Signed-off-by: Terri Oda <terri.oda@intel.com>

.github/workflows/testing.yml

@@ -201,44 +201,8 @@ jobs:
     timeout-minutes: 120
     env:
       LONG_TESTS: 1
-    steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
-        with:
-          egress-policy: block
-          allowed-endpoints: >
-            access.redhat.com:443
-            api.codecov.io:443
-            api.github.com:443
-            archives.fedoraproject.org:443
-            azure.archive.ubuntu.com:80
-            cli.codecov.io:443
-            codecov.io:443
-            curl.se:443
-            epss.cyentia.com:443
-            esm.ubuntu.com:443
-            files.pythonhosted.org:443
-            ftp.fr.debian.org:80
-            github.com:443
-            gitlab.com:443
-            mirror.cveb.in:443
-            mirror.cveb.in:80
-            motd.ubuntu.com:443
-            nvd.nist.gov:443
-            osv-vulnerabilities.storage.googleapis.com:443
-            packages.microsoft.com:443
-            ppa.launchpadcontent.net:443
-            pypi.org:443
-            raw.githubusercontent.com:443
-            release-monitoring.org:443
-            rpmfind.net:443
-            security-tracker.debian.org:443
-            services.nvd.nist.gov:443
-            storage.googleapis.com:443
-            uploader.codecov.io:443
-            www.cisa.gov:443
-            www.sqlite.org:443
 
+    steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
@@ -278,24 +242,7 @@ jobs:
         with:
           path: cache
           key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}
-      - uses: technote-space/get-diff-action@f27caffdd0fb9b13f4fc191c016bb4e0632844af # v6.1.2
-        with:
-          PATTERNS: |
-            cve_bin_tool/*.py
-            cve_bin_tool/data_sources/*.py
-            cve_bin_tool/checkers/*.py
-            test/condensed-downloads/*
-          FILES: |
-            cvedb.py
-            test_scanner.py
-            test_cli.py
-            cli.py
-      - uses: haya14busa/action-cond@94f77f7a80cd666cb3155084e428254fea4281fd # v1.2.1
-        id: git-diff
-        with:
-          cond: ${{ (github.event.inputs.longTests == 'true') || (env.GIT_DIFF != '') }}
-          if_true: '1'
-          if_false: '0'
+
       - name: Install cabextract
         if: env.sbom != 'true'
         run: sudo apt-get update && sudo apt-get install cabextract
@@ -320,19 +267,273 @@ jobs:
           cp -r ~/.cache/cve-bin-tool cache
       - name: Run async tests
         if: env.sbom != 'true'
-        env:
-          LONG_TESTS: ${{ steps.git-diff.outputs.value }}
         run: >
-          pytest --cov --cov-append -n 4 -v --durations=50
+          pytest --cov --cov-append -n 32 --cov-report=xml -v --durations=50
           --ignore=test/test_cli.py
           --ignore=test/test_cvedb.py
           --ignore=test/test_requirements.py
           --ignore=test/test_html.py
           --ignore=test/test_json.py
-      - name: Run synchronous tests
+          --ignore=test/test_scanner.py
+          --ignore=test/test_language_parser.py
+      - name: Upload code coverage to codecov
+        if: env.sbom != 'true'
+        uses: codecov/codecov-action@5c47607acb93fed5485fdbf7232e8a31425f672a # v5.0.2
+        with:
+          files: ./coverage.xml
+          flags: longtests
+          name: codecov-umbrella
+          fail_ci_if_error: false
+
+  long_tests_languages:
+    name: Long tests on Python 3.10 (language parsers)
+    permissions:
+      contents: read
+    if: |
+      ! github.event.pull_request.user.login == 'github-actions[bot]' ||
+      ! (
+        startsWith(github.head_ref, 'chore-sbom-py') ||
+        contains(
+          fromJSON('["chore-update-table","chore-precommit-config","chore-spdx-header"]'),
+          github.head_ref
+        )
+      )
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 120
+    env:
+      LONG_TESTS: 1
+
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: "Skip tests if this is an automated sbom job"
+        env:
+          COMMIT_VAR: ${{ startsWith(github.head_ref, 'chore-sbom-py') && github.event.pull_request.user.login == 'github-actions[bot]' }}
+        run: |
+          if ${COMMIT_VAR} == true; then
+            echo "sbom=true" >> $GITHUB_ENV
+            echo "sbom set to true"
+          else
+            echo "sbom=false" >> $GITHUB_ENV
+            echo "sbom set to false"
+          fi
+
+      - name: Get date
+        id: get-date
+        run: |
+          echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+          echo "yesterday=$(/bin/date -d "-1 day" -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+      - name: Print Cache Keys
+        run: |
+          echo "Today's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}"
+          echo "Yesterday's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}"
+      - name: Get today's cached database
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        id: todays-cache
+        with:
+          path: cache
+          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}
+      - name: Get yesterday's cached database if today's is not available
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        if: steps.todays-cache.outputs.cache-hit != 'true'
+        with:
+          path: cache
+          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}
+
+      - name: Install cve-bin-tool
+        if: env.sbom != 'true'
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools
+          python -m pip install --upgrade wheel
+          python -m pip install --upgrade -r dev-requirements.txt
+          python -m pip install --editable .
+      - name: Try single CLI run of tool
+        if: env.sbom != 'true'
+        run: |
+          [[ -e cache ]] && mkdir -p .cache && mv cache ~/.cache/cve-bin-tool
+          NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets/test-kerberos-5-1.15.1.out
+          cp -r ~/.cache/cve-bin-tool cache
+      - name: Run language scanner tests
+        if: env.sbom != 'true'
+        run: >
+          pytest --cov --cov-append -n 32 --cov-report=xml -v --durations=50
+          test/test_language_scanner.py 
+      - name: Upload code coverage to codecov
+        if: env.sbom != 'true'
+        uses: codecov/codecov-action@5c47607acb93fed5485fdbf7232e8a31425f672a # v5.0.2
+        with:
+          files: ./coverage.xml
+          flags: longtests
+          name: codecov-umbrella
+          fail_ci_if_error: false
+
+  long_tests_scanners:
+    name: Long tests on Python 3.10 (scanners)
+    permissions:
+      contents: read
+    if: |
+      ! github.event.pull_request.user.login == 'github-actions[bot]' ||
+      ! (
+        startsWith(github.head_ref, 'chore-sbom-py') ||
+        contains(
+          fromJSON('["chore-update-table","chore-precommit-config","chore-spdx-header"]'),
+          github.head_ref
+        )
+      )
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 120
+    env:
+      LONG_TESTS: 1
+
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: "Skip tests if this is an automated sbom job"
+        env:
+          COMMIT_VAR: ${{ startsWith(github.head_ref, 'chore-sbom-py') && github.event.pull_request.user.login == 'github-actions[bot]' }}
+        run: |
+          if ${COMMIT_VAR} == true; then
+            echo "sbom=true" >> $GITHUB_ENV
+            echo "sbom set to true"
+          else
+            echo "sbom=false" >> $GITHUB_ENV
+            echo "sbom set to false"
+          fi
+
+      - name: Get date
+        id: get-date
+        run: |
+          echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+          echo "yesterday=$(/bin/date -d "-1 day" -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+      - name: Print Cache Keys
+        run: |
+          echo "Today's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}"
+          echo "Yesterday's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}"
+      - name: Get today's cached database
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        id: todays-cache
+        with:
+          path: cache
+          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}
+      - name: Get yesterday's cached database if today's is not available
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        if: steps.todays-cache.outputs.cache-hit != 'true'
+        with:
+          path: cache
+          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}
+
+      - name: Install cve-bin-tool
+        if: env.sbom != 'true'
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools
+          python -m pip install --upgrade wheel
+          python -m pip install --upgrade -r dev-requirements.txt
+          python -m pip install --editable .
+      - name: Try single CLI run of tool
+        if: env.sbom != 'true'
+        run: |
+          [[ -e cache ]] && mkdir -p .cache && mv cache ~/.cache/cve-bin-tool
+          NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets/test-kerberos-5-1.15.1.out
+          cp -r ~/.cache/cve-bin-tool cache
+      - name: Run binary scanner tests
         if: env.sbom != 'true'
+        run: >
+          pytest --cov --cov-append -n 32 --cov-report=xml -v --durations=50
+          test/test_scanner.py
+      - name: Upload code coverage to codecov
+        if: env.sbom != 'true'
+        uses: codecov/codecov-action@5c47607acb93fed5485fdbf7232e8a31425f672a # v5.0.2
+        with:
+          files: ./coverage.xml
+          flags: longtests
+          name: codecov-umbrella
+          fail_ci_if_error: false
+
+  long_tests_sync:
+    name: Long tests on Python 3.10 (synchronous)
+    permissions:
+      contents: read
+    if: |
+      ! github.event.pull_request.user.login == 'github-actions[bot]' ||
+      ! (
+        startsWith(github.head_ref, 'chore-sbom-py') ||
+        contains(
+          fromJSON('["chore-update-table","chore-precommit-config","chore-spdx-header"]'),
+          github.head_ref
+        )
+      )
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 120
+    env:
+      LONG_TESTS: 1
+
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: "Skip tests if this is an automated sbom job"
         env:
-          LONG_TESTS: ${{ steps.git-diff.outputs.value }}
+          COMMIT_VAR: ${{ startsWith(github.head_ref, 'chore-sbom-py') && github.event.pull_request.user.login == 'github-actions[bot]' }}
+        run: |
+          if ${COMMIT_VAR} == true; then
+            echo "sbom=true" >> $GITHUB_ENV
+            echo "sbom set to true"
+          else
+            echo "sbom=false" >> $GITHUB_ENV
+            echo "sbom set to false"
+          fi
+
+      - name: Get date
+        id: get-date
+        run: |
+          echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+          echo "yesterday=$(/bin/date -d "-1 day" -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+      - name: Print Cache Keys
+        run: |
+          echo "Today's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}"
+          echo "Yesterday's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}"
+      - name: Get today's cached database
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        id: todays-cache
+        with:
+          path: cache
+          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}
+      - name: Get yesterday's cached database if today's is not available
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        if: steps.todays-cache.outputs.cache-hit != 'true'
+        with:
+          path: cache
+          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}
+
+      - name: Install pdftotext, reportlab and cve-bin-tool
+        if: env.sbom != 'true'
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools
+          python -m pip install --upgrade wheel
+          python -m pip install --upgrade -r dev-requirements.txt
+          python -m pip install --editable .
+      - name: Try single CLI run of tool
+        if: env.sbom != 'true'
+        run: |
+          [[ -e cache ]] && mkdir -p .cache && mv cache ~/.cache/cve-bin-tool
+          NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets/test-kerberos-5-1.15.1.out
+          cp -r ~/.cache/cve-bin-tool cache
+      - name: Run synchronous tests
+        if: env.sbom != 'true'
         run: >
           pytest -v --cov --cov-append --cov-report=xml --durations=50
           test/test_cli.py
@@ -346,6 +547,7 @@ jobs:
           name: codecov-umbrella
           fail_ci_if_error: false
 
+
   linux-mayfail:
     name: Tests that may fail due to network or HTML
     permissions: