Feature/evinse python (#676)

* python services Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> Update packages. Add sample test for python evinse Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> Integrate evidence into cdxgen Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Bug fix Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Improve error message Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Bump version Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * fedora image for testing Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> Co-authored-by: Jacek Puchta <jacek.puchta@dotdata.com>
CycloneDX · Oct 31, 2023 · 6715e21 · 6715e21
1 parent 3cb6551
commit 6715e21
Show file tree

Hide file tree

Showing 24 changed files with 551 additions and 170 deletions.
diff --git a/.github/workflows/app-release.yml b/.github/workflows/app-release.yml
@@ -14,7 +14,7 @@ jobs:
     - name: Use Node.js
       uses: actions/setup-node@v3
       with:
-        node-version: 20.5
+        node-version: '21.x'
     - name: Install dependencies
       run: |
         sudo apt-get install -y python3.8 python3.8-dev python3-pip python3-testresources python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev

diff --git a/.github/workflows/dockertests.yml b/.github/workflows/dockertests.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
         java-version: ['19']
     steps:
       - uses: actions/checkout@v4
@@ -22,7 +22,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -68,7 +68,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
         java-version: ['19']
     steps:
       - uses: actions/checkout@v4
@@ -79,7 +79,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -108,7 +108,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
         java-version: ['19']
     steps:
       - uses: actions/checkout@v4
@@ -119,7 +119,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x, 18.x, 20.x]
+        node-version: ['16.x', '18.x', '20.x', '21.x']
 
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/npm-release.yml b/.github/workflows/npm-release.yml
@@ -120,3 +120,22 @@ jobs:
         labels: ${{ steps.meta3.outputs.labels }}
         cache-from: type=gha,scope=cdxgen-ppc64
         cache-to: type=gha,mode=max,scope=cdxgen-ppc64
+    - name: Extract metadata (tags, labels) for Docker
+      id: meta4
+      uses: docker/metadata-action@v4
+      with:
+        images: |
+          ghcr.io/cyclonedx/cdxgen-next
+      if: github.ref == 'refs/heads/master'
+    - name: Build and push Docker images
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        file: ci/Dockerfile-fedora
+        platforms: linux/amd64
+        push: true
+        tags: ${{ steps.meta4.outputs.tags }}
+        labels: ${{ steps.meta4.outputs.labels }}
+        cache-from: type=gha,scope=cdxgen-next
+        cache-to: type=gha,mode=max,scope=cdxgen-next
+      if: github.ref == 'refs/heads/master'
diff --git a/.github/workflows/python-atom-tests.yml b/.github/workflows/python-atom-tests.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
     steps:
       - uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
         os: ['ubuntu-latest', 'windows-latest']
     runs-on: ${{ matrix.os }}
     steps:
@@ -148,18 +148,21 @@ jobs:
         with:
           repository: 'hoolicorp/java-sec-code'
           path: 'repotests/java-sec-code'
+      - uses: actions/checkout@v3
+        with:
+          repository: 'DefectDojo/django-DefectDojo'
+          path: 'repotests/django-DefectDojo'
       - uses: dtolnay/rust-toolchain@stable
       - name: repotests
         run: |
-          bin/cdxgen.js -p -t js --no-recurse -o bom.json .
-          bin/evinse.js -l js -i bom.json -o bom.evinse.json --with-reachables .
+          bin/cdxgen.js -p -t js --no-recurse -o bom.json --evidence .
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json
           bin/cdxgen.js -p -t java --author foo --author bar repotests/java-sec-code -o bomresults/bom-java-sec-code.json
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --required-only
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --filter postgres --filter json
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --only spring
-          bin/cdxgen.js -p -t java repotests/java-sec-code -o repotests/java-sec-code/bom.json --deep
-          node bin/evinse.js -i repotests/java-sec-code/bom.json -o bomresults/java-sec-code.evinse.json -l java --with-reachables -p repotests/java-sec-code
+          bin/cdxgen.js -p -t java repotests/java-sec-code -o repotests/java-sec-code/bom.json --deep --evidence
+          bin/cdxgen.js -t python repotests/django-DefectDojo -o repotests/django-DefectDojo/bom.json --deep --evidence
           bin/cdxgen.js -p -r -t java repotests/shiftleft-java-example -o bomresults/bom-java.json --generate-key-and-sign
           node bin/evinse.js -i bomresults/bom-java.json -o bomresults/bom-java.evinse.json -l java --with-data-flow -p repotests/shiftleft-java-example
           SBOM_SIGN_ALGORITHM=RS512 SBOM_SIGN_PRIVATE_KEY=bomresults/private.key SBOM_SIGN_PUBLIC_KEY=bomresults/public.key bin/cdxgen.js -p -r -t github repotests/shiftleft-java-example -o bomresults/bom-github.json

diff --git a/README.md b/README.md
@@ -170,10 +170,7 @@ Options:
                                faults to true. Pass --no-validate to disable.
                                                        [boolean] [default: true]
       --evidence               Generate SBOM with evidence for supported languag
-                               es. WIP                [boolean] [default: false]
-      --usages-slices-file     Path for the usages slice file created by atom.
-      --data-flow-slices-file  Path for the data-flow slice file created by atom
-                               .
+                               es.                    [boolean] [default: false]
       --spec-version           CycloneDX Specification version to use. Defaults
                                to 1.5                             [default: 1.5]
       --filter                 Filter components containining this word in purl.

diff --git a/bin/cdxgen.js b/bin/cdxgen.js
@@ -10,10 +10,19 @@ import crypto from "node:crypto";
 import { fileURLToPath } from "node:url";
 import globalAgent from "global-agent";
 import process from "node:process";
-import { printTable, printDependencyTree } from "../display.js";
+import {
+  printCallStack,
+  printOccurrences,
+  printServices,
+  printReachables,
+  printTable,
+  printDependencyTree
+} from "../display.js";
 import { findUpSync } from "find-up";
 import { load as _load } from "js-yaml";
 import { postProcess } from "../postgen.js";
+import { analyzeProject, createEvinseFile, prepareDB } from "../evinser.js";
+import { ATOM_DB } from "../utils.js";
 
 // Support for config files
 const configPath = findUpSync([
@@ -48,7 +57,14 @@ const args = yargs(hideBin(process.argv))
   .env("CDXGEN")
   .option("output", {
     alias: "o",
-    description: "Output file for bom.xml or bom.json. Default bom.json"
+    description: "Output file for bom.xml or bom.json. Default bom.json",
+    default: "bom.json"
+  })
+  .option("evinse-output", {
+    description:
+      "Create bom with evidence as a separate file. Default bom.json",
+    default: "bom.json",
+    hidden: true
   })
   .option("type", {
     alias: "t",
@@ -143,16 +159,29 @@ const args = yargs(hideBin(process.argv))
       "Validate the generated SBOM using json schema. Defaults to true. Pass --no-validate to disable."
   })
   .option("evidence", {
-    hidden: true,
     type: "boolean",
     default: false,
-    description: "Generate SBOM with evidence for supported languages. WIP"
+    description: "Generate SBOM with evidence for supported languages."
+  })
+  .option("deps-slices-file", {
+    description: "Path for the parsedeps slice file created by atom.",
+    default: "deps.slices.json",
+    hidden: true
   })
   .option("usages-slices-file", {
-    description: "Path for the usages slice file created by atom."
+    description: "Path for the usages slices file created by atom.",
+    default: "usages.slices.json",
+    hidden: true
   })
   .option("data-flow-slices-file", {
-    description: "Path for the data-flow slice file created by atom."
+    description: "Path for the data-flow slices file created by atom.",
+    default: "data-flow.slices.json",
+    hidden: true
+  })
+  .option("reachables-slices-file", {
+    description: "Path for the reachables slices file created by atom.",
+    default: "reachables.slices.json",
+    hidden: true
   })
   .option("spec-version", {
     description: "CycloneDX Specification version to use. Defaults to 1.5",
@@ -289,9 +318,6 @@ const checkPermissions = (filePath) => {
   if (options.requiredOnly || options["filter"] || options["only"]) {
     bomNSData = postProcess(bomNSData, options);
   }
-  if (!args.output) {
-    args.output = "bom.json";
-  }
   if (
     args.output &&
     (typeof args.output === "string" || args.output instanceof String)
@@ -458,6 +484,34 @@ const checkPermissions = (filePath) => {
       console.log("Try running the command with -t <type> or -r argument");
     }
   }
+  // Evidence generation
+  if (args.evidence) {
+    const evinseOptions = {
+      _: args._,
+      input: options.output,
+      output: options.evinseOutput,
+      language: options.projectType || "java",
+      dbPath: process.env.ATOM_DB || ATOM_DB,
+      skipMavenCollector: false,
+      force: false,
+      withReachables: options.deep,
+      usagesSlicesFile: options.usagesSlicesFile,
+      dataFlowSlicesFile: options.dataFlowSlicesFile,
+      reachablesSlicesFile: options.reachablesSlicesFile
+    };
+    const dbObjMap = await prepareDB(evinseOptions);
+    if (dbObjMap) {
+      const sliceArtefacts = await analyzeProject(dbObjMap, evinseOptions);
+      const evinseJson = createEvinseFile(sliceArtefacts, evinseOptions);
+      bomNSData.bomJson = evinseJson;
+      if (args.print && evinseJson) {
+        printOccurrences(evinseJson);
+        printCallStack(evinseJson);
+        printReachables(sliceArtefacts);
+        printServices(evinseJson);
+      }
+    }
+  }
   // Perform automatic validation
   if (args.validate) {
     if (!validateBom(bomNSData.bomJson)) {

diff --git a/bin/evinse.js b/bin/evinse.js
@@ -3,9 +3,7 @@
 // Evinse (Evinse Verification Is Nearly SBOM Evidence)
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
-import { join } from "node:path";
 import fs from "node:fs";
-import { homedir, platform as _platform } from "node:os";
 import process from "node:process";
 import { analyzeProject, createEvinseFile, prepareDB } from "../evinser.js";
 import { validateBom } from "../validator.js";
@@ -15,6 +13,7 @@ import {
   printServices,
   printReachables
 } from "../display.js";
+import { ATOM_DB } from "../utils.js";
 import { findUpSync } from "find-up";
 import { load as _load } from "js-yaml";
 
@@ -38,22 +37,6 @@ if (configPath) {
   }
 }
 
-const isWin = _platform() === "win32";
-const isMac = _platform() === "darwin";
-let ATOM_DB = join(homedir(), ".local", "share", ".atomdb");
-if (isWin) {
-  ATOM_DB = join(homedir(), "AppData", "Local", ".atomdb");
-} else if (isMac) {
-  ATOM_DB = join(homedir(), "Library", "Application Support", ".atomdb");
-}
-
-if (!process.env.ATOM_DB && !fs.existsSync(ATOM_DB)) {
-  try {
-    fs.mkdirSync(ATOM_DB, { recursive: true });
-  } catch (e) {
-    // ignore
-  }
-}
 const args = yargs(hideBin(process.argv))
   .env("EVINSE")
   .option("input", {

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -4,7 +4,7 @@ LABEL maintainer="cyclonedx" \
       org.opencontainers.image.authors="Prabhu Subramanian <prabhu@appthreat.com>" \
       org.opencontainers.image.source="https://github.com/cyclonedx/cdxgen" \
       org.opencontainers.image.url="https://github.com/cyclonedx/cdxgen" \
-      org.opencontainers.image.version="9.8.x" \
+      org.opencontainers.image.version="9.9.x" \
       org.opencontainers.image.vendor="cyclonedx" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cdxgen" \
@@ -17,7 +17,7 @@ ARG SWIFT_BRANCH=swift-5.8-release
 ARG SWIFT_VERSION=swift-5.8-RELEASE
 ARG SWIFT_WEBROOT=https://download.swift.org
 ARG JAVA_VERSION=20.0.2-graalce
-ARG SBT_VERSION=1.9.6
+ARG SBT_VERSION=1.9.7
 ARG MAVEN_VERSION=3.9.4
 ARG GRADLE_VERSION=8.3
 ARG GO_VERSION=1.21.1
@@ -58,7 +58,7 @@ RUN set -e; \
             ;; \
         *) echo >&2 "error: unsupported architecture: '$ARCH_NAME'"; exit 1 ;; \
     esac; \
-    echo -e "[nodejs]\nname=nodejs\nstream=20\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
+    echo -e "[nodejs]\nname=nodejs\nstream=21\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
     && microdnf module enable php ruby -y \
     && microdnf install -y php php-curl php-zip php-bcmath php-json php-pear php-mbstring php-devel make gcc git-core \
         python3.11 python3.11-devel python3.11-pip ruby ruby-devel \

diff --git a/ci/Dockerfile-deno b/ci/Dockerfile-deno
@@ -4,7 +4,7 @@ LABEL maintainer="cyclonedx" \
       org.opencontainers.image.authors="Prabhu Subramanian <prabhu@appthreat.com>" \
       org.opencontainers.image.source="https://github.com/cyclonedx/cdxgen" \
       org.opencontainers.image.url="https://github.com/cyclonedx/cdxgen" \
-      org.opencontainers.image.version="9.8.x" \
+      org.opencontainers.image.version="9.9.x" \
       org.opencontainers.image.vendor="cyclonedx" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cdxgen" \
@@ -17,7 +17,7 @@ ARG SWIFT_BRANCH=swift-5.8-release
 ARG SWIFT_VERSION=swift-5.8-RELEASE
 ARG SWIFT_WEBROOT=https://download.swift.org
 ARG JAVA_VERSION=20.0.2-graalce
-ARG SBT_VERSION=1.9.6
+ARG SBT_VERSION=1.9.7
 ARG MAVEN_VERSION=3.9.4
 ARG GRADLE_VERSION=8.3
 ARG GO_VERSION=1.21.1