python services

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> Update packages. Add sample test for python evinse Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> Integrate evidence into cdxgen Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
CycloneDX · Oct 30, 2023 · 14ea397 · 14ea397
1 parent 4c2b218
commit 14ea397
Show file tree

Hide file tree

Showing 18 changed files with 405 additions and 154 deletions.
diff --git a/.github/workflows/app-release.yml b/.github/workflows/app-release.yml
@@ -14,7 +14,7 @@ jobs:
     - name: Use Node.js
       uses: actions/setup-node@v3
       with:
-        node-version: 20.5
+        node-version: '21.x'
     - name: Install dependencies
       run: |
         sudo apt-get install -y python3.8 python3.8-dev python3-pip python3-testresources python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev

diff --git a/.github/workflows/dockertests.yml b/.github/workflows/dockertests.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
         java-version: ['19']
     steps:
       - uses: actions/checkout@v4
@@ -22,7 +22,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -68,7 +68,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
         java-version: ['19']
     steps:
       - uses: actions/checkout@v4
@@ -79,7 +79,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:
@@ -108,7 +108,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
         java-version: ['19']
     steps:
       - uses: actions/checkout@v4
@@ -119,7 +119,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x, 18.x, 20.x]
+        node-version: ['16.x', '18.x', '20.x', '21.x']
 
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/python-atom-tests.yml b/.github/workflows/python-atom-tests.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
     steps:
       - uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [18.x]
+        node-version: ['21.x']
         os: ['ubuntu-latest', 'windows-latest']
     runs-on: ${{ matrix.os }}
     steps:
@@ -148,18 +148,21 @@ jobs:
         with:
           repository: 'hoolicorp/java-sec-code'
           path: 'repotests/java-sec-code'
+      - uses: actions/checkout@v3
+        with:
+          repository: 'DefectDojo/django-DefectDojo'
+          path: 'repotests/django-DefectDojo'
       - uses: dtolnay/rust-toolchain@stable
       - name: repotests
         run: |
-          bin/cdxgen.js -p -t js --no-recurse -o bom.json .
-          bin/evinse.js -l js -i bom.json -o bom.evinse.json --with-reachables .
+          bin/cdxgen.js -p -t js --no-recurse -o bom.json --evidence .
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json
           bin/cdxgen.js -p -t java --author foo --author bar repotests/java-sec-code -o bomresults/bom-java-sec-code.json
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --required-only
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --filter postgres --filter json
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --only spring
-          bin/cdxgen.js -p -t java repotests/java-sec-code -o repotests/java-sec-code/bom.json --deep
-          node bin/evinse.js -i repotests/java-sec-code/bom.json -o bomresults/java-sec-code.evinse.json -l java --with-reachables -p repotests/java-sec-code
+          bin/cdxgen.js -p -t java repotests/java-sec-code -o repotests/java-sec-code/bom.json --deep --evidence
+          bin/cdxgen.js -t python repotests/django-DefectDojo -o repotests/django-DefectDojo/bom.json --deep --evidence
           bin/cdxgen.js -p -r -t java repotests/shiftleft-java-example -o bomresults/bom-java.json --generate-key-and-sign
           node bin/evinse.js -i bomresults/bom-java.json -o bomresults/bom-java.evinse.json -l java --with-data-flow -p repotests/shiftleft-java-example
           SBOM_SIGN_ALGORITHM=RS512 SBOM_SIGN_PRIVATE_KEY=bomresults/private.key SBOM_SIGN_PUBLIC_KEY=bomresults/public.key bin/cdxgen.js -p -r -t github repotests/shiftleft-java-example -o bomresults/bom-github.json

diff --git a/README.md b/README.md
@@ -170,10 +170,7 @@ Options:
                                faults to true. Pass --no-validate to disable.
                                                        [boolean] [default: true]
       --evidence               Generate SBOM with evidence for supported languag
-                               es. WIP                [boolean] [default: false]
-      --usages-slices-file     Path for the usages slice file created by atom.
-      --data-flow-slices-file  Path for the data-flow slice file created by atom
-                               .
+                               es.                    [boolean] [default: false]
       --spec-version           CycloneDX Specification version to use. Defaults
                                to 1.5                             [default: 1.5]
       --filter                 Filter components containining this word in purl.

diff --git a/bin/cdxgen.js b/bin/cdxgen.js
@@ -10,10 +10,19 @@ import crypto from "node:crypto";
 import { fileURLToPath } from "node:url";
 import globalAgent from "global-agent";
 import process from "node:process";
-import { printTable, printDependencyTree } from "../display.js";
+import {
+  printCallStack,
+  printOccurrences,
+  printServices,
+  printReachables,
+  printTable,
+  printDependencyTree
+} from "../display.js";
 import { findUpSync } from "find-up";
 import { load as _load } from "js-yaml";
 import { postProcess } from "../postgen.js";
+import { analyzeProject, createEvinseFile, prepareDB } from "../evinser.js";
+import { ATOM_DB } from "../utils.js";
 
 // Support for config files
 const configPath = findUpSync([
@@ -48,7 +57,14 @@ const args = yargs(hideBin(process.argv))
   .env("CDXGEN")
   .option("output", {
     alias: "o",
-    description: "Output file for bom.xml or bom.json. Default bom.json"
+    description: "Output file for bom.xml or bom.json. Default bom.json",
+    default: "bom.json"
+  })
+  .option("evinse-output", {
+    description:
+      "Create bom with evidence as a separate file. Default bom.json",
+    default: "bom.json",
+    hidden: true
   })
   .option("type", {
     alias: "t",
@@ -143,16 +159,29 @@ const args = yargs(hideBin(process.argv))
       "Validate the generated SBOM using json schema. Defaults to true. Pass --no-validate to disable."
   })
   .option("evidence", {
-    hidden: true,
     type: "boolean",
     default: false,
-    description: "Generate SBOM with evidence for supported languages. WIP"
+    description: "Generate SBOM with evidence for supported languages."
+  })
+  .option("deps-slices-file", {
+    description: "Path for the parsedeps slice file created by atom.",
+    default: "deps.slices.json",
+    hidden: true
   })
   .option("usages-slices-file", {
-    description: "Path for the usages slice file created by atom."
+    description: "Path for the usages slices file created by atom.",
+    default: "usages.slices.json",
+    hidden: true
   })
   .option("data-flow-slices-file", {
-    description: "Path for the data-flow slice file created by atom."
+    description: "Path for the data-flow slices file created by atom.",
+    default: "data-flow.slices.json",
+    hidden: true
+  })
+  .option("reachables-slices-file", {
+    description: "Path for the reachables slices file created by atom.",
+    default: "reachables.slices.json",
+    hidden: true
   })
   .option("spec-version", {
     description: "CycloneDX Specification version to use. Defaults to 1.5",
@@ -289,9 +318,6 @@ const checkPermissions = (filePath) => {
   if (options.requiredOnly || options["filter"] || options["only"]) {
     bomNSData = postProcess(bomNSData, options);
   }
-  if (!args.output) {
-    args.output = "bom.json";
-  }
   if (
     args.output &&
     (typeof args.output === "string" || args.output instanceof String)
@@ -458,6 +484,34 @@ const checkPermissions = (filePath) => {
       console.log("Try running the command with -t <type> or -r argument");
     }
   }
+  // Evidence generation
+  if (args.evidence) {
+    const evinseOptions = {
+      _: args._,
+      input: options.output,
+      output: options.evinseOutput,
+      language: options.projectType || "java",
+      dbPath: process.env.ATOM_DB || ATOM_DB,
+      skipMavenCollector: false,
+      force: false,
+      withReachables: options.deep,
+      usagesSlicesFile: options.usagesSlicesFile,
+      dataFlowSlicesFile: options.dataFlowSlicesFile,
+      reachablesSlicesFile: options.reachablesSlicesFile
+    };
+    const dbObjMap = await prepareDB(evinseOptions);
+    if (dbObjMap) {
+      const sliceArtefacts = await analyzeProject(dbObjMap, evinseOptions);
+      const evinseJson = createEvinseFile(sliceArtefacts, evinseOptions);
+      bomNSData.bomJson = evinseJson;
+      if (args.print && evinseJson) {
+        printOccurrences(evinseJson);
+        printCallStack(evinseJson);
+        printReachables(sliceArtefacts);
+        printServices(evinseJson);
+      }
+    }
+  }
   // Perform automatic validation
   if (args.validate) {
     if (!validateBom(bomNSData.bomJson)) {

diff --git a/bin/evinse.js b/bin/evinse.js
@@ -3,9 +3,7 @@
 // Evinse (Evinse Verification Is Nearly SBOM Evidence)
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
-import { join } from "node:path";
 import fs from "node:fs";
-import { homedir, platform as _platform } from "node:os";
 import process from "node:process";
 import { analyzeProject, createEvinseFile, prepareDB } from "../evinser.js";
 import { validateBom } from "../validator.js";
@@ -15,6 +13,7 @@ import {
   printServices,
   printReachables
 } from "../display.js";
+import { ATOM_DB } from "../utils.js";
 import { findUpSync } from "find-up";
 import { load as _load } from "js-yaml";
 
@@ -38,22 +37,6 @@ if (configPath) {
   }
 }
 
-const isWin = _platform() === "win32";
-const isMac = _platform() === "darwin";
-let ATOM_DB = join(homedir(), ".local", "share", ".atomdb");
-if (isWin) {
-  ATOM_DB = join(homedir(), "AppData", "Local", ".atomdb");
-} else if (isMac) {
-  ATOM_DB = join(homedir(), "Library", "Application Support", ".atomdb");
-}
-
-if (!process.env.ATOM_DB && !fs.existsSync(ATOM_DB)) {
-  try {
-    fs.mkdirSync(ATOM_DB, { recursive: true });
-  } catch (e) {
-    // ignore
-  }
-}
 const args = yargs(hideBin(process.argv))
   .env("EVINSE")
   .option("input", {

diff --git a/data/frameworks-list.json b/data/frameworks-list.json
@@ -4,8 +4,8 @@
     "System.ServiceModel",
     "System.Data",
     "spring",
-    "flask",
-    "django",
+    "pkg:pypi/flask",
+    "pkg:pypi/django",
     "beego",
     "chi",
     "echo",
@@ -30,15 +30,33 @@
     "express",
     "knex",
     "vue",
-    "aiohttp",
-    "bottle",
-    "cherrypy",
-    "drt",
-    "falcon",
-    "hug",
-    "pyramid",
-    "sanic",
-    "tornado",
+    "pkg:pypi/aiohttp",
+    "pkg:pypi/bottle",
+    "pkg:pypi/cherrypy",
+    "pkg:pypi/drt",
+    "pkg:pypi/falcon",
+    "pkg:pypi/hug",
+    "pkg:pypi/pyramid",
+    "pkg:pypi/sanic",
+    "pkg:pypi/tornado",
+    "pkg:pypi/fastapi",
+    "pkg:pypi/pyqt",
+    "pkg:pypi/tkinter",
+    "pkg:pypi/kivy",
+    "pkg:pypi/pyside",
+    "pkg:pypi/scikit",
+    "pkg:pypi/tensorflow",
+    "pkg:pypi/pytorch",
+    "pkg:pypi/keras",
+    "pkg:pypi/numpy",
+    "pkg:pypi/scipy",
+    "pkg:pypi/pandas",
+    "pkg:pypi/matplotlib",
+    "pkg:pypi/google-api-core",
+    "pkg:pypi/google-cloud",
+    "pkg:pypi/botocore",
+    "pkg:pypi/boto3",
+    "pkg:pypi/azure",
     "vibora",
     "koa",
     "-sdk",

diff --git a/data/pypi-pkg-aliases.json b/data/pypi-pkg-aliases.json
@@ -553,6 +553,7 @@
   "creole": "python-creole",
   "creoleparser": "creoleparser",
   "crispy-forms": "django-crispy-forms",
+  "crum": "django-crum",
   "cronlog": "python-crontab",
   "crontab": "python-crontab",
   "crypto": "pycryptodome",
@@ -589,6 +590,7 @@
   "djcelery": "django-celery",
   "djkombu": "django-kombu",
   "djorm-pgarray": "djorm-ext-pgarray",
+  "django-filters": "filters-django",
   "dns": "dnspython",
   "docgen": "ansible-docgenerator",
   "docker": "docker-py",
@@ -631,6 +633,7 @@
   "fdpexpect": "pexpect",
   "fedora": "python-fedora",
   "fias": "ailove-django-fias",
+  "fieldsignals": "django-fieldsignals",
   "fiftyone-degrees": "51degrees-mobile-detector",
   "fiftyonedegrees": "51degrees-mobile-detector-v3-wrapper",
   "five": "five.customerize",
@@ -709,6 +712,7 @@
   "igraph": "python-igraph",
   "imdb": "imdbpy",
   "impala": "impyla",
+  "imagekit": "django-imagekit",
   "impersonate": "django-impersonate",
   "inmemorystorage": "ambition-inmemorystorage",
   "ipaddress": "backport-ipaddress",
@@ -845,6 +849,7 @@
   "path": "path.py",
   "patricia": "patricia-trie",
   "paver": "paver",
+  "packageurl": "packageurl-python",
   "peak": "proxytypes",
   "picasso": "anderson.picasso",
   "picklefield": "django-picklefield",
@@ -1057,6 +1062,7 @@
   "slugify": "unicode-slugify",
   "smarkets": "smk-python-sdk",
   "snappy": "ctypes-snappy",
+  "social-core": "social-auth-core",
   "social-django": "social-auth-app-django",
   "socketio": "python-socketio",
   "socketserver": "pies2overrides",