Merge branch 'feature/reachable-slice' of https://github.com/CycloneD…

…X/cdxgen into feature/reachable-slice

.github/workflows/repotests.yml

@@ -155,6 +155,8 @@ jobs:
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --required-only
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --filter postgres --filter json
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --only spring
+          bin/cdxgen.js -p -t java repotests/java-sec-code -o repotests/java-sec-code/bom.json --deep
+          node bin/evinse.js -i repotests/java-sec-code/bom.json -o bomresults/java-sec-code.evinse.json -l java --with-reachables -p repotests/java-sec-code
           bin/cdxgen.js -p -r -t java repotests/shiftleft-java-example -o bomresults/bom-java.json --generate-key-and-sign
           node bin/evinse.js -i bomresults/bom-java.json -o bomresults/bom-java.evinse.json -l java --with-data-flow -p repotests/shiftleft-java-example
           SBOM_SIGN_ALGORITHM=RS512 SBOM_SIGN_PRIVATE_KEY=bomresults/private.key SBOM_SIGN_PUBLIC_KEY=bomresults/public.key bin/cdxgen.js -p -r -t github repotests/shiftleft-java-example -o bomresults/bom-github.json

README.md

@@ -377,6 +377,7 @@ cdxgen can retain the dependency tree under the `dependencies` attribute for a s
 | CDX_MAVEN_INCLUDE_TEST_SCOPE | Whether test scoped dependencies should be included from Maven projects, Default: true                                               |
 | ASTGEN_IGNORE_DIRS           | Comma separated list of directories to ignore while analyzing using babel. The environment variable is also used by atom and astgen. |
 | ASTGEN_IGNORE_FILE_PATTERN   | Ignore regex to use                                                                                                                  |
+| PYPI_URL                     | Override pypi url. Default: https://pypi.org/pypi/                                                                                   |
 
 ## Plugins
 

bin/evinse.js

@@ -98,6 +98,12 @@ const args = yargs(hideBin(process.argv))
     default: false,
     type: "boolean"
   })
+  .option("with-reachables", {
+    description:
+      "Enable auto-tagged reachable slicing. Requires SBOM generated with --deep mode.",
+    default: false,
+    type: "boolean"
+  })
   .option("usages-slices-file", {
     description: "Use an existing usages slices file.",
     default: "usages.slices.json"
@@ -106,6 +112,10 @@ const args = yargs(hideBin(process.argv))
     description: "Use an existing data-flow slices file.",
     default: "data-flow.slices.json"
   })
+  .option("reachables-slices-file", {
+    description: "Use an existing reachables slices file.",
+    default: "reachables.slices.json"
+  })
   .option("print", {
     alias: "p",
     type: "boolean",

binary.js

@@ -1,5 +1,11 @@
-import { platform as _platform, arch as _arch, tmpdir } from "node:os";
-import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { platform as _platform, arch as _arch, tmpdir, homedir } from "node:os";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync
+} from "node:fs";
 import { join, dirname, basename } from "node:path";
 import { spawnSync } from "node:child_process";
 import { PackageURL } from "packageurl-js";
@@ -284,6 +290,13 @@ export const getOSPackages = (src) => {
   const allTypes = new Set();
   if (TRIVY_BIN) {
     let imageType = "image";
+    const trivyCacheDir = join(homedir(), ".cache", "trivy");
+    try {
+      mkdirSync(join(trivyCacheDir, "db"), { recursive: true });
+      mkdirSync(join(trivyCacheDir, "java-db"), { recursive: true });
+    } catch (err) {
+      // ignore errors
+    }
     if (existsSync(src)) {
       imageType = "rootfs";
     }
@@ -292,12 +305,17 @@ export const getOSPackages = (src) => {
     const args = [
       imageType,
       "--skip-db-update",
+      "--skip-java-db-update",
       "--offline-scan",
+      "--skip-files",
+      "**/*.jar",
       "--no-progress",
       "--exit-code",
       "0",
       "--format",
       "cyclonedx",
+      "--cache-dir",
+      trivyCacheDir,
       "--output",
       bomJsonFile
     ];

docker.js

@@ -333,7 +333,12 @@ export const parseImageName = (fullImageName) => {
  */
 export const getImage = async (fullImageName) => {
   let localData = undefined;
+  let pullData = undefined;
   const { repo, tag, digest } = parseImageName(fullImageName);
+  let repoWithTag = `${repo}:${tag !== "" ? tag : ":latest"}`;
+  if (repoWithTag.startsWith("library/")) {
+    repoWithTag = repoWithTag.replace("library/", "");
+  }
   // Fetch only the latest tag if none is specified
   if (tag === "" && digest === "") {
     fullImageName = fullImageName + ":latest";
@@ -379,6 +384,14 @@ export const getImage = async (fullImageName) => {
       }
     }
   }
+  try {
+    localData = await makeRequest(`images/${repoWithTag}/json`);
+    if (localData) {
+      return localData;
+    }
+  } catch (err) {
+    // ignore
+  }
   try {
     localData = await makeRequest(`images/${repo}/json`);
   } catch (err) {
@@ -397,7 +410,7 @@ export const getImage = async (fullImageName) => {
     }
     // If the data is not available locally
     try {
-      const pullData = await makeRequest(
+      pullData = await makeRequest(
         `images/create?fromImage=${fullImageName}`,
         "POST"
       );
@@ -415,15 +428,42 @@ export const getImage = async (fullImageName) => {
         return undefined;
       }
     } catch (err) {
-      // continue regardless of error
+      try {
+        if (DEBUG_MODE) {
+          console.log(`Re-trying the pull with the name ${repoWithTag}.`);
+        }
+        pullData = await makeRequest(
+          `images/create?fromImage=${repoWithTag}`,
+          "POST"
+        );
+      } catch (err) {
+        // continue regardless of error
+      }
     }
     try {
       if (DEBUG_MODE) {
-        console.log(`Trying with ${repo}`);
+        console.log(`Trying with ${repoWithTag}`);
+      }
+      localData = await makeRequest(`images/${repoWithTag}/json`);
+      if (localData) {
+        return localData;
       }
-      localData = await makeRequest(`images/${repo}/json`);
     } catch (err) {
       try {
+        if (DEBUG_MODE) {
+          console.log(`Trying with ${repo}`);
+        }
+        localData = await makeRequest(`images/${repo}/json`);
+        if (localData) {
+          return localData;
+        }
+      } catch (err) {
+        // continue regardless of error
+      }
+      try {
+        if (DEBUG_MODE) {
+          console.log(`Trying with ${fullImageName}`);
+        }
         localData = await makeRequest(`images/${fullImageName}/json`);
       } catch (err) {
         // continue regardless of error

docs/ADVANCED.md

@@ -133,13 +133,18 @@ Options:
                                  directory. Useful to improve the recall for cal
                                  lstack evidence.     [boolean] [default: false]
       --annotate                 Include contents of atom slices as annotations
-                                                       [boolean] [default: true]
+                                                      [boolean] [default: false]
       --with-data-flow           Enable inter-procedural data-flow slicing.
                                                       [boolean] [default: false]
+      --with-reachables          Enable auto-tagged reachable slicing. Requires
+                                 SBOM generated with --deep mode.
+                                                      [boolean] [default: false]
       --usages-slices-file       Use an existing usages slices file.
                                                  [default: "usages.slices.json"]
       --data-flow-slices-file    Use an existing data-flow slices file.
                                               [default: "data-flow.slices.json"]
+      --reachables-slices-file   Use an existing reachables slices file.
+                                             [default: "reachables.slices.json"]
   -p, --print                    Print the evidences as table          [boolean]
       --version                  Show version number                   [boolean]
   -h                             Show help                             [boolean]
@@ -151,18 +156,38 @@ To generate an SBOM with evidence for a java project.
 evinse -i bom.json -o bom.evinse.json <path to the application>
 ```
 
-By default, only occurrence evidences are determined by creating usages slices. To generate callstack evidence, pass `--with-data-flow`
+By default, only occurrence evidences are determined by creating usages slices. To generate callstack evidence, pass either `--with-data-flow` or `--with-reachables`.
+
+#### Reachability-based callstack evidence
+
+atom supports reachability-based slicing for Java applications. Two necessary prerequisites for this slicing mode are that the input SBOM must be generated in deep mode (with --deep argument) and must be placed within the application directory.
+
+```shell
+cd <path to the application>
+cdxgen -t java --deep -o bom.json .
+evinse -i bom.json -o bom.evinse.json --with-reachables .
+```
+
+This is because
+
+#### Data Flow based slicing
+
+Often reachability cannot be computed reliably due to the presence of wrapper libraries or mitigating layers. In such cases, data-flow based slicing can be used to compute callstack using a reverse reachability algorithm. This is however a time and resource-consuming operation and might even require atom to be run externally in [java mode](https://cyclonedx.github.io/cdxgen/#/ADVANCED?id=use-atom-in-java-mode).
 
 ```shell
 evinse -i bom.json -o bom.evinse.json --with-data-flow <path to the application>
 ```
 
+#### Performance tuning
+
 To improve performance, you can cache the generated usages and data-flow slices file along with the bom file.
 
 ```shell
 evinse -i bom.json -o bom.evinse.json --usages-slices-file usages.json --data-flow-slices-file data-flow.json --with-data-flow <path to the application>
 ```
 
+#### Other languages
+
 For JavaScript or TypeScript projects, pass `-l javascript`.
 
 ```shell

docs/ENV.md

@@ -36,3 +36,4 @@ The following environment variables are available to configure the bom generatio
 | CDX_MAVEN_INCLUDE_TEST_SCOPE | Whether test scoped dependencies should be included from Maven projects, Default: true                                               |
 | ASTGEN_IGNORE_DIRS           | Comma separated list of directories to ignore while analyzing using babel. The environment variable is also used by atom and astgen. |
 | ASTGEN_IGNORE_FILE_PATTERN   | Ignore regex to use                                                                                                                  |
+| PYPI_URL                     | Override pypi url. Default: https://pypi.org/pypi/                                                                                   |

evinser.js

@@ -260,8 +260,10 @@ export const analyzeProject = async (dbObjMap, options) => {
   const language = options.language;
   let usageSlice = undefined;
   let dataFlowSlice = undefined;
+  let reachablesSlice = undefined;
   let usagesSlicesFile = undefined;
   let dataFlowSlicesFile = undefined;
+  let reachablesSlicesFile = undefined;
   let dataFlowFrames = {};
   let servicesMap = {};
   let retMap = {};
@@ -330,10 +332,36 @@ export const analyzeProject = async (dbObjMap, options) => {
       purlImportsMap
     );
   }
+  if (options.withReachables) {
+    if (
+      options.reachablesSlicesFile &&
+      fs.existsSync(options.reachablesSlicesFile)
+    ) {
+      reachablesSlicesFile = options.reachablesSlicesFile;
+      reachablesSlice = JSON.parse(
+        fs.readFileSync(options.reachablesSlicesFile, "utf-8")
+      );
+    } else {
+      retMap = createSlice(language, dirPath, "reachables");
+      if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
+        reachablesSlicesFile = retMap.slicesFile;
+        reachablesSlice = JSON.parse(
+          fs.readFileSync(retMap.slicesFile, "utf-8")
+        );
+        console.log(
+          `To speed up this step, cache ${reachablesSlicesFile} and invoke evinse with the --reachables-slices-file argument.`
+        );
+      }
+    }
+  }
+  if (reachablesSlice && Object.keys(reachablesSlice).length) {
+    dataFlowFrames = await collectReachableFrames(language, reachablesSlice);
+  }
   return {
     atomFile: retMap.atomFile,
     usagesSlicesFile,
     dataFlowSlicesFile,
+    reachablesSlicesFile,
     purlLocationMap,
     servicesMap,
     dataFlowFrames,
@@ -752,6 +780,7 @@ export const createEvinseFile = (sliceArtefacts, options) => {
     tempDir,
     usagesSlicesFile,
     dataFlowSlicesFile,
+    reachablesSlicesFile,
     purlLocationMap,
     servicesMap,
     dataFlowFrames
@@ -830,6 +859,14 @@ export const createEvinseFile = (sliceArtefacts, options) => {
         text: fs.readFileSync(dataFlowSlicesFile, "utf8")
       });
     }
+    if (reachablesSlicesFile && fs.existsSync(reachablesSlicesFile)) {
+      bomJson.annotations.push({
+        subjects: [bomJson.serialNumber],
+        annotator: { component: bomJson.metadata.tools.components[0] },
+        timestamp: new Date().toISOString(),
+        text: fs.readFileSync(reachablesSlicesFile, "utf8")
+      });
+    }
   }
   // Increment the version
   bomJson.version = (bomJson.version || 1) + 1;
@@ -973,6 +1010,46 @@ export const collectDataFlowFrames = async (
   return dfFrames;
 };
 
+/**
+ * Method to convert reachable slice into usable callstack frames
+ * Implemented based on the logic proposed here - https://github.com/AppThreat/atom/blob/main/specification/docs/slices.md#data-flow-slice
+ *
+ * @param {string} language Application language
+ * @param {object} reachablesSlice Reachables slice object from atom
+ */
+export const collectReachableFrames = async (language, reachablesSlice) => {
+  const reachableNodes = reachablesSlice?.reachables || [];
+  // purl key and an array of frames array
+  // CycloneDX 1.5 currently accepts only 1 frame as evidence
+  // so this method is more future-proof
+  const dfFrames = {};
+  for (const anode of reachableNodes) {
+    let aframe = [];
+    let referredPurls = new Set(anode.purls || []);
+    for (const fnode of anode.flows) {
+      aframe.push({
+        package: fnode.parentPackageName,
+        module: fnode.parentClassName || "",
+        function: fnode.parentMethodName || "",
+        line: fnode.lineNumber || undefined,
+        column: fnode.columnNumber || undefined,
+        fullFilename: fnode.parentFileName || ""
+      });
+    }
+    referredPurls = Array.from(referredPurls);
+    if (referredPurls.length) {
+      for (const apurl of referredPurls) {
+        if (!dfFrames[apurl]) {
+          dfFrames[apurl] = [];
+        }
+        // Store this frame as an evidence for this purl
+        dfFrames[apurl].push(aframe);
+      }
+    }
+  }
+  return dfFrames;
+};
+
 /**
  * Method to pick a callstack frame as an evidence. This method is required since CycloneDX 1.5 accepts only a single frame as evidence.
  *