Filters and config files support (#634)

* Filters support Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Docs Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Support for filters in server mode Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Limit auto compositions to spec versions 1.5 and above Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Fixes #635 Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
CycloneDX · Oct 13, 2023 · 4359dee · 4359dee
1 parent 3acd1e6
commit 4359dee
Show file tree

Hide file tree

Showing 15 changed files with 59,542 additions and 61 deletions.
diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -144,9 +144,17 @@ jobs:
         with:
           repository: 'appthreat/blint'
           path: 'repotests/blint'
+      - uses: actions/checkout@v4
+        with:
+          repository: 'hoolicorp/java-sec-code'
+          path: 'repotests/java-sec-code'    
       - uses: dtolnay/rust-toolchain@stable
       - name: repotests
         run: |
+          bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json
+          bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --required-only
+          bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --filter postgres --filter json
+          bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code.json --only spring
           bin/cdxgen.js -p -r -t java repotests/shiftleft-java-example -o bomresults/bom-java.json --generate-key-and-sign
           node bin/evinse.js -i bomresults/bom-java.json -o bomresults/bom-java.evinse.json -l java --with-data-flow -p repotests/shiftleft-java-example
           SBOM_SIGN_ALGORITHM=RS512 SBOM_SIGN_PRIVATE_KEY=bomresults/private.key SBOM_SIGN_PUBLIC_KEY=bomresults/public.key bin/cdxgen.js -p -r -t github repotests/shiftleft-java-example -o bomresults/bom-github.json

diff --git a/README.md b/README.md
@@ -126,6 +126,7 @@ import { createBom, submitBom } from "npm:@cyclonedx/cdxgen@^9.0.1";
 
 ```text
 $ cdxgen -h
+Options:
   -o, --output                 Output file for bom.xml or bom.json. Default bom.
                                json
   -t, --type                   Project type
@@ -149,7 +150,9 @@ $ cdxgen -h
                                d or the project name and version together
       --parent-project-id      Dependency track parent project id
       --required-only          Include only the packages with required scope on
-                               the SBOM.                               [boolean]
+                               the SBOM. Would set compositions.aggregate to inc
+                               omplete unless --no-auto-compositions is passed.
+                                                                       [boolean]
       --fail-on-error          Fail if any dependency extractor fails. [boolean]
       --no-babel               Do not use babel to perform usage analysis for Ja
                                vaScript/TypeScript projects.           [boolean]
@@ -166,11 +169,21 @@ $ cdxgen -h
       --validate               Validate the generated SBOM using json schema. De
                                faults to true. Pass --no-validate to disable.
                                                        [boolean] [default: true]
+      --evidence               Generate SBOM with evidence for supported languag
+                               es. WIP                [boolean] [default: false]
       --usages-slices-file     Path for the usages slice file created by atom.
       --data-flow-slices-file  Path for the data-flow slice file created by atom
                                .
       --spec-version           CycloneDX Specification version to use. Defaults
                                to 1.5                             [default: 1.5]
+      --filter                 Filter components containining this word in purl.
+                                Multiple values allowed.                 [array]
+      --only                   Include components only containining this word in
+                                purl. Useful to generate BOM with first party co
+                               mponents alone. Multiple values allowed.  [array]
+      --auto-compositions      Automatically set compositions when the BOM was f
+                               iltered. Defaults to true
+                                                       [boolean] [default: true]
   -h, --help                   Show help                               [boolean]
   -v, --version                Show version number                     [boolean]
 ```

diff --git a/bin/cdxgen.js b/bin/cdxgen.js
@@ -11,6 +11,29 @@ import { fileURLToPath } from "node:url";
 import globalAgent from "global-agent";
 import process from "node:process";
 import { printTable, printDependencyTree } from "../display.js";
+import { findUpSync } from "find-up";
+import { load as _load } from "js-yaml";
+import { postProcess } from "../postgen.js";
+
+// Support for config files
+const configPath = findUpSync([
+  ".cdxgenrc",
+  ".cdxgen.json",
+  ".cdxgen.yml",
+  ".cdxgen.yaml"
+]);
+let config = {};
+if (configPath) {
+  try {
+    if (configPath.endsWith(".yml") || configPath.endsWith(".yaml")) {
+      config = _load(fs.readFileSync(configPath, "utf-8"));
+    } else {
+      config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+    }
+  } catch (e) {
+    console.log("Invalid config file", configPath);
+  }
+}
 
 let url = import.meta.url;
 if (!url.startsWith("file://")) {
@@ -22,6 +45,7 @@ import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 
 const args = yargs(hideBin(process.argv))
+  .env("CDXGEN")
   .option("output", {
     alias: "o",
     description: "Output file for bom.xml or bom.json. Default bom.json"
@@ -77,7 +101,8 @@ const args = yargs(hideBin(process.argv))
   })
   .option("required-only", {
     type: "boolean",
-    description: "Include only the packages with required scope on the SBOM."
+    description:
+      "Include only the packages with required scope on the SBOM. Would set compositions.aggregate to incomplete unless --no-auto-compositions is passed."
   })
   .option("fail-on-error", {
     type: "boolean",
@@ -132,6 +157,28 @@ const args = yargs(hideBin(process.argv))
     description: "CycloneDX Specification version to use. Defaults to 1.5",
     default: 1.5
   })
+  .option("filter", {
+    description:
+      "Filter components containining this word in purl. Multiple values allowed."
+  })
+  .option("only", {
+    description:
+      "Include components only containining this word in purl. Useful to generate BOM with first party components alone. Multiple values allowed."
+  })
+  .array("filter")
+  .array("only")
+  .option("auto-compositions", {
+    type: "boolean",
+    default: true,
+    description:
+      "Automatically set compositions when the BOM was filtered. Defaults to true"
+  })
+  .example([
+    ["$0 -t java .", "Generate a Java SBOM for the current directory"],
+    ["$0 --server", "Run cdxgen as a server"]
+  ])
+  .epilogue("for documentation, visit https://cyclonedx.github.io/cdxgen")
+  .config(config)
   .scriptName("cdxgen")
   .version()
   .alias("v", "version")
@@ -177,32 +224,14 @@ if (process.argv[1].includes("obom") && !args.type) {
 }
 
 /**
- * projectType: python, nodejs, java, golang
- * multiProject: Boolean to indicate monorepo or multi-module projects
+ * Command line options
  */
-const options = {
+const options = Object.assign({}, args, {
   projectType: args.type,
   multiProject: args.recurse,
-  output: args.output,
-  resolveClass: args.resolveClass,
-  installDeps: args.installDeps,
-  requiredOnly: args.requiredOnly,
-  failOnError: args.failOnError,
   noBabel: args.noBabel || args.babel === false,
-  deep: args.deep,
-  generateKeyAndSign: args.generateKeyAndSign,
-  project: args.projectId,
-  projectName: args.projectName,
-  projectGroup: args.projectGroup,
-  projectVersion: args.projectVersion,
-  server: args.server,
-  serverHost: args.serverHost,
-  serverPort: args.serverPort,
-  specVersion: args.specVersion,
-  evidence: args.evidence,
-  usagesSlicesFile: args.usagesSlicesFile,
-  dataFlowSlicesFile: args.dataFlowSlicesFile
-};
+  project: args.projectId
+});
 
 /**
  * Check for node >= 20 permissions
@@ -243,7 +272,7 @@ const checkPermissions = (filePath) => {
   // Start SBOM server
   if (args.server) {
     const serverModule = await import("../server.js");
-    return await serverModule.start(options);
+    return serverModule.start(options);
   }
   // Check if cdxgen has the required permissions
   if (!checkPermissions(filePath)) {
@@ -253,7 +282,10 @@ const checkPermissions = (filePath) => {
   if (!options.usagesSlicesFile) {
     options.usagesSlicesFile = `${options.projectName}-usages.json`;
   }
-  const bomNSData = (await createBom(filePath, options)) || {};
+  let bomNSData = (await createBom(filePath, options)) || {};
+  if (options.requiredOnly || options["filter"] || options["only"]) {
+    bomNSData = postProcess(bomNSData, options);
+  }
   if (!args.output) {
     args.output = "bom.json";
   }

diff --git a/bin/evinse.js b/bin/evinse.js
@@ -10,6 +10,28 @@ import process from "node:process";
 import { analyzeProject, createEvinseFile, prepareDB } from "../evinser.js";
 import { validateBom } from "../validator.js";
 import { printCallStack, printOccurrences, printServices } from "../display.js";
+import { findUpSync } from "find-up";
+import { load as _load } from "js-yaml";
+
+// Support for config files
+const configPath = findUpSync([
+  ".cdxgenrc",
+  ".cdxgen.json",
+  ".cdxgen.yml",
+  ".cdxgen.yaml"
+]);
+let config = {};
+if (configPath) {
+  try {
+    if (configPath.endsWith(".yml") || configPath.endsWith(".yaml")) {
+      config = _load(fs.readFileSync(configPath, "utf-8"));
+    } else {
+      config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+    }
+  } catch (e) {
+    console.log("Invalid config file", configPath);
+  }
+}
 
 const isWin = _platform() === "win32";
 const isMac = _platform() === "darwin";
@@ -28,6 +50,7 @@ if (!process.env.ATOM_DB && !fs.existsSync(ATOM_DB)) {
   }
 }
 const args = yargs(hideBin(process.argv))
+  .env("EVINSE")
   .option("input", {
     alias: "i",
     description: "Input SBOM file. Default bom.json",
@@ -88,6 +111,7 @@ const args = yargs(hideBin(process.argv))
     type: "boolean",
     description: "Print the evidences as table"
   })
+  .config(config)
   .scriptName("evinse")
   .version()
   .help("h").argv;

diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md
@@ -1,5 +1,98 @@
 # Advanced Usage
 
+## Filtering components
+
+cdxgen can filter the components and the dependency tree before writing to a BOM json file. Three kinds of filters are allowed:
+
+### Required only filter
+
+Pass `--required-only` to only store components with the `scope` attribute set to `required`. These are usually considered direct dependencies.
+
+```shell
+cdxgen -t java -o /tmp/bom.json -p --required-only
+```
+
+Languages supported:
+
+- Java with Maven
+- Node.js
+- Go
+- Php
+
+### Purl filter
+
+Use `--filter` to filter components containing the string in the purl.
+
+```shell
+cdxgen -t java -o /tmp/bom.json -p --filter org.springframework
+```
+
+### Include only filter
+
+Use `--only` to include only those components containing the string in the purl. This can be used to generate BOM with "first party" components only.
+
+```shell
+cdxgen -t java -o /tmp/bom.json -p --only org.springframework
+```
+
+## Automatic compositions
+
+When using any filters, cdxgen would automatically set the [compositions.aggregate](https://cyclonedx.org/docs/1.5/json/#compositions_items_aggregate) property to "incomplete" or "incomplete_first_party_only".
+
+To disable this behavior, pass `--no-auto-compositions`.
+
+## Configuration files
+
+Tired of passing command line arguments to cdxgen?
+
+JSON format
+
+- .cdxgenrc
+- .cdxgen.json
+
+YAML format
+
+- .cdxgen.yml
+- .cdxgen.yaml
+
+Examples:
+
+```json
+{
+  "type": "java",
+  "print": true,
+  "output": "bom.json"
+}
+```
+
+```yaml
+# Java type
+type: java
+# Print the BOM as table and tree
+print: true
+# Set the output file
+output: bom.json
+# Only include these components in the BOM
+only: org.springframework
+```
+
+### Environment variables
+
+All command line arguments can also be passed as environment variables using the "CDXGEN\_" prefix.
+
+```shell
+export CDXGEN_TYPE=java
+export CDXGEN_PROJECT_NAME=foo
+```
+
+Environment variables override values from the configuration files.
+
+### Config value ordering
+
+- Command-line arguments
+- Environment variables
+- Configuration files (JSON first, followed by yaml)
+
 ## Evinse Mode / SaaSBOM
 
 Evinse (Evinse Verification Is Nearly SBOM Evidence) is a new command with cdxgen to generate component evidence and SaaSBOM for supported languages. The tool is powered by [atom](https://github.com/AppThreat/atom).

diff --git a/docs/CLI.md b/docs/CLI.md
@@ -63,6 +63,8 @@ import { createBom, submitBom } from "npm:@cyclonedx/cdxgen@^9.0.1";
 
 ```text
 $ cdxgen -h
+
+Options:
   -o, --output                 Output file for bom.xml or bom.json. Default bom.
                                json
   -t, --type                   Project type
@@ -86,7 +88,9 @@ $ cdxgen -h
                                d or the project name and version together
       --parent-project-id      Dependency track parent project id
       --required-only          Include only the packages with required scope on
-                               the SBOM.                               [boolean]
+                               the SBOM. Would set compositions.aggregate to inc
+                               omplete unless --no-auto-compositions is passed.
+                                                                       [boolean]
       --fail-on-error          Fail if any dependency extractor fails. [boolean]
       --no-babel               Do not use babel to perform usage analysis for Ja
                                vaScript/TypeScript projects.           [boolean]
@@ -103,11 +107,21 @@ $ cdxgen -h
       --validate               Validate the generated SBOM using json schema. De
                                faults to true. Pass --no-validate to disable.
                                                        [boolean] [default: true]
+      --evidence               Generate SBOM with evidence for supported languag
+                               es. WIP                [boolean] [default: false]
       --usages-slices-file     Path for the usages slice file created by atom.
       --data-flow-slices-file  Path for the data-flow slice file created by atom
                                .
       --spec-version           CycloneDX Specification version to use. Defaults
                                to 1.5                             [default: 1.5]
+      --filter                 Filter components containining this word in purl.
+                                Multiple values allowed.                 [array]
+      --only                   Include components only containining this word in
+                                purl. Useful to generate BOM with first party co
+                               mponents alone. Multiple values allowed.  [array]
+      --auto-compositions      Automatically set compositions when the BOM was f
+                               iltered. Defaults to true
+                                                       [boolean] [default: true]
   -h, --help                   Show help                               [boolean]
   -v, --version                Show version number                     [boolean]
 ```