Skip to content

Commit

Permalink
Support for fail-on-error for container sbom generation. Env variable…
Browse files Browse the repository at this point in the history 
… to force non-strict tar extraction.

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
  • Loading branch information
@prabhu
prabhu committed Dec 26, 2024
1 parent 5398caa commit 3d78683
Show file tree
Hide file tree
Showing 7 changed files with 78 additions and 41 deletions.
41 changes: 25 additions & 16 deletions .github/workflows/dockertests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,32 +50,36 @@ jobs:
path: 'repotests/grafana-operator'
- name: dockertests
run: |
bin/cdxgen.js ubuntu:latest -t docker -o bomresults/bom-ubuntu.json
bin/cdxgen.js ubuntu:latest -t docker -o bomresults/bom-ubuntu.json --fail-on-error
docker rmi ubuntu:latest
bin/cdxgen.js almalinux:9.4-minimal -t docker -o bomresults/bom-almalinux.json
bin/cdxgen.js alpine:latest -t docker -o bomresults/bom-alpine.json --fail-on-error
docker rmi alpine:latest
bin/cdxgen.js almalinux:9.4-minimal -t docker -o bomresults/bom-almalinux.json --fail-on-error
docker rmi almalinux:9.4-minimal
bin/cdxgen.js centos:latest -t oci -o bomresults/bom-centos.json
bin/cdxgen.js centos:latest -t oci -o bomresults/bom-centos.json --fail-on-error
docker rmi centos:latest
bin/cdxgen.js phpmyadmin@sha256:1092481630056189e43cc0fe66fd01defcc9334d78ab4611b22f65e9a39869bd -o bomresults/bom-phpmyadmin.json --validate
bin/cdxgen.js phpmyadmin@sha256:1092481630056189e43cc0fe66fd01defcc9334d78ab4611b22f65e9a39869bd -o bomresults/bom-phpmyadmin.json --fail-on-error
docker rmi phpmyadmin@sha256:1092481630056189e43cc0fe66fd01defcc9334d78ab4611b22f65e9a39869bd
bin/cdxgen.js shiftleft/scan-slim -o bomresults/bom-scanslim.json -t container --validate
bin/cdxgen.js shiftleft/scan-slim -o bomresults/bom-scanslim.json -t container --fail-on-error
docker rmi shiftleft/scan-slim
bin/cdxgen.js redmine@sha256:a5c5f8a64a0d9a436a0a6941bc3fb156be0c89996add834fe33b66ebeed2439e -o bomresults/bom-redmine.json --validate
bin/cdxgen.js redmine@sha256:a5c5f8a64a0d9a436a0a6941bc3fb156be0c89996add834fe33b66ebeed2439e -o bomresults/bom-redmine.json --fail-on-error
docker rmi redmine@sha256:a5c5f8a64a0d9a436a0a6941bc3fb156be0c89996add834fe33b66ebeed2439e
bin/cdxgen.js rocket.chat@sha256:379f7afa0e67497c363ac9a9b3e7e6a6d31deee228233307c987e4a0c68b28e6 -o bomresults/bom-rocket.json --validate
bin/cdxgen.js rocket.chat@sha256:379f7afa0e67497c363ac9a9b3e7e6a6d31deee228233307c987e4a0c68b28e6 -o bomresults/bom-rocket.json --fail-on-error
docker rmi rocket.chat@sha256:379f7afa0e67497c363ac9a9b3e7e6a6d31deee228233307c987e4a0c68b28e6
bin/cdxgen.js sonarqube@sha256:7c0edcb99c964984db6d24330db33bb12de1e8ae0d5974d77640b1efea1483d1 -o bomresults/bom-sonar.json --validate
bin/cdxgen.js sonarqube@sha256:7c0edcb99c964984db6d24330db33bb12de1e8ae0d5974d77640b1efea1483d1 -o bomresults/bom-sonar.json --fail-on-error
docker rmi sonarqube@sha256:7c0edcb99c964984db6d24330db33bb12de1e8ae0d5974d77640b1efea1483d1
bin/cdxgen.js zookeeper@sha256:5bf00616677db5ef57d8a2da7c5dadf67f1a6be54b0c33a79be3332c9c80aeb6 -o bomresults/bom-zoo.json --validate
bin/cdxgen.js zookeeper@sha256:5bf00616677db5ef57d8a2da7c5dadf67f1a6be54b0c33a79be3332c9c80aeb6 -o bomresults/bom-zoo.json --fail-on-error
docker rmi zookeeper@sha256:5bf00616677db5ef57d8a2da7c5dadf67f1a6be54b0c33a79be3332c9c80aeb6
docker pull shiftleft/scan-slim:latest
docker save shiftleft/scan-slim:latest -o /tmp/scanslim.tar
docker rmi shiftleft/scan-slim:latest
bin/cdxgen.js /tmp/scanslim.tar -o bomresults/bom-scanarch.json --validate
bin/cdxgen.js -t docker-compose test/data -o bomresults/bom-dc.json --validate
bin/cdxgen.js -t operator repotests/grafana-operator -o bomresults/bom-op.json --validate
bin/cdxgen.js /tmp/scanslim.tar -o bomresults/bom-scanarch.json --fail-on-error
bin/cdxgen.js -t docker-compose test/data -o bomresults/bom-dc.json --fail-on-error
bin/cdxgen.js -t operator repotests/grafana-operator -o bomresults/bom-op.json --fail-on-error
rm /tmp/scanslim.tar
ls -ltr bomresults
env:
CDXGEN_DEBUG_MODE: debug
linux-dockertar-tests:
strategy:
matrix:
Expand Down Expand Up @@ -119,9 +123,14 @@ jobs:
docker pull elasticsearch@sha256:3686a5757ed46c9dbcf00f6f71fce48ffc5413b193a80d1c46a21e7aad4c53ad
docker save -o /tmp/elastic.tar elasticsearch@sha256:3686a5757ed46c9dbcf00f6f71fce48ffc5413b193a80d1c46a21e7aad4c53ad
docker rmi elasticsearch@sha256:3686a5757ed46c9dbcf00f6f71fce48ffc5413b193a80d1c46a21e7aad4c53ad
bin/cdxgen.js /tmp/elastic.tar -t docker -o bomresults/bom-elastic.tar.json --validate
bin/cdxgen.js /tmp/elastic.tar -t docker -o bomresults/bom-elastic.tar.json --fail-on-error
docker pull alpine:latest
docker save -o /tmp/alpine.tar alpine:latest
docker rmi alpine:latest
bin/cdxgen.js /tmp/alpine.tar -t docker -o bomresults/bom-alpine.tar.json --fail-on-error
ls -ltr bomresults
env:
CDXGEN_DEBUG_MODE: debug
os-tests:
runs-on: ubuntu-latest

Expand Down Expand Up @@ -157,7 +166,7 @@ jobs:
CI: true
- name: ostests
run: |
bin/cdxgen.js -t os -o bomresults/bom-os.json --validate
bin/cdxgen.js -t os -o bomresults/bom-os.json --fail-on-error
env:
CDXGEN_DEBUG_MODE: debug
- uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -196,7 +205,7 @@ jobs:
CI: true
- name: wintests
run: |
node bin/cdxgen.js -t os -o bomresults/bom-win.json --validate
node bin/cdxgen.js -t os -o bomresults/bom-win.json --fail-on-error
dir bomresults
env:
CDXGEN_DEBUG_MODE: debug
Expand Down
4 changes: 2 additions & 2 deletions lib/cli/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6598,7 +6598,7 @@ export async function createBom(path, options) {
// Docker and image archive support
// TODO: Support any source archive
if (path.endsWith(".tar") || path.endsWith(".tar.gz")) {
exportData = await exportArchive(path);
exportData = await exportArchive(path, options);
if (!exportData) {
console.log(
`OS BOM generation has failed due to problems with exporting the image ${path}`,
Expand All @@ -6617,7 +6617,7 @@ export async function createBom(path, options) {
path.includes("@sha256") ||
path.includes(":latest")
) {
exportData = await exportImage(path);
exportData = await exportImage(path, options);
if (exportData) {
isContainerMode = true;
} else {
Expand Down
6 changes: 6 additions & 0 deletions lib/helpers/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1331,6 +1331,9 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
console.log(
`Unable to parse ${pkgLockFile} without legacy peer dependencies. Retrying ...`,
);
if (DEBUG_MODE) {
console.log(e);
}
try {
arb = new Arborist({
path: path.dirname(pkgLockFile),
Expand All @@ -1341,6 +1344,9 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
console.log(
`Unable to parse ${pkgLockFile} in legacy and non-legacy mode. The resulting SBOM would be incomplete.`,
);
if (DEBUG_MODE) {
console.log(e);
}
return { pkgList, dependenciesList };
}
}
Expand Down
56 changes: 39 additions & 17 deletions lib/managers/docker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,16 @@ import { DEBUG_MODE, getAllFiles } from "../helpers/utils.js";
export const isWin = _platform() === "win32";
export const DOCKER_HUB_REGISTRY = "docker.io";

// Should we extract the tar image in non-strict mode
const NON_STRICT_TAR_EXTRACT = ["true", "1"].includes(
process?.env?.NON_STRICT_TAR_EXTRACT,
);
if (NON_STRICT_TAR_EXTRACT) {
console.log(
"Warning: Extracting container images and tar files in non-strict mode could lead to security risks!",
);
}

let dockerConn = undefined;
let isPodman = false;
let isPodmanRootless = true;
Expand Down Expand Up @@ -777,7 +787,7 @@ function handleAbsolutePath(entry) {
}
}

export const extractTar = async (fullImageName, dir) => {
export const extractTar = async (fullImageName, dir, options) => {
try {
await stream.pipeline(
createReadStream(fullImageName),
Expand All @@ -786,11 +796,13 @@ export const extractTar = async (fullImageName, dir) => {
preserveOwner: false,
noMtime: true,
noChmod: true,
strict: true,
strict: !NON_STRICT_TAR_EXTRACT,
C: dir,
portable: true,
onwarn: () => {
// ignore
onwarn: (code, message) => {
if (DEBUG_MODE) {
console.log(code, message);
}
},
onReadEntry: handleAbsolutePath,
filter: (path, entry) => {
Expand Down Expand Up @@ -865,8 +877,9 @@ export const extractTar = async (fullImageName, dir) => {
} else if (!["TAR_ENTRY_INFO", "TAR_ENTRY_INVALID"].includes(err.code)) {
console.log(err);
} else if (DEBUG_MODE) {
console.log(err.code, "is not handled yet.");
console.log(err.code, "is not handled yet in extractTar method.");
}
options.failOnError && process.exit(1);
return false;
}
};
Expand All @@ -875,7 +888,7 @@ export const extractTar = async (fullImageName, dir) => {
* Method to export a container image archive.
* Returns the location of the layers with additional packages related metadata
*/
export const exportArchive = async (fullImageName) => {
export const exportArchive = async (fullImageName, options = {}) => {
if (!existsSync(fullImageName)) {
console.log(`Unable to find container image archive ${fullImageName}`);
return undefined;
Expand All @@ -887,7 +900,7 @@ export const exportArchive = async (fullImageName) => {
mkdirSync(allLayersExplodedDir);
const manifestFile = join(tempDir, "manifest.json");
try {
await extractTar(fullImageName, tempDir);
await extractTar(fullImageName, tempDir, options);
// podman use blobs dir
if (existsSync(blobsDir)) {
if (DEBUG_MODE) {
Expand All @@ -900,7 +913,7 @@ export const exportArchive = async (fullImageName) => {
if (DEBUG_MODE) {
console.log(`Extracting ${ablob} to ${allLayersExplodedDir}`);
}
await extractTar(ablob, allLayersExplodedDir);
await extractTar(ablob, allLayersExplodedDir, options);
}
const lastLayerConfig = {};
const lastWorkingDir = "";
Expand All @@ -921,9 +934,11 @@ export const exportArchive = async (fullImageName) => {
{},
tempDir,
allLayersExplodedDir,
options,
);
}
console.log(`Unable to extract image archive to ${tempDir}`);
options.failOnError && process.exit(1);
} catch (err) {
// ignore
}
Expand All @@ -935,6 +950,7 @@ export const extractFromManifest = async (
localData,
tempDir,
allLayersExplodedDir,
options,
) => {
// Example of manifests
// [{"Config":"blobs/sha256/dedc100afa8d6718f5ac537730dd4a5ceea3563e695c90f1a8ac6df32c4cb291","RepoTags":["shiftleft/core:latest"],"Layers":["blobs/sha256/eaead16dc43bb8811d4ff450935d607f9ba4baffda4fc110cc402fa43f601d83","blobs/sha256/2039af03c0e17a3025b989335e9414149577fa09e7d0dcbee80155333639d11f"]}]
Expand Down Expand Up @@ -986,7 +1002,7 @@ export const extractFromManifest = async (
console.log(`Extracting layer ${layer} to ${allLayersExplodedDir}`);
}
try {
await extractTar(join(tempDir, layer), allLayersExplodedDir);
await extractTar(join(tempDir, layer), allLayersExplodedDir, options);
} catch (err) {
if (err.code === "TAR_BAD_ARCHIVE") {
if (DEBUG_MODE) {
Expand All @@ -995,6 +1011,7 @@ export const extractFromManifest = async (
} else {
console.log(err);
}
options.failOnError && process.exit(1);
}
}
if (manifest.Config) {
Expand Down Expand Up @@ -1037,7 +1054,7 @@ export const extractFromManifest = async (
* Method to export a container image by using the export feature in docker or podman service.
* Returns the location of the layers with additional packages related metadata
*/
export const exportImage = async (fullImageName) => {
export const exportImage = async (fullImageName, options) => {
// Safely ignore local directories
if (
!fullImageName ||
Expand Down Expand Up @@ -1084,7 +1101,7 @@ export const exportImage = async (fullImageName) => {
}
return localData;
}
await extractTar(imageTarFile, tempDir);
await extractTar(imageTarFile, tempDir, options);
if (DEBUG_MODE) {
console.log(`Cleaning up ${imageTarFile}`);
}
Expand All @@ -1110,11 +1127,13 @@ export const exportImage = async (fullImageName) => {
preserveOwner: false,
noMtime: true,
noChmod: true,
strict: true,
strict: !NON_STRICT_TAR_EXTRACT,
C: tempDir,
portable: true,
onwarn: () => {
// ignore
onwarn: (code, message) => {
if (DEBUG_MODE) {
console.log(code, message);
}
},
onReadEntry: handleAbsolutePath,
}),
Expand All @@ -1130,11 +1149,13 @@ export const exportImage = async (fullImageName) => {
preserveOwner: false,
noMtime: true,
noChmod: true,
strict: true,
strict: !NON_STRICT_TAR_EXTRACT,
C: tempDir,
portable: true,
onwarn: () => {
// ignore
onwarn: (code, message) => {
if (DEBUG_MODE) {
console.log(code, message);
}
},
onReadEntry: handleAbsolutePath,
}),
Expand Down Expand Up @@ -1168,6 +1189,7 @@ export const exportImage = async (fullImageName) => {
localData,
tempDir,
allLayersExplodedDir,
options,
);
}
console.log(`Unable to export image to ${tempDir}`);
Expand Down
Loading

0 comments on commit 3d78683

Please sign in to comment.