From f6a83ab61d18d2651b88a4a6bdb01919eeb466f6 Mon Sep 17 00:00:00 2001 From: Adam Setch Date: Thu, 1 Aug 2024 08:45:31 +1000 Subject: [PATCH 1/3] feat(server): add fetchLicense arg Signed-off-by: Adam Setch --- docs/SERVER.md | 1 + server.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/SERVER.md b/docs/SERVER.md index 952eaf151..ad863a024 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -52,6 +52,7 @@ Arguments can be passed either via the query string or as a JSON body. The follo | only | Include components only containing this word in purl. Useful to generate BOM with first party components alone. Multiple values allowed. [array] | | autoCompositions | Automatically set compositions when the BOM was filtered. [boolean] [default: true] | | gitBranch | Git branch used when cloning the repository. If not specified will use the default branch assigned to the repository. | +| fetchLicense | Automatically query public registries such as maven, npm, or nuget to resolve the package licenses. This is a time-consuming operation and is disabled by default. | ## Ways to use server mode diff --git a/server.js b/server.js index c70b9b618..d2be87319 100644 --- a/server.js +++ b/server.js @@ -92,6 +92,7 @@ const parseQueryString = (q, body, options = {}) => { "includeFormulation", "includeCrypto", "standard", + "fetchLicense" ]; for (const param of queryParams) { @@ -115,6 +116,9 @@ const parseQueryString = (q, body, options = {}) => { if (options.profile) { applyProfileOptions(options); } + if (options.fetchLicense) { + process.env.FETCH_LICENSE = options.fetchLicense + } return options; }; From 5d6b39042ce7210f5cb95524d40a35c83afdacbf Mon Sep 17 00:00:00 2001 From: Adam Setch Date: Fri, 2 Aug 2024 08:02:06 +1000 Subject: [PATCH 2/3] feat(server): add fetchLicense arg Signed-off-by: Adam Setch --- index.js | 8 ++++---- utils.js | 51 +++++++++++++++++++++++++++++---------------------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/index.js b/index.js index 934fda893..ead34e6ec 100644 --- a/index.js +++ b/index.js @@ -32,7 +32,6 @@ import { CARGO_CMD, CLJ_CMD, DEBUG_MODE, - FETCH_LICENSE, LEIN_CMD, MAX_BUFFER, PREFER_MAVEN_DEPS_TREE, @@ -136,6 +135,7 @@ import { parseYarnLock, readZipEntry, splitOutputByGradleProjects, + shouldFetchLicense } from "./utils.js"; let url = import.meta.url; if (!url.startsWith("file://")) { @@ -3100,7 +3100,7 @@ export async function createPythonBom(path, options) { if (tempDir?.startsWith(tmpdir()) && rmSync) { rmSync(tempDir, { recursive: true, force: true }); } - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { pkgList = await getPyMetadata(pkgList, false); } return buildBomNSData(options, pkgList, "pypi", { @@ -4280,7 +4280,7 @@ export async function createSwiftBom(path, options) { } } } - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { pkgList = await getSwiftPackageMetadata(pkgList); } return buildBomNSData(options, pkgList, "swift", { @@ -5209,7 +5209,7 @@ export async function createCsharpBom(path, options) { dependsOn: Array.from(parentDependsOn), }); } - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { const retMap = await getNugetMetadata(pkgList, dependencies); if (retMap.dependencies?.length) { dependencies = mergeDependencies( diff --git a/utils.js b/utils.js index d6eacc885..91678e49b 100644 --- a/utils.js +++ b/utils.js @@ -128,9 +128,9 @@ export const PREFER_MAVEN_DEPS_TREE = ["true", "1"].includes(process.env.PREFER_MAVEN_DEPS_TREE); // Whether license information should be fetched -export const FETCH_LICENSE = - process.env.FETCH_LICENSE && - ["true", "1"].includes(process.env.FETCH_LICENSE); +export function shouldFetchLicense() { + return process.env.FETCH_LICENSE && ["true", "1"].includes(process.env.FETCH_LICENSE); +} // Whether search.maven.org will be used to identify jars without maven metadata; default, if unset shall be 'true' export const SEARCH_MAVEN_ORG = @@ -765,12 +765,18 @@ export async function getNpmMetadata(pkgList) { let body = {}; if (metadata_cache[key]) { body = metadata_cache[key]; + if (DEBUG_MODE) { + console.log(`npm metadata: using ${key} metadata from local cache`); + } } else { const res = await cdxgenAgent.get(NPM_URL + key, { responseType: "json", }); body = res.body; metadata_cache[key] = body; + if (DEBUG_MODE) { + console.log(`npm metadata: looking up ${key} metadata on npm`); + } } p.description = body.versions?.[p.version]?.description || body.description; @@ -785,6 +791,7 @@ export async function getNpmMetadata(pkgList) { p.homepage = { url: body.homepage }; } cdepList.push(p); + } catch (err) { cdepList.push(p); if (DEBUG_MODE) { @@ -872,7 +879,7 @@ export async function parsePkgJson(pkgJsonFile, simple = false) { // continue regardless of error } } - if (!simple && FETCH_LICENSE && pkgList && pkgList.length) { + if (!simple && shouldFetchLicense() && pkgList && pkgList.length) { if (DEBUG_MODE) { console.log( `About to fetch license information for ${pkgList.length} packages in parsePkgJson`, @@ -1215,7 +1222,7 @@ export async function parsePkgLock(pkgLockFile, options = {}) { options, )); - if (FETCH_LICENSE && pkgList && pkgList.length) { + if (shouldFetchLicense() && pkgList && pkgList.length) { if (DEBUG_MODE) { console.log( `About to fetch license information for ${pkgList.length} packages in parsePkgLock`, @@ -1532,7 +1539,7 @@ export async function parseYarnLock(yarnLockFile) { } }); } - if (FETCH_LICENSE && pkgList && pkgList.length) { + if (shouldFetchLicense() && pkgList && pkgList.length) { if (DEBUG_MODE) { console.log( `About to fetch license information for ${pkgList.length} packages in parseYarnLock`, @@ -1609,7 +1616,7 @@ export async function parseNodeShrinkwrap(swFile) { } } } - if (FETCH_LICENSE && pkgList && pkgList.length) { + if (shouldFetchLicense() && pkgList && pkgList.length) { if (DEBUG_MODE) { console.log( `About to fetch license information for ${pkgList.length} packages in parseNodeShrinkwrap`, @@ -2043,7 +2050,7 @@ export async function parsePnpmLock(pnpmLock, parentComponent = null) { }); } } - if (FETCH_LICENSE && pkgList && pkgList.length) { + if (shouldFetchLicense() && pkgList && pkgList.length) { if (DEBUG_MODE) { console.log( `About to fetch license information for ${pkgList.length} packages in parsePnpmLock`, @@ -2102,7 +2109,7 @@ export async function parseBowerJson(bowerJsonFile) { // continue regardless of error } } - if (FETCH_LICENSE && pkgList && pkgList.length) { + if (shouldFetchLicense() && pkgList && pkgList.length) { if (DEBUG_MODE) { console.log( `About to fetch license information for ${pkgList.length} packages in parseBowerJson`, @@ -2187,7 +2194,7 @@ export async function parseMinJs(minJsFile) { // continue regardless of error } } - if (FETCH_LICENSE && pkgList && pkgList.length) { + if (shouldFetchLicense() && pkgList && pkgList.length) { if (DEBUG_MODE) { console.log( `About to fetch license information for ${pkgList.length} packages in parseMinJs`, @@ -3176,7 +3183,7 @@ export async function getMvnMetadata(pkgList, jarNSMapping = {}) { if (!pkgList || !pkgList.length) { return pkgList; } - if (DEBUG_MODE && FETCH_LICENSE) { + if (DEBUG_MODE && shouldFetchLicense()) { console.log(`About to query maven for ${pkgList.length} packages`); } for (const p of pkgList) { @@ -3212,7 +3219,7 @@ export async function getMvnMetadata(pkgList, jarNSMapping = {}) { } const group = p.group || ""; // If the package already has key metadata skip querying maven - if (group && p.name && p.version && !FETCH_LICENSE) { + if (group && p.name && p.version && !shouldFetchLicense()) { cdepList.push(p); continue; } @@ -3454,7 +3461,7 @@ export function guessPypiMatchingVersion(versionsList, versionSpecifiers) { * @param {Boolean} fetchDepsInfo Fetch dependencies info from pypi */ export async function getPyMetadata(pkgList, fetchDepsInfo) { - if (!FETCH_LICENSE && !fetchDepsInfo) { + if (!shouldFetchLicense() && !fetchDepsInfo) { return pkgList; } const PYPI_URL = process.env.PYPI_URL || "https://pypi.org/pypi/"; @@ -4368,7 +4375,7 @@ export async function getGoPkgLicense(repoMetadata) { export async function getGoPkgComponent(group, name, version, hash) { let pkg = {}; let license = undefined; - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { if (DEBUG_MODE) { console.log( `About to fetch go package license information for ${group}:${name}`, @@ -4697,7 +4704,7 @@ export async function parseGosumData(gosumData) { const version = tmpA[1].replace("/go.mod", ""); const hash = tmpA[tmpA.length - 1].replace("h1:", "sha256-"); let license = undefined; - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { if (DEBUG_MODE) { console.log( `About to fetch go package license information for ${name}`, @@ -4749,7 +4756,7 @@ export async function parseGopkgData(gopkgData) { case "name": pkg.group = ""; pkg.name = value; - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { pkg.license = await getGoPkgLicense({ group: pkg.group, name: pkg.name, @@ -4959,7 +4966,7 @@ export async function parseGemspecData(gemspecData) { } }); pkgList = [pkg]; - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { return await getRubyGemsMetadata(pkgList); } return pkgList; @@ -5211,7 +5218,7 @@ export async function parseGemfileLockData(gemLockData, lockFile) { dependsOn: Array.from(dependenciesMap[k]), }); } - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { pkgList = await getRubyGemsMetadata(pkgList); return { pkgList, dependenciesList }; } @@ -5569,7 +5576,7 @@ export async function parseCargoTomlData( if (pkg) { addPackageToList(pkgList, pkg, { packageMode, simple }); } - if (!simple && FETCH_LICENSE) { + if (!simple && shouldFetchLicense()) { return await getCratesMetadata(pkgList); } return pkgList; @@ -5709,7 +5716,7 @@ export async function parseCargoData( if (pkg) { addPackageToList(pkgList, pkg, { simple }); } - if (FETCH_LICENSE && !simple) { + if (!simple && shouldFetchLicense()) { return await getCratesMetadata(pkgList); } return pkgList; @@ -5875,7 +5882,7 @@ export async function parseCargoAuditableData(cargoData) { }); } }); - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { return await getCratesMetadata(pkgList); } return pkgList; @@ -5914,7 +5921,7 @@ export async function parsePubLockData(pubLockData) { } } }); - if (FETCH_LICENSE) { + if (shouldFetchLicense()) { return await getDartMetadata(pkgList); } return pkgList; From 074b4d36810cbe44ac33d5b9b00201bb389b94ab Mon Sep 17 00:00:00 2001 From: Adam Setch Date: Fri, 2 Aug 2024 08:06:44 +1000 Subject: [PATCH 3/3] feat(server): add fetchLicense arg Signed-off-by: Adam Setch --- index.js | 2 +- server.js | 4 ++-- utils.js | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index ead34e6ec..581f39862 100644 --- a/index.js +++ b/index.js @@ -134,8 +134,8 @@ import { parseSwiftResolved, parseYarnLock, readZipEntry, + shouldFetchLicense, splitOutputByGradleProjects, - shouldFetchLicense } from "./utils.js"; let url = import.meta.url; if (!url.startsWith("file://")) { diff --git a/server.js b/server.js index d2be87319..adbe22a8b 100644 --- a/server.js +++ b/server.js @@ -92,7 +92,7 @@ const parseQueryString = (q, body, options = {}) => { "includeFormulation", "includeCrypto", "standard", - "fetchLicense" + "fetchLicense", ]; for (const param of queryParams) { @@ -117,7 +117,7 @@ const parseQueryString = (q, body, options = {}) => { applyProfileOptions(options); } if (options.fetchLicense) { - process.env.FETCH_LICENSE = options.fetchLicense + process.env.FETCH_LICENSE = options.fetchLicense; } return options; }; diff --git a/utils.js b/utils.js index 91678e49b..e94340892 100644 --- a/utils.js +++ b/utils.js @@ -129,7 +129,10 @@ export const PREFER_MAVEN_DEPS_TREE = // Whether license information should be fetched export function shouldFetchLicense() { - return process.env.FETCH_LICENSE && ["true", "1"].includes(process.env.FETCH_LICENSE); + return ( + process.env.FETCH_LICENSE && + ["true", "1"].includes(process.env.FETCH_LICENSE) + ); } // Whether search.maven.org will be used to identify jars without maven metadata; default, if unset shall be 'true' @@ -791,7 +794,6 @@ export async function getNpmMetadata(pkgList) { p.homepage = { url: body.homepage }; } cdepList.push(p); - } catch (err) { cdepList.push(p); if (DEBUG_MODE) {