diff --git a/utils.js b/utils.js index f59d702c67..cb50090690 100644 --- a/utils.js +++ b/utils.js @@ -2608,89 +2608,66 @@ export const parsePoetrylockData = async function (lockData, lockFile) { export async function parseReqFile(reqData, fetchDepsInfo) { const pkgList = []; let compScope = undefined; - reqData.split("\n").forEach((l) => { - l = l.replace("\r", ""); - l = l.trim(); - if (l.startsWith("Skipping line") || l.startsWith("(add")) { - return; - } - if (l.includes("# Basic requirements")) { - compScope = "required"; - } else if (l.includes("added by pip freeze")) { - compScope = undefined; - } - if (!l.startsWith("#") && !l.startsWith("-")) { - if (l.includes(" ")) { - l = l.split(" ")[0]; - } - if (l.indexOf("=") > -1) { - let tmpA = l.split(/(==|<=|~=|>=)/); - if (tmpA.includes("#")) { - tmpA = tmpA.split("#")[0]; - } - let versionStr = tmpA[tmpA.length - 1].trim().replace("*", "0"); - if (versionStr.indexOf(" ") > -1) { - versionStr = versionStr.split(" ")[0]; + reqData + .replace(/ [\\]\n/g, "") + .replace(/ {4}/g, " ") + .split("\n") + .forEach((l) => { + l = l.replace("\r", ""); + l = l.trim(); + let markers = undefined; + if (l.includes(" ; ")) { + const tmpA = l.split(" ; "); + if (tmpA && tmpA.length === 2) { + l = tmpA[0]; + markers = tmpA[1]; } - if (versionStr === "0") { - versionStr = null; + } + if (l.startsWith("Skipping line") || l.startsWith("(add")) { + return; + } + if (l.includes("# Basic requirements")) { + compScope = "required"; + } else if (l.includes("added by pip freeze")) { + compScope = undefined; + } + if (!l.startsWith("#") && !l.startsWith("-")) { + if (l.includes(" ")) { + l = l.split(" ")[0]; } - if (!tmpA[0].includes("=") && !tmpA[0].trim().includes(" ")) { - const name = tmpA[0].trim().replace(";", ""); - if (!PYTHON_STD_MODULES.includes(name)) { - pkgList.push({ - name, - version: versionStr, - scope: compScope - }); + if (l.indexOf("=") > -1) { + let tmpA = l.split(/(==|<=|~=|>=)/); + if (tmpA.includes("#")) { + tmpA = tmpA.split("#")[0]; } - } - } else if (l.includes("<") && l.includes(">")) { - const tmpA = l.split(">"); - const name = tmpA[0].trim().replace(";", ""); - const versionSpecifiers = l.replace(name, ""); - if (!PYTHON_STD_MODULES.includes(name)) { - pkgList.push({ - name, - version: undefined, - scope: compScope, - properties: [ - { - name: "cdx:pypi:versionSpecifiers", - value: versionSpecifiers + let versionStr = tmpA[tmpA.length - 1].trim().replace("*", "0"); + if (versionStr.indexOf(" ") > -1) { + versionStr = versionStr.split(" ")[0]; + } + if (versionStr === "0") { + versionStr = null; + } + if (!tmpA[0].includes("=") && !tmpA[0].trim().includes(" ")) { + const name = tmpA[0].trim().replace(";", ""); + if (!PYTHON_STD_MODULES.includes(name)) { + const apkg = { + name, + version: versionStr, + scope: compScope + }; + if (markers) { + apkg.properties = [ + { + name: "cdx:pip:markers", + value: markers + } + ]; } - ] - }); - } - } else if (/[>|[|@]/.test(l)) { - let tmpA = l.split(/(>|\[|@)/); - if (tmpA.includes("#")) { - tmpA = tmpA.split("#")[0]; - } - if (!tmpA[0].trim().includes(" ")) { - const name = tmpA[0].trim().replace(";", ""); - const versionSpecifiers = l.replace(name, ""); - if (!PYTHON_STD_MODULES.includes(name)) { - pkgList.push({ - name, - version: undefined, - scope: compScope, - properties: [ - { - name: "cdx:pypi:versionSpecifiers", - value: versionSpecifiers - } - ] - }); + pkgList.push(apkg); + } } - } - } else if (l) { - if (l.includes("#")) { - l = l.split("#")[0]; - } - l = l.trim(); - const tmpA = l.split(/(<|>)/); - if (tmpA && tmpA.length === 3) { + } else if (l.includes("<") && l.includes(">")) { + const tmpA = l.split(">"); const name = tmpA[0].trim().replace(";", ""); const versionSpecifiers = l.replace(name, ""); if (!PYTHON_STD_MODULES.includes(name)) { @@ -2706,26 +2683,70 @@ export async function parseReqFile(reqData, fetchDepsInfo) { ] }); } - } else if (!l.includes(" ")) { - const name = l.replace(";", ""); - const versionSpecifiers = l.replace(name, ""); - if (!PYTHON_STD_MODULES.includes(name)) { - pkgList.push({ - name, - version: null, - scope: compScope, - properties: [ - { - name: "cdx:pypi:versionSpecifiers", - value: versionSpecifiers - } - ] - }); + } else if (/[>|[|@]/.test(l)) { + let tmpA = l.split(/(>|\[|@)/); + if (tmpA.includes("#")) { + tmpA = tmpA.split("#")[0]; + } + if (!tmpA[0].trim().includes(" ")) { + const name = tmpA[0].trim().replace(";", ""); + const versionSpecifiers = l.replace(name, ""); + if (!PYTHON_STD_MODULES.includes(name)) { + pkgList.push({ + name, + version: undefined, + scope: compScope, + properties: [ + { + name: "cdx:pypi:versionSpecifiers", + value: versionSpecifiers + } + ] + }); + } + } + } else if (l) { + if (l.includes("#")) { + l = l.split("#")[0]; + } + l = l.trim(); + const tmpA = l.split(/(<|>)/); + if (tmpA && tmpA.length === 3) { + const name = tmpA[0].trim().replace(";", ""); + const versionSpecifiers = l.replace(name, ""); + if (!PYTHON_STD_MODULES.includes(name)) { + pkgList.push({ + name, + version: undefined, + scope: compScope, + properties: [ + { + name: "cdx:pypi:versionSpecifiers", + value: versionSpecifiers + } + ] + }); + } + } else if (!l.includes(" ")) { + const name = l.replace(";", ""); + const versionSpecifiers = l.replace(name, ""); + if (!PYTHON_STD_MODULES.includes(name)) { + pkgList.push({ + name, + version: null, + scope: compScope, + properties: [ + { + name: "cdx:pypi:versionSpecifiers", + value: versionSpecifiers + } + ] + }); + } } } } - } - }); + }); return await getPyMetadata(pkgList, fetchDepsInfo); } diff --git a/utils.test.js b/utils.test.js index 95bf154e1c..968fc6a24c 100644 --- a/utils.test.js +++ b/utils.test.js @@ -2261,6 +2261,25 @@ test("parse requirements.txt", async () => { version: "8.6.2", scope: "required" }); + deps = await parseReqFile( + readFileSync("./test/data/chen-science-requirements.txt", { + encoding: "utf-8" + }), + false + ); + expect(deps.length).toEqual(87); + expect(deps[0]).toEqual({ + name: "aiofiles", + version: "23.2.1", + scope: undefined, + properties: [ + { + name: "cdx:pip:markers", + value: + 'python_full_version >= "3.8.1" and python_version < "3.12" --hash=sha256:19297512c647d4b27a2cf7c34caa7e405c0d60b5560618a29a9fe027b18b0107 --hash=sha256:84ec2218d8419404abcb9f0c02df3f34c6e0a68ed41072acfb1cef5cbc29051a' + } + ] + }); }); test("parse pyproject.toml", async () => {