Improved requirements txt parsing

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
CycloneDX · Oct 2, 2023 · be7a7e1 · be7a7e1
1 parent 81a084d
commit be7a7e1
Show file tree

Hide file tree

Showing 2 changed files with 135 additions and 95 deletions.
diff --git a/utils.js b/utils.js
@@ -2608,89 +2608,66 @@ export const parsePoetrylockData = async function (lockData, lockFile) {
 export async function parseReqFile(reqData, fetchDepsInfo) {
   const pkgList = [];
   let compScope = undefined;
-  reqData.split("\n").forEach((l) => {
-    l = l.replace("\r", "");
-    l = l.trim();
-    if (l.startsWith("Skipping line") || l.startsWith("(add")) {
-      return;
-    }
-    if (l.includes("# Basic requirements")) {
-      compScope = "required";
-    } else if (l.includes("added by pip freeze")) {
-      compScope = undefined;
-    }
-    if (!l.startsWith("#") && !l.startsWith("-")) {
-      if (l.includes(" ")) {
-        l = l.split(" ")[0];
-      }
-      if (l.indexOf("=") > -1) {
-        let tmpA = l.split(/(==|<=|~=|>=)/);
-        if (tmpA.includes("#")) {
-          tmpA = tmpA.split("#")[0];
-        }
-        let versionStr = tmpA[tmpA.length - 1].trim().replace("*", "0");
-        if (versionStr.indexOf(" ") > -1) {
-          versionStr = versionStr.split(" ")[0];
+  reqData
+    .replace(/ [\\]\n/g, "")
+    .replace(/ {4}/g, " ")
+    .split("\n")
+    .forEach((l) => {
+      l = l.replace("\r", "");
+      l = l.trim();
+      let markers = undefined;
+      if (l.includes(" ; ")) {
+        const tmpA = l.split(" ; ");
+        if (tmpA && tmpA.length === 2) {
+          l = tmpA[0];
+          markers = tmpA[1];
         }
-        if (versionStr === "0") {
-          versionStr = null;
+      }
+      if (l.startsWith("Skipping line") || l.startsWith("(add")) {
+        return;
+      }
+      if (l.includes("# Basic requirements")) {
+        compScope = "required";
+      } else if (l.includes("added by pip freeze")) {
+        compScope = undefined;
+      }
+      if (!l.startsWith("#") && !l.startsWith("-")) {
+        if (l.includes(" ")) {
+          l = l.split(" ")[0];
         }
-        if (!tmpA[0].includes("=") && !tmpA[0].trim().includes(" ")) {
-          const name = tmpA[0].trim().replace(";", "");
-          if (!PYTHON_STD_MODULES.includes(name)) {
-            pkgList.push({
-              name,
-              version: versionStr,
-              scope: compScope
-            });
+        if (l.indexOf("=") > -1) {
+          let tmpA = l.split(/(==|<=|~=|>=)/);
+          if (tmpA.includes("#")) {
+            tmpA = tmpA.split("#")[0];
           }
-        }
-      } else if (l.includes("<") && l.includes(">")) {
-        const tmpA = l.split(">");
-        const name = tmpA[0].trim().replace(";", "");
-        const versionSpecifiers = l.replace(name, "");
-        if (!PYTHON_STD_MODULES.includes(name)) {
-          pkgList.push({
-            name,
-            version: undefined,
-            scope: compScope,
-            properties: [
-              {
-                name: "cdx:pypi:versionSpecifiers",
-                value: versionSpecifiers
+          let versionStr = tmpA[tmpA.length - 1].trim().replace("*", "0");
+          if (versionStr.indexOf(" ") > -1) {
+            versionStr = versionStr.split(" ")[0];
+          }
+          if (versionStr === "0") {
+            versionStr = null;
+          }
+          if (!tmpA[0].includes("=") && !tmpA[0].trim().includes(" ")) {
+            const name = tmpA[0].trim().replace(";", "");
+            if (!PYTHON_STD_MODULES.includes(name)) {
+              const apkg = {
+                name,
+                version: versionStr,
+                scope: compScope
+              };
+              if (markers) {
+                apkg.properties = [
+                  {
+                    name: "cdx:pip:markers",
+                    value: markers
+                  }
+                ];
               }
-            ]
-          });
-        }
-      } else if (/[>|[|@]/.test(l)) {
-        let tmpA = l.split(/(>|\[|@)/);
-        if (tmpA.includes("#")) {
-          tmpA = tmpA.split("#")[0];
-        }
-        if (!tmpA[0].trim().includes(" ")) {
-          const name = tmpA[0].trim().replace(";", "");
-          const versionSpecifiers = l.replace(name, "");
-          if (!PYTHON_STD_MODULES.includes(name)) {
-            pkgList.push({
-              name,
-              version: undefined,
-              scope: compScope,
-              properties: [
-                {
-                  name: "cdx:pypi:versionSpecifiers",
-                  value: versionSpecifiers
-                }
-              ]
-            });
+              pkgList.push(apkg);
+            }
           }
-        }
-      } else if (l) {
-        if (l.includes("#")) {
-          l = l.split("#")[0];
-        }
-        l = l.trim();
-        const tmpA = l.split(/(<|>)/);
-        if (tmpA && tmpA.length === 3) {
+        } else if (l.includes("<") && l.includes(">")) {
+          const tmpA = l.split(">");
           const name = tmpA[0].trim().replace(";", "");
           const versionSpecifiers = l.replace(name, "");
           if (!PYTHON_STD_MODULES.includes(name)) {
@@ -2706,26 +2683,70 @@ export async function parseReqFile(reqData, fetchDepsInfo) {
               ]
             });
           }
-        } else if (!l.includes(" ")) {
-          const name = l.replace(";", "");
-          const versionSpecifiers = l.replace(name, "");
-          if (!PYTHON_STD_MODULES.includes(name)) {
-            pkgList.push({
-              name,
-              version: null,
-              scope: compScope,
-              properties: [
-                {
-                  name: "cdx:pypi:versionSpecifiers",
-                  value: versionSpecifiers
-                }
-              ]
-            });
+        } else if (/[>|[|@]/.test(l)) {
+          let tmpA = l.split(/(>|\[|@)/);
+          if (tmpA.includes("#")) {
+            tmpA = tmpA.split("#")[0];
+          }
+          if (!tmpA[0].trim().includes(" ")) {
+            const name = tmpA[0].trim().replace(";", "");
+            const versionSpecifiers = l.replace(name, "");
+            if (!PYTHON_STD_MODULES.includes(name)) {
+              pkgList.push({
+                name,
+                version: undefined,
+                scope: compScope,
+                properties: [
+                  {
+                    name: "cdx:pypi:versionSpecifiers",
+                    value: versionSpecifiers
+                  }
+                ]
+              });
+            }
+          }
+        } else if (l) {
+          if (l.includes("#")) {
+            l = l.split("#")[0];
+          }
+          l = l.trim();
+          const tmpA = l.split(/(<|>)/);
+          if (tmpA && tmpA.length === 3) {
+            const name = tmpA[0].trim().replace(";", "");
+            const versionSpecifiers = l.replace(name, "");
+            if (!PYTHON_STD_MODULES.includes(name)) {
+              pkgList.push({
+                name,
+                version: undefined,
+                scope: compScope,
+                properties: [
+                  {
+                    name: "cdx:pypi:versionSpecifiers",
+                    value: versionSpecifiers
+                  }
+                ]
+              });
+            }
+          } else if (!l.includes(" ")) {
+            const name = l.replace(";", "");
+            const versionSpecifiers = l.replace(name, "");
+            if (!PYTHON_STD_MODULES.includes(name)) {
+              pkgList.push({
+                name,
+                version: null,
+                scope: compScope,
+                properties: [
+                  {
+                    name: "cdx:pypi:versionSpecifiers",
+                    value: versionSpecifiers
+                  }
+                ]
+              });
+            }
           }
         }
       }
-    }
-  });
+    });
   return await getPyMetadata(pkgList, fetchDepsInfo);
 }
 

diff --git a/utils.test.js b/utils.test.js
@@ -2261,6 +2261,25 @@ test("parse requirements.txt", async () => {
     version: "8.6.2",
     scope: "required"
   });
+  deps = await parseReqFile(
+    readFileSync("./test/data/chen-science-requirements.txt", {
+      encoding: "utf-8"
+    }),
+    false
+  );
+  expect(deps.length).toEqual(87);
+  expect(deps[0]).toEqual({
+    name: "aiofiles",
+    version: "23.2.1",
+    scope: undefined,
+    properties: [
+      {
+        name: "cdx:pip:markers",
+        value:
+          'python_full_version >= "3.8.1" and python_version < "3.12" --hash=sha256:19297512c647d4b27a2cf7c34caa7e405c0d60b5560618a29a9fe027b18b0107 --hash=sha256:84ec2218d8419404abcb9f0c02df3f34c6e0a68ed41072acfb1cef5cbc29051a'
+      }
+    ]
+  });
 });
 
 test("parse pyproject.toml", async () => {