parsedeps improvements (#146)

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

build.sbt

@@ -1,6 +1,6 @@
 name                     := "atom"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "2.0.12"
+ThisBuild / version      := "2.0.13"
 ThisBuild / scalaVersion := "3.4.1"
 
 val chenVersion      = "2.0.11"

codemeta.json

@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/atom",
   "issueTracker": "https://github.com/AppThreat/atom/issues",
   "name": "atom",
-  "version": "2.0.12",
+  "version": "2.0.13",
   "description": "Atom is a novel intermediate representation for next-generation code analysis.",
   "applicationCategory": "code-analysis",
   "keywords": [

src/main/scala/io/appthreat/atom/Atom.scala

@@ -184,7 +184,7 @@ object Atom:
             )
         cmd("parsedeps")
             .text("Extract dependencies from the build file and imports")
-            .action((_, *) => AtomParseDepsConfig().withRemoveAtom(true))
+            .action((_, *) => AtomParseDepsConfig().withRemoveAtom(false))
         cmd("data-flow")
             .text("Extract backward data-flow slices")
             .action((_, _) => AtomDataFlowConfig().withDataDependencies(true))

src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala

@@ -25,13 +25,17 @@ object PythonDependencyParser extends XDependencyParser:
 
     private def parseSetupPy(cpg: Cpg): Set[ModuleWithVersion] =
         val dataFlowEnabled     = cpg.metaData.overlays.contains(OssDataFlow.overlayName)
-        val requirementsPattern = """([\[\]/.\w_-]+)\s?((=>|<=|==|>=|=<|<|>|!=).*)""".r
+        val requirementsPattern = """([\[\]/.\w_-]+)\s?((=>|<=|==|>=|=<|<|>|!=|~=).*)""".r
 
         def dataSourcesToRequires = (cpg.literal ++ cpg.identifier)
             .where(_.file.name(".*setup.py"))
             .where(_.argumentName("install_requires"))
             .collectAll[CfgNode]
 
+        def installRequires = cpg.call.where(_.file.name(".*setup.py")).where(_.argumentName(
+          "install_requires"
+        )).argument.collectAll[Literal]
+
         def setupCall = cpg.call("setup").where(_.file.name(".*setup.py"))
 
         def findOriginalDeclaration(xs: Traversal[CfgNode]): Iterable[Literal] =
@@ -52,10 +56,11 @@ object PythonDependencyParser extends XDependencyParser:
                 .to(Iterable)
 
         val initialTraversal = if dataFlowEnabled then setupCall.reachableBy(dataSourcesToRequires)
-        else dataSourcesToRequires
+        else (dataSourcesToRequires ++ installRequires)
         findOriginalDeclaration(initialTraversal)
             .map(x => X2Cpg.stripQuotes(x.code))
             .map {
+
                 case requirementsPattern(name, versionSpecifiers, _)
                     if versionSpecifiers.contains("==") =>
                     val versions     = versionSpecifiers.split(',').toSeq

src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala

@@ -67,7 +67,8 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa
         |    "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*",
         |    "proto-plus >= 1.22.0, <2.0.0dev",
         |    "proto-plus >= 1.22.2, <2.0.0dev; python_version>='3.11'",
-        |    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5"
+        |    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
+        |    "dbt-core~=1.7,<1.8"
         |]
         |test_requirements = [
         |    "pytest-httpbin==2.0.0",
@@ -205,6 +206,7 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa
         ModuleWithVersion("PickyThing", "2.4c1", "<1.6,>1.9,!=1.9.6,<2.0a0", ""),
         ModuleWithVersion("certifi", "", ">=2017.4.17", ""),
         ModuleWithVersion("charset_normalizer", "", ">=2,<4", ""),
+        ModuleWithVersion("dbt-core", "", "~=1.7,<1.8", ""),
         ModuleWithVersion("google-api-core[grpc]", "", ">= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", ""),
         ModuleWithVersion("idna", "", ">=2.5,<4", ""),
         ModuleWithVersion("os", "", "", "os.path"),
@@ -221,4 +223,66 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa
     }
   }
 
+    "dependencies from the `impacket` library" should {
+        lazy val cpg = code(
+            """
+              |#!/usr/bin/env python
+              |# $Id$
+              |
+              |import glob
+              |import os
+              |import platform
+              |
+              |from setuptools import setup
+              |
+              |PACKAGE_NAME = "impacket2"
+              |
+              |if platform.system() != 'Darwin':
+              |    data_files = [(os.path.join('share', 'doc', PACKAGE_NAME), ['README.md', 'LICENSE']+glob.glob('doc/*'))]
+              |else:
+              |    data_files = []
+              |
+              |def read(fname):
+              |    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+              |
+              |setup(name = PACKAGE_NAME,
+              |      version = "0.9.21-dev",
+              |      package_dir={'': 'src'},
+              |      platforms = ["Unix"],
+              |      packages=['impacket2', 'impacket2.dcerpc', 'impacket2.examples', 'impacket2.dcerpc.v5', 'impacket2.dcerpc.v5.dcom',
+              |                'impacket2.krb5', 'impacket2.ldap', 'impacket2.examples.ntlmrelayx',
+              |                'impacket2.examples.ntlmrelayx.clients', 'impacket2.examples.ntlmrelayx.servers',
+              |                'impacket2.examples.ntlmrelayx.servers.socksplugins', 'impacket2.examples.ntlmrelayx.utils',
+              |                'impacket2.examples.ntlmrelayx.attacks'],
+              |      data_files = data_files,
+              |      install_requires=['pyasn1>=0.2.3', 'pycryptodomex', 'pyOpenSSL>=0.13.1', 'six', 'ldap3==2.5.1', 'ldapdomaindump>=0.9.0', 'flask>=1.0'],
+              |      extras_require={
+              |                      'pyreadline:sys_platform=="win32"': [],
+              |                      'python_version<"2.7"': [ 'argparse' ],
+              |                    },
+              |      classifiers = [
+              |          "Programming Language :: Python :: 3.6",
+              |          "Programming Language :: Python :: 2.7",
+              |          "Programming Language :: Python :: 2.6",
+              |      ]
+              |)
+              |
+              |""".stripMargin,
+            "setup.py"
+        )
+
+        "have the modules scanned successfully" in {
+            val scanResult = PythonDependencyParser.parse(cpg)
+            scanResult.modules shouldBe List(
+                ModuleWithVersion("flask", "", ">=1.0", ""),
+                ModuleWithVersion("ldap3", "2.5.1", "", ""),
+                ModuleWithVersion("ldapdomaindump", "", ">=0.9.0", ""),
+                ModuleWithVersion("pyOpenSSL", "", ">=0.13.1", ""),
+                ModuleWithVersion("pyasn1", "", ">=0.2.3", ""),
+                ModuleWithVersion("pycryptodomex", "", "", ""),
+                ModuleWithVersion("six", "", "", "")
+            )
+        }
+    }
+
 }

wrapper/nodejs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appthreat/atom",
-  "version": "2.0.12",
+  "version": "2.0.13",
   "description": "Create atom (⚛) representation for your application, packages and libraries",
   "exports": "./index.js",
   "type": "module",
@@ -9,12 +9,12 @@
     "lint": "eslint *.mjs *.js"
   },
   "dependencies": {
-    "@babel/parser": "^7.24.4",
+    "@babel/parser": "^7.24.7",
     "typescript": "^5.4.5",
     "yargs": "^17.7.2"
   },
   "devDependencies": {
-    "eslint": "^8.57.0"
+    "eslint": "8.57.0"
   },
   "bin": {
     "atom": "./index.js",