Python

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
AppThreat · Oct 24, 2023 · b4162ef · b4162ef
1 parent 4690502
commit b4162ef
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 7 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.0.21"
+ThisBuild / version      := "0.5.0"
 ThisBuild / scalaVersion := "3.3.1"
 
 val cpgVersion = "1.4.22"

diff --git a/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/Jimple2Cpg.scala b/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/Jimple2Cpg.scala
@@ -8,7 +8,6 @@ import io.appthreat.x2cpg.X2Cpg.withNewEmptyCpg
 import io.appthreat.x2cpg.X2CpgFrontend
 import io.appthreat.x2cpg.datastructures.Global
 import io.appthreat.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass}
-import io.appthreat.x2cpg.passes.taggers.CdxPass
 import io.shiftleft.codepropertygraph.Cpg
 import org.slf4j.LoggerFactory
 import soot.options.Options
@@ -120,7 +119,6 @@ class Jimple2Cpg extends X2CpgFrontend[Config] {
       .createAndApply()
     DeclarationRefPass(cpg).createAndApply()
     new ConfigFileCreationPass(cpg).createAndApply()
-    new CdxPass(cpg).createAndApply()
   }
 
   override def createCpg(config: Config): Try[Cpg] =

diff --git a/platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/PythonAstVisitor.scala b/platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/PythonAstVisitor.scala
@@ -2019,7 +2019,9 @@ object PythonAstVisitor {
   // There is a corresponding list in policies which needs to be updated if this one is updated and vice versa.
   val builtinFunctionsV3: Iterable[String] = Iterable(
     "abs",
+    "aiter",
     "all",
+    "anext",
     "any",
     "ascii",
     "bin",
@@ -2051,6 +2053,7 @@ object PythonAstVisitor {
     "locals",
     "map",
     "max",
+    "memoryview",
     "min",
     "next",
     "oct",

diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala
@@ -7,6 +7,7 @@ import io.shiftleft.codepropertygraph.generated.Languages
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.*
 
+import java.io.File
 import java.util.regex.Pattern
 import scala.collection.mutable
 import scala.io.Source
@@ -45,6 +46,16 @@ class CdxPass(atom: Cpg) extends CpgPass(atom) {
 
   private val BOM_JSON_FILE = ".*(bom|cdx).json"
 
+  private def toPyModuleForm(str: String) = {
+    if (str.nonEmpty && str.count(_ == '.') > 0) {
+      s".*${str.split("\\.").take(2).mkString(Pattern.quote(File.separator))}.*"
+    } else if (str.nonEmpty) {
+      s"$str${Pattern.quote(File.separator)}.*"
+    } else {
+      str
+    }
+  }
+
   override def run(dstGraph: DiffGraphBuilder): Unit = {
     atom.configFile.name(BOM_JSON_FILE).content.foreach { cdxData =>
       val cdxJson         = parse(cdxData).getOrElse(Json.Null)
@@ -72,12 +83,19 @@ class CdxPass(atom: Cpg) extends CpgPass(atom) {
             .filterNot(_.contains("mock"))
             .filterNot(_.endsWith(".lock"))
             .filterNot(_.endsWith(".json"))
+            .filterNot(_.endsWith(".txt"))
             .foreach { (pkg: String) =>
               var bpkg = pkg.takeWhile(_ != '$')
-              if (language == Languages.JAVA || language == Languages.JAVASRC)
+              if (language == Languages.JAVA || language == Languages.JAVASRC) {
                 bpkg = bpkg.split("\\.").take(PKG_NS_SIZE).mkString(".").concat(".*")
-              if (language == Languages.JSSRC || language == Languages.JAVASCRIPT) bpkg = s".*${bpkg}.*"
-              if (!donePkgs.contains(bpkg)) {
+                bpkg = bpkg.replace(File.separator, Pattern.quote(File.separator))
+              }
+              if (language == Languages.JSSRC || language == Languages.JAVASCRIPT) {
+                bpkg = s".*${bpkg}.*"
+                bpkg = bpkg.replace(File.separator, Pattern.quote(File.separator))
+              }
+              if (language == Languages.PYTHON || language == Languages.PYTHONSRC) bpkg = toPyModuleForm(bpkg)
+              if (bpkg.nonEmpty && !donePkgs.contains(bpkg)) {
                 donePkgs.put(bpkg, true)
                 if (!containsRegex(bpkg)) {
                   atom.call.typeFullNameExact(bpkg).newTagNode(compPurl).store()(dstGraph)

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "appthreat-chen"
-version = "0.0.21"
+version = "0.5.0"
 description = "Code Hierarchy Exploration Net (chen)"
 authors = ["Team AppThreat <cloud@appthreat.com>"]
 license = "Apache-2.0"