Auto tagging

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
AppThreat · Oct 3, 2023 · fbcf10b · fbcf10b
1 parent 507bd24
commit fbcf10b
Show file tree

Hide file tree

Showing 12 changed files with 103 additions and 44 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.0.10"
+ThisBuild / version      := "0.0.11"
 ThisBuild / scalaVersion := "3.3.1"
 
 val cpgVersion = "1.4.22"

diff --git a/console/src/main/scala/io/appthreat/console/Console.scala b/console/src/main/scala/io/appthreat/console/Console.scala
@@ -412,14 +412,12 @@ class Console[T <: Project](loader: WorkspaceLoader[T], baseDir: File = File.cur
       workspace.deleteProject(name)
     }
 
-    if (enhance) {
-      cpgOpt
-        .filter(_.metaData.hasNext)
-        .foreach { cpg =>
-          applyDefaultOverlays(cpg)
-          applyPostProcessingPasses(cpg)
-        }
-    }
+    cpgOpt
+      .filter(_.metaData.hasNext)
+      .foreach { cpg =>
+        if (enhance) applyDefaultOverlays(cpg)
+        applyPostProcessingPasses(cpg)
+      }
     cpgOpt
   }
 

diff --git a/console/src/main/scala/io/appthreat/console/cpgcreation/AtomGenerator.scala b/console/src/main/scala/io/appthreat/console/cpgcreation/AtomGenerator.scala
@@ -2,7 +2,7 @@ package io.appthreat.console.cpgcreation
 
 import io.appthreat.console.FrontendConfig
 import io.shiftleft.codepropertygraph.Cpg
-
+import io.appthreat.x2cpg.passes.taggers.{CdxPass, ChennaiTagsPass}
 import java.nio.file.Path
 import scala.util.Try
 
@@ -20,6 +20,8 @@ case class AtomGenerator(config: FrontendConfig, rootPath: Path, language: Strin
   override def isAvailable: Boolean = true
 
   override def applyPostProcessingPasses(atom: Cpg): Cpg = {
+    new CdxPass(atom).createAndApply()
+    new ChennaiTagsPass(atom).createAndApply()
     atom
   }
 

diff --git a/dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/language/Path.scala b/dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/language/Path.scala
@@ -38,14 +38,19 @@ object Path {
     var caption = ""
     if (path.elements.size > 2) {
       val srcNode  = path.elements.head
+      val srcTags  = if (srcNode.tag.nonEmpty) srcNode.tag.filterNot(_.name == "purl").mkString(", ") else ""
       val sinkNode = path.elements.last
       var sinkCode = sinkNode.code
+      val sinkTags = if (sinkNode.tag.nonEmpty) sinkNode.tag.filterNot(_.name == "purl").mkString(", ") else ""
       sinkNode match {
         case cfgNode: CfgNode =>
           val method = cfgNode.method
           sinkCode = method.fullName
       }
-      caption = s"Source: ${srcNode.code}\nSink: ${sinkCode}\n"
+      caption = s"Source: ${srcNode.code}"
+      if (srcTags.nonEmpty) caption += s"\nSource Tags: ${srcTags}"
+      caption += s"\nSink: ${sinkCode}\n"
+      if (sinkTags.nonEmpty) caption += s"Sink Tags: ${sinkTags}\n"
     }
     val tableRows  = ArrayBuffer[Array[String]]()
     val addedPaths = Set[String]()
@@ -54,20 +59,31 @@ object Path {
       val lineNumber   = astNode.lineNumber.getOrElse("").toString
       val fileName     = astNode.file.name.headOption.getOrElse("").replace("<unknown>", "")
       var fileLocation = s"${fileName}#${lineNumber}"
+      var tags: String = if (astNode.tag.nonEmpty) astNode.tag.filterNot(_.name == "purl").name.mkString(", ") else ""
       if (fileLocation == "#") fileLocation = "N/A"
       astNode match {
         case methodParameterIn: MethodParameterIn =>
           val methodName = methodParameterIn.method.name
+          if (tags.isEmpty && methodParameterIn.method.tag.nonEmpty) {
+            tags = methodParameterIn.method.tag.filterNot(_.name == "purl").name.mkString(", ")
+          }
+          if (tags.isEmpty && methodParameterIn.tag.nonEmpty) {
+            tags = methodParameterIn.tag.filterNot(_.name == "purl").name.mkString(", ")
+          }
           tableRows += Array[String](
             "methodParameterIn",
             fileLocation,
             methodName,
             s"[bold red]${methodParameterIn.name}[/bold red]",
             methodParameterIn.method.fullName + (if (methodParameterIn.method.isExternal) " :right_arrow_curving_up:"
-                                                 else "")
+                                                 else ""),
+            tags
           )
         case identifier: Identifier =>
           val methodName = identifier.method.name
+          if (tags.isEmpty && identifier.inCall.nonEmpty && identifier.inCall.head.tag.nonEmpty) {
+            tags = identifier.inCall.head.tag.filterNot(_.name == "purl").name.mkString(", ")
+          }
           if (!addedPaths.contains(s"${fileName}#${lineNumber}")) {
             tableRows += Array[String](
               "identifier",
@@ -76,14 +92,23 @@ object Path {
               identifier.name,
               if (identifier.inCall.nonEmpty)
                 identifier.inCall.head.code
-              else identifier.code
+              else identifier.code,
+              tags
             )
           }
         case member: Member =>
           val methodName = "<not-in-method>"
-          tableRows += Array[String]("member", fileLocation, methodName, nodeType, member.name, member.code)
+          tableRows += Array[String]("member", fileLocation, methodName, nodeType, member.name, member.code, tags)
         case call: Call =>
           if (!call.code.startsWith("<operator") || !call.methodFullName.startsWith("<operator")) {
+            if (
+              tags.isEmpty && call.callee(NoResolve).head.isExternal && !call.methodFullName.startsWith(
+                "<operator"
+              ) && !call.name
+                .startsWith("<operator") && !call.methodFullName.startsWith("new ")
+            ) {
+              tags = call.callee(NoResolve).head.tag.filterNot(_.name == "purl").name.mkString(", ")
+            }
             var callIcon =
               if (
                 call.callee(NoResolve).head.isExternal && !call.name
@@ -96,20 +121,31 @@ object Path {
               fileLocation,
               call.method.name,
               call.code,
-              call.methodFullName + callIcon
+              call.methodFullName + callIcon,
+              tags
             )
           }
         case cfgNode: CfgNode =>
-          val method     = cfgNode.method
+          val method = cfgNode.method
+          if (tags.isEmpty && method.tag.nonEmpty) {
+            tags = method.tag.filterNot(_.name == "purl").name.mkString(", ")
+          }
           val methodName = method.name
           val statement = cfgNode match {
             case _: MethodParameterIn =>
+              if (tags.isEmpty && method.parameter.tag.nonEmpty) {
+                tags = method.parameter.tag.filterNot(_.name == "purl").name.mkString(", ")
+              }
               val paramsPretty = method.parameter.toList.sortBy(_.index).map(_.code).mkString(", ")
               s"$methodName($paramsPretty)"
-            case _ => cfgNode.statement.repr
+            case _ =>
+              if (tags.isEmpty && cfgNode.statement.tag.nonEmpty) {
+                tags = cfgNode.statement.tag.filterNot(_.name == "purl").name.mkString(", ")
+              }
+              cfgNode.statement.repr
           }
           val tracked = StringUtils.normalizeSpace(StringUtils.abbreviate(statement, maxTrackedWidth))
-          tableRows += Array[String]("cfgNode", fileLocation, methodName, "", tracked)
+          tableRows += Array[String]("cfgNode", fileLocation, methodName, "", tracked, tags)
       }
       addedPaths += s"${fileName}#${lineNumber}"
     }
@@ -130,7 +166,9 @@ object Path {
       {
         val end_section         = row.head == "call"
         val trow: Array[String] = row.tail
-        table.add_row(trow(0), trow(1), trow(2), trow(3), end_section = end_section)
+        val tagsStr: String     = if (trow(4).nonEmpty) s"Tags: ${trow(4)}" else ""
+        val methodStr           = s"${trow(1)}\n${tagsStr}"
+        table.add_row(trow(0), methodStr.stripMargin, trow(2), trow(3), end_section = end_section)
       }
     }
     richConsole.print(table)

diff --git a/...orm/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/ConfigFileCreationPass.scala b/...orm/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/ConfigFileCreationPass.scala
@@ -19,7 +19,7 @@ class ConfigFileCreationPass(cpg: Cpg) extends XConfigFileCreationPass(cpg) {
     extensionFilter(".build"),
     pathEndFilter("CMakeLists.txt"),
     pathEndFilter("bom.json"),
-    pathEndFilter(".chennai.json"),
+    pathEndFilter("chennai.json"),
     pathEndFilter("setup.cfg"),
     pathEndFilter("setup.py")
   )

diff --git a/platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/JavaSrc2Cpg.scala b/platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/JavaSrc2Cpg.scala
@@ -11,7 +11,6 @@ import io.appthreat.javasrc2cpg.passes.{
 import io.appthreat.x2cpg.X2Cpg.withNewEmptyCpg
 import io.appthreat.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass, XTypeRecoveryConfig}
 import io.appthreat.x2cpg.X2CpgFrontend
-import io.appthreat.x2cpg.passes.taggers.CdxPass
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.Languages
 import io.shiftleft.passes.CpgPassBase
@@ -36,7 +35,6 @@ class JavaSrc2Cpg extends X2CpgFrontend[Config] {
         TypeNodePass.withRegisteredTypes(astCreationPass.global.usedTypes.keys().asScala.toList, cpg).createAndApply()
         new TypeInferencePass(cpg).createAndApply()
       }
-      new CdxPass(cpg).createAndApply()
     }
   }
 

diff --git a/...s/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/passes/ConfigFileCreationPass.scala b/...s/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/passes/ConfigFileCreationPass.scala
@@ -36,7 +36,7 @@ class ConfigFileCreationPass(cpg: Cpg) extends XConfigFileCreationPass(cpg) {
     pathEndFilter("AndroidManifest.xml"),
     // Bom
     pathEndFilter("bom.json"),
-    pathEndFilter(".chennai.json")
+    pathEndFilter("chennai.json")
   )
 
   private def mybatisFilter(file: File): Boolean = {

diff --git a/...nds/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/passes/ConfigFileCreationPass.scala b/...nds/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/passes/ConfigFileCreationPass.scala
@@ -31,7 +31,7 @@ class ConfigFileCreationPass(cpg: Cpg) extends XConfigFileCreationPass(cpg) {
     pathEndFilter("AndroidManifest.xml"),
     // Bom
     pathEndFilter("bom.json"),
-    pathEndFilter(".chennai.json")
+    pathEndFilter("chennai.json")
   )
 
 }
diff --git a/...rm/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/ConfigFileCreationPass.scala b/...rm/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/ConfigFileCreationPass.scala
@@ -16,7 +16,7 @@ class ConfigFileCreationPass(cpg: Cpg, requirementsTxt: String = "requirement.tx
     extensionFilter(".yaml"),
     extensionFilter(".lock"),
     pathEndFilter("bom.json"),
-    pathEndFilter(".chennai.json"),
+    pathEndFilter("chennai.json"),
     pathEndFilter("setup.cfg"),
     // Requirements.txt
     pathEndFilter(requirementsTxt)

diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala
@@ -8,6 +8,8 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewNamespace
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.*
 
+import java.util.regex.Pattern
+
 /** Creates tags on typeDecl and call nodes based on a cdx document
   */
 class CdxPass(cpg: Cpg) extends CpgPass(cpg) {
@@ -106,18 +108,21 @@ class CdxPass(cpg: Cpg) extends CpgPass(cpg) {
           val nsstr = ns.hcursor.downField("value").as[String].getOrElse("")
           nsstr.split("\n").foreach { (pkg: String) =>
             val bpkg = pkg.takeWhile(_ != '$')
-            cpg.typeDecl.fullNameExact(bpkg).newTagNodePair("purl", compPurl).store()(dstGraph)
             cpg.call.typeFullNameExact(bpkg).newTagNodePair("purl", compPurl).store()(dstGraph)
             cpg.method.parameter.typeFullNameExact(bpkg).newTagNodePair("purl", compPurl).store()(dstGraph)
+            if (!bpkg.contains("[") && !bpkg.contains("*"))
+              cpg.method.fullName(s"${Pattern.quote(bpkg)}.*").newTagNodePair("purl", compPurl).store()(dstGraph)
             if (compType != "library") {
-              cpg.typeDecl.fullNameExact(bpkg).newTagNode(compType).store()(dstGraph)
               cpg.call.typeFullNameExact(bpkg).newTagNode(compType).store()(dstGraph)
               cpg.method.parameter.typeFullNameExact(bpkg).newTagNode(compType).store()(dstGraph)
+              if (!bpkg.contains("[") && !bpkg.contains("*"))
+                cpg.method.fullName(s"${Pattern.quote(bpkg)}.*").newTagNode(compType).store()(dstGraph)
             }
             descTags.foreach { t =>
-              cpg.typeDecl.fullNameExact(bpkg).newTagNode(t).store()(dstGraph)
               cpg.call.typeFullNameExact(bpkg).newTagNode(t).store()(dstGraph)
               cpg.method.parameter.typeFullNameExact(bpkg).newTagNode(t).store()(dstGraph)
+              if (!bpkg.contains("[") && !bpkg.contains("*"))
+                cpg.method.fullName(s"${Pattern.quote(bpkg)}.*").newTagNode(t).store()(dstGraph)
             }
           }
         }

diff --git a/...rm/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala b/...rm/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala
@@ -7,39 +7,57 @@ import io.shiftleft.codepropertygraph.generated.EdgeTypes
 import io.shiftleft.codepropertygraph.generated.nodes.NewNamespace
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.*
+import java.util.regex.Pattern
 
 /** Creates tags on any node
   */
 class ChennaiTagsPass(cpg: Cpg) extends CpgPass(cpg) {
 
   override def run(dstGraph: DiffGraphBuilder): Unit = {
-    cpg.configFile(".chennai.json").content.foreach { cdxData =>
+    cpg.configFile("chennai.json").content.foreach { cdxData =>
       val ctagsJson       = parse(cdxData).getOrElse(Json.Null)
       val cursor: HCursor = ctagsJson.hcursor
       val tags            = cursor.downField("tags").focus.flatMap(_.asArray).getOrElse(Vector.empty)
       tags.foreach { comp =>
         val tagName    = comp.hcursor.downField("name").as[String].getOrElse("")
-        val tagParams  = comp.hcursor.downField("parameters").downArray.as[String]
-        val tagMethods = comp.hcursor.downField("methods").downArray.as[String]
-        val tagTypes   = comp.hcursor.downField("types").downArray.as[String]
-        val tagFiles   = comp.hcursor.downField("files").downArray.as[String]
+        val tagParams  = comp.hcursor.downField("parameters").focus.flatMap(_.asArray).getOrElse(Vector.empty)
+        val tagMethods = comp.hcursor.downField("methods").focus.flatMap(_.asArray).getOrElse(Vector.empty)
+        val tagTypes   = comp.hcursor.downField("types").focus.flatMap(_.asArray).getOrElse(Vector.empty)
+        val tagFiles   = comp.hcursor.downField("files").focus.flatMap(_.asArray).getOrElse(Vector.empty)
         tagParams.foreach { paramName =>
-          cpg.method.parameter.typeFullNameExact(paramName).newTagNode(tagName).store()(dstGraph)
-          cpg.method.parameter.typeFullName(paramName).newTagNode(tagName).store()(dstGraph)
+          val pn = paramName.asString.getOrElse("")
+          if (pn.nonEmpty) {
+            cpg.method.parameter.typeFullNameExact(pn).newTagNode(tagName).store()(dstGraph)
+            if (!pn.contains("[") && !pn.contains("*"))
+              cpg.method.parameter.typeFullName(s".*${Pattern.quote(pn)}.*").newTagNode(tagName).store()(dstGraph)
+          }
         }
         tagMethods.foreach { methodName =>
-          cpg.method.fullNameExact(methodName).newTagNode(tagName).store()(dstGraph)
-          cpg.method.fullName(methodName).newTagNode(tagName).store()(dstGraph)
-          cpg.call.typeFullNameExact(methodName).newTagNode(tagName).store()(dstGraph)
+          val mn = methodName.asString.getOrElse("")
+          if (mn.nonEmpty) {
+            cpg.method.fullNameExact(mn).newTagNode(tagName).store()(dstGraph)
+            if (!mn.contains("[") && !mn.contains("*"))
+              cpg.method.fullName(s".*${Pattern.quote(mn)}.*").newTagNode(tagName).store()(dstGraph)
+          }
         }
         tagTypes.foreach { typeName =>
-          cpg.typeDecl.fullNameExact(typeName).newTagNode(tagName).store()(dstGraph)
-          cpg.typeDecl.fullName(typeName).newTagNode(tagName).store()(dstGraph)
-          cpg.call.typeFullNameExact(typeName).newTagNode(tagName).store()(dstGraph)
+          val tn = typeName.asString.getOrElse("")
+          if (tn.nonEmpty) {
+            cpg.method.parameter.typeFullNameExact(tn).newTagNode(tagName).store()(dstGraph)
+            if (!tn.contains("[") && !tn.contains("*"))
+              cpg.method.parameter.typeFullName(s".*${Pattern.quote(tn)}.*").newTagNode(tagName).store()(dstGraph)
+            cpg.call.typeFullNameExact(tn).newTagNode(tagName).store()(dstGraph)
+            if (!tn.contains("[") && !tn.contains("*"))
+              cpg.call.typeFullName(s".*${Pattern.quote(tn)}.*").newTagNode(tagName).store()(dstGraph)
+          }
         }
         tagFiles.foreach { fileName =>
-          cpg.file.nameExact(fileName).newTagNode(tagName).store()(dstGraph)
-          cpg.file.name(fileName).newTagNode(tagName).store()(dstGraph)
+          val fn = fileName.asString.getOrElse("")
+          if (fn.nonEmpty) {
+            cpg.file.nameExact(fn).newTagNode(tagName).store()(dstGraph)
+            if (!fn.contains("[") && !fn.contains("*"))
+              cpg.file.name(s".*${Pattern.quote(fn)}.*").newTagNode(tagName).store()(dstGraph)
+          }
         }
       }
     }

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "appthreat-chen"
-version = "0.0.10"
+version = "0.0.11"
 description = "Code Hierarchy Exploration Net (chen)"
 authors = ["Team AppThreat <cloud@appthreat.com>"]
 license = "Apache-2.0"