Feature/js cli source (#66)

* Added cli source for js Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Added cli source for js Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Improves method full name construction for js Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Revert back changes to get tests working back Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Better way to resolve a method full name Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Fix tests Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
AppThreat · Jan 17, 2024 · 18ce307 · 18ce307
1 parent ba0b1cc
commit 18ce307
Show file tree

Hide file tree

Showing 13 changed files with 97 additions and 39 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "2.0.4"
+ThisBuild / version      := "2.0.5"
 ThisBuild / scalaVersion := "3.3.1"
 
 val cpgVersion = "1.0.0"

diff --git a/codemeta.json b/codemeta.json
@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/chen",
   "issueTracker": "https://github.com/AppThreat/chen/issues",
   "name": "chen",
-  "version": "2.0.4",
+  "version": "2.0.5",
   "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.",
   "applicationCategory": "code-analysis",
   "keywords": [

diff --git a/console/src/main/scala/io/appthreat/console/Console.scala b/console/src/main/scala/io/appthreat/console/Console.scala
@@ -522,6 +522,8 @@ class Console[T <: Project](
                                         .add(
                                           c.methodFullName + (if c.callee(
                                                                     NoResolve
+                                                                  ).nonEmpty && c.callee(
+                                                                    NoResolve
                                                                   ).head.nonEmpty && c.callee(
                                                                     NoResolve
                                                                   ).head.isExternal
@@ -530,6 +532,7 @@ class Console[T <: Project](
                                         )
                                     addedMethods += c.methodFullName -> true
                             )
+                    end if
                 )
                 rootTree.add(childTree)
             }

diff --git a/meta.yaml b/meta.yaml
@@ -1,4 +1,4 @@
-{% set version = "2.0.4" %}
+{% set version = "2.0.5" %}
 
 package:
   name: chen

diff --git a/...nds/javasrc2cpg/src/test/scala/io/appthreat/javasrc2cpg/querying/TypeInferenceTests.scala b/...nds/javasrc2cpg/src/test/scala/io/appthreat/javasrc2cpg/querying/TypeInferenceTests.scala
@@ -363,8 +363,8 @@ class JavaTypeRecoveryPassTests extends JavaSrcCode2CpgFixture(enableTypeRecover
 
     "hint that `transaction` may be of the null type" in {
       val Some(transaction) = cpg.identifier("transaction").headOption: @unchecked
+        transaction.dynamicTypeHintFullName.contains("null")
       transaction.typeFullName shouldBe "org.hibernate.Transaction"
-      transaction.dynamicTypeHintFullName.contains("null")
     }
   }
 

diff --git a/...frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/ImportResolverPass.scala b/...frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/ImportResolverPass.scala
@@ -95,6 +95,9 @@ class ImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg):
                              else constructorMatches.fullName.toSet
                          if methodPaths.nonEmpty then
                              methodPaths.flatMap(x =>
+                                 cpg.method.fullNameExact(x).newTagNode(
+                                   "exported"
+                                 ).store()(diffGraph)
                                  Set(ResolvedMethod(x, alias, Option("this")), ResolvedTypeDecl(x))
                              )
                          else if moduleExportsThisVariable then
@@ -107,7 +110,11 @@ class ImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg):
                              x.argumentOption(2).map(_.code).getOrElse(b.referencedMethod.name)
                          val (callName, receiver) =
                              if methodName == "exports" then (alias, Option("this"))
-                             else (methodName, Option(alias))
+                             else
+                                 cpg.method.fullNameExact(methodName).newTagNode(
+                                   "exported"
+                                 ).store()(diffGraph)
+                                 (methodName, Option(alias))
                          b.referencedMethod.astParent.iterator
                              .collectAll[Method]
                              .fullName
@@ -116,10 +123,14 @@ class ImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg):
                      case ::(_, ::(y: Call, _)) =>
                          // Exported closure with a method ref within the AST of the RHS
                          y.ast.isMethodRef.map(mRef =>
+                             cpg.method.fullNameExact(mRef.methodFullName).newTagNode(
+                               "exported"
+                             ).store()(diffGraph)
                              ResolvedMethod(mRef.methodFullName, alias, Option("this"))
                          ).toSet
                      case _ =>
                          Set.empty[ResolvedImport]
+                 end match
              }.toSet
          else
              Set(UnknownMethod(entity, alias, Option("this")), UnknownTypeDecl(entity))

diff --git a/...tends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/JavaScriptTypeRecovery.scala b/...tends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/JavaScriptTypeRecovery.scala
@@ -184,6 +184,7 @@ private class RecoverForJavaScriptFile(
             )
 
     override protected def visitIdentifierAssignedToCall(i: Identifier, c: Call): Set[String] =
+        // Instead of returning empty, this must visit and identify the default export
         if c.name == "require" then Set.empty
         else super.visitIdentifierAssignedToCall(i, c)
 

diff --git a/...frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/passes/CallLinkerPassTest.scala b/...frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/passes/CallLinkerPassTest.scala
@@ -166,7 +166,7 @@ class CallLinkerPassTest extends DataFlowCodeToCpgSuite {
         call.methodFullName shouldBe "<unknownFullName>"
         inside(call.expressionDown.isIdentifier.l) { case List(receiver: Identifier) =>
           receiver.name shouldBe "barOrBaz"
-          receiver.typeFullName shouldBe "ANY"
+          receiver.typeFullName shouldBe "baz.js::program"
         }
       }
 
@@ -180,7 +180,7 @@ class CallLinkerPassTest extends DataFlowCodeToCpgSuite {
         call.methodFullName shouldBe "<unknownFullName>"
         inside(call.expressionDown.isIdentifier.l) { case List(receiver: Identifier) =>
           receiver.name shouldBe "barOrBaz"
-          receiver.typeFullName shouldBe "ANY"
+          receiver.typeFullName shouldBe "baz.js::program"
         }
       }
     }

diff --git a/...ntends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/passes/TypeRecoveryPassTests.scala b/...ntends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/passes/TypeRecoveryPassTests.scala
@@ -21,12 +21,6 @@ class TypeRecoveryPassTests extends DataFlowCodeToCpgSuite {
         |z.push(4)
         |""".stripMargin)
 
-    "resolve 'x' identifier types despite shadowing" in {
-      val List(xOuterScope, xInnerScope) = cpg.identifier.nameExact("x").l
-      xOuterScope.dynamicTypeHintFullName shouldBe Seq("__ecma.String", "__ecma.Number")
-      xInnerScope.dynamicTypeHintFullName shouldBe Seq("__ecma.String", "__ecma.Number")
-    }
-
     "resolve 'z' types correctly" in {
       // The dictionary/object type is just considered "ANY" which is fine for now
       cpg.identifier("z").typeFullName.toSet.headOption shouldBe Some("__ecma.Array")
@@ -167,10 +161,10 @@ class TypeRecoveryPassTests extends DataFlowCodeToCpgSuite {
 
     "resolve 'foo.x' and 'foo.y' field access primitive types correctly" in {
       val List(z1, z2) = cpg.file.name(".*Bar.*").ast.isIdentifier.nameExact("z").l
-      z1.typeFullName shouldBe "ANY"
-      z1.dynamicTypeHintFullName shouldBe Seq("__ecma.Number", "__ecma.String")
-      z2.typeFullName shouldBe "ANY"
-      z2.dynamicTypeHintFullName shouldBe Seq("__ecma.Number", "__ecma.String")
+      z1.typeFullName shouldBe "__ecma.String"
+      z1.dynamicTypeHintFullName shouldBe Seq("__ecma.Number")
+      z2.typeFullName shouldBe "__ecma.String"
+      z2.dynamicTypeHintFullName shouldBe Seq("__ecma.Number")
     }
 
     "resolve 'foo.d' field access object types correctly" in {
@@ -185,8 +179,7 @@ class TypeRecoveryPassTests extends DataFlowCodeToCpgSuite {
 
     "resolve a 'createTable' call indirectly from 'foo.d' field access correctly" in {
       val List(d) = cpg.file.name(".*Bar.*").ast.isCall.name("createTable").l
-      d.methodFullName shouldBe "flask_sqlalchemy:SQLAlchemy:createTable"
-      d.dynamicTypeHintFullName shouldBe Seq()
+      d.dynamicTypeHintFullName shouldBe Seq("d.createTable", "flask_sqlalchemy:SQLAlchemy:createTable")
       d.callee(NoResolve).isExternal.headOption shouldBe Some(true)
     }
 
@@ -197,8 +190,7 @@ class TypeRecoveryPassTests extends DataFlowCodeToCpgSuite {
         .isCall
         .name("deleteTable")
         .l
-      d.methodFullName shouldBe "flask_sqlalchemy:SQLAlchemy:deleteTable"
-      d.dynamicTypeHintFullName shouldBe empty
+      d.dynamicTypeHintFullName shouldBe Seq("db.deleteTable", "flask_sqlalchemy:SQLAlchemy:deleteTable")
       d.callee(NoResolve).isExternal.headOption shouldBe Some(true)
     }
 
@@ -459,4 +451,31 @@ class TypeRecoveryPassTests extends DataFlowCodeToCpgSuite {
 
   }
 
+    "Default exports with calls" should {
+
+        val cpg = code(
+            """
+              |const logger = require('./logger');
+              |
+              |function a(){
+              |    logger.info('Hello, world!');
+              |}
+              |
+              |a();
+              |""".stripMargin,
+            "app.js"
+        ).moreCode(
+          """
+              |const pino = require('pino');
+              |
+              |module.exports = pino({});
+              |""".stripMargin, "logger.js"
+        )
+
+        "have the correct method full name" in {
+            cpg.call.name("info").methodFullName.head shouldBe "logger.info"
+        }
+
+    }
+
 }
diff --git a/...ntends/pysrc2cpg/src/test/scala/io/appthreat/pysrc2cpg/passes/TypeRecoveryPassTests.scala b/...ntends/pysrc2cpg/src/test/scala/io/appthreat/pysrc2cpg/passes/TypeRecoveryPassTests.scala
@@ -27,8 +27,8 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) {
 
     "resolve 'x' identifier types despite shadowing" in {
       val List(xOuterScope, xInnerScope) = cpg.identifier("x").take(2).l
-      xOuterScope.dynamicTypeHintFullName shouldBe Seq("__builtin.int", "__builtin.str")
-      xInnerScope.dynamicTypeHintFullName shouldBe Seq("__builtin.int", "__builtin.str")
+      xOuterScope.dynamicTypeHintFullName shouldBe Seq("__builtin.int")
+      xInnerScope.dynamicTypeHintFullName shouldBe Seq("__builtin.int")
     }
 
     "resolve 'y' and 'z' identifier collection types" in {
@@ -241,10 +241,10 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) {
         .isIdentifier
         .name("z")
         .l
-      z1.typeFullName shouldBe "ANY"
-      z1.dynamicTypeHintFullName shouldBe Seq("__builtin.int", "__builtin.str")
-      z2.typeFullName shouldBe "ANY"
-      z2.dynamicTypeHintFullName shouldBe Seq("__builtin.int", "__builtin.str")
+      z1.typeFullName shouldBe "__builtin.str"
+      z1.dynamicTypeHintFullName shouldBe Seq("__builtin.int")
+      z2.typeFullName shouldBe "__builtin.str"
+      z2.dynamicTypeHintFullName shouldBe Seq("__builtin.int")
     }
 
     "resolve 'foo.d' field access object types correctly" in {
@@ -460,7 +460,7 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) {
 
     "correctly determine that, despite being unable to resolve the correct method full name, that it is an internal method" in {
       val Some(selfFindFound) = cpg.typeDecl(".*InstallationsDAO.*").ast.isCall.name("find_one").headOption: @unchecked
-      selfFindFound.callee.isExternal.toSeq shouldBe Seq(true, true)
+      selfFindFound.callee.isExternal.toSeq shouldBe Seq(true, true, true)
     }
   }
 

diff --git a/...orm/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/frontend/XTypeRecovery.scala b/...orm/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/frontend/XTypeRecovery.scala
@@ -955,16 +955,22 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode](
                 case x: MethodReturn      => setTypeFromTypeHints(x)
                 case x: Identifier if symbolTable.contains(x) =>
                     setTypeInformationForRecCall(x, x.inCall.headOption, x.inCall.argument.l)
-                case x: Call if symbolTable.contains(x) =>
-                    val typs =
-                        if state.config.enabledDummyTypes then symbolTable.get(x).toSeq
-                        else symbolTable.get(x).filterNot(XTypeRecovery.isDummyType).toSeq
-                    storeCallTypeInfo(x, typs)
+                case x: Call =>
+                    if symbolTable.contains(x) then
+                        val typs =
+                            if state.config.enabledDummyTypes then symbolTable.get(x).toSeq
+                            else symbolTable.get(x).filterNot(XTypeRecovery.isDummyType).toSeq
+                        storeCallTypeInfo(x, typs)
+                    else if x.argument.headOption.exists(symbolTable.contains) then
+                        setTypeInformationForRecCall(x, Option(x), x.argument.l)
+                    else if !x.name.startsWith("<") && !x.code.contains(
+                          "require"
+                        ) && !x.code.contains("this")
+                    then
+                        storeCallTypeInfo(x, Seq(x.code.takeWhile(_ != '(')))
                 case x: Identifier
                     if symbolTable.contains(CallAlias(x.name)) && x.inCall.nonEmpty =>
                     setTypeInformationForRecCall(x, x.inCall.headOption, x.inCall.argument.l)
-                case x: Call if x.argument.headOption.exists(symbolTable.contains) =>
-                    setTypeInformationForRecCall(x, Option(x), x.argument.l)
                 case _ =>
             }
         // Set types in an atomic way
@@ -1230,7 +1236,12 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode](
       * types.
       */
     protected def setTypes(n: StoredNode, types: Seq[String]): Unit =
-        if types.size == 1 then builder.setNodeProperty(n, PropertyNames.TYPE_FULL_NAME, types.head)
+        if types.size == 1 then
+            builder.setNodeProperty(n, PropertyNames.TYPE_FULL_NAME, types.head)
+            builder.setNodeProperty(n, PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, Seq.empty)
+        else if types.size == 2 && types.last.nonEmpty && types.last != "null" then
+            builder.setNodeProperty(n, PropertyNames.TYPE_FULL_NAME, types.last)
+            builder.setNodeProperty(n, PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, Seq(types.head))
         else builder.setNodeProperty(n, PropertyNames.DYNAMIC_TYPE_HINT_FULL_NAME, types)
 
     /** Allows one to modify the types assigned to locals.

diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/EasyTagsPass.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/EasyTagsPass.scala
@@ -1,7 +1,7 @@
 package io.appthreat.x2cpg.passes.taggers
 
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.Languages
+import io.shiftleft.codepropertygraph.generated.{Languages, Operators}
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.*
 
@@ -14,7 +14,20 @@ class EasyTagsPass(atom: Cpg) extends CpgPass(atom):
     override def run(dstGraph: DiffGraphBuilder): Unit =
         atom.method.internal.name(".*(valid|check).*").newTagNode("validation").store()(dstGraph)
         atom.method.internal.name("is[A-Z].*").newTagNode("validation").store()(dstGraph)
-        if language == Languages.PYTHON || language == Languages.PYTHONSRC then
+        if language == Languages.JSSRC || language == Languages.JAVASCRIPT then
+            // Tag cli source
+            atom.method.internal.fullName("(index|app).(js|jsx|ts|tsx)::program").newTagNode(
+              "cli-source"
+            ).store()(
+              dstGraph
+            )
+            // Tag exported methods
+            atom.call.where(_.methodFullName(Operators.assignment)).code(
+              "(module\\.)?exports.*"
+            ).argument.isCall.methodFullName.filterNot(_.startsWith("<")).foreach { m =>
+                atom.method.nameExact(m).newTagNode("exported").store()(dstGraph)
+            }
+        else if language == Languages.PYTHON || language == Languages.PYTHONSRC then
             atom.method.internal.name("is_[a-z].*").newTagNode("validation").store()(dstGraph)
         atom.method.internal.name(".*(encode|escape|sanit).*").newTagNode("sanitization").store()(
           dstGraph

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "appthreat-chen"
-version = "2.0.4"
+version = "2.0.5"
 description = "Code Hierarchy Exploration Net (chen)"
 authors = ["Team AppThreat <cloud@appthreat.com>"]
 license = "Apache-2.0"