Remove naive call linker. Use routes in-out tagger

build.sbt

@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.5.2"
+ThisBuild / version      := "0.5.3"
 ThisBuild / scalaVersion := "3.3.1"
 
 val cpgVersion = "1.4.22"

console/src/main/scala/io/appthreat/console/cpgcreation/PythonSrcCpgGenerator.scala

@@ -1,20 +1,9 @@
 package io.appthreat.console.cpgcreation
 
 import io.appthreat.console.FrontendConfig
-import io.appthreat.pysrc2cpg.{
-  DynamicTypeHintFullNamePass,
-  ImportResolverPass,
-  ImportsPass,
-  NewMain,
-  Py2CpgOnFileSystemConfig,
-  PythonInheritanceNamePass,
-  PythonTypeHintCallLinker,
-  PythonTypeRecoveryPass
-}
-import io.appthreat.console.FrontendConfig
+import io.appthreat.pysrc2cpg.*
 import io.appthreat.x2cpg.X2Cpg
 import io.appthreat.x2cpg.passes.base.AstLinkerPass
-import io.appthreat.x2cpg.passes.callgraph.NaiveCallLinker
 import io.appthreat.x2cpg.passes.frontend.XTypeRecoveryConfig
 import io.shiftleft.codepropertygraph.Cpg
 
@@ -46,7 +35,6 @@ case class PythonSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends
       case None         => XTypeRecoveryConfig()
     new PythonTypeRecoveryPass(cpg, typeRecoveryConfig).createAndApply()
     new PythonTypeHintCallLinker(cpg).createAndApply()
-    new NaiveCallLinker(cpg).createAndApply()
 
     // Some of passes above create new methods, so, we
     // need to run the ASTLinkerPass one more time

platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/JsSrc2Cpg.scala

@@ -22,7 +22,6 @@ import io.appthreat.jssrc2cpg.passes.{
 import io.appthreat.jssrc2cpg.passes.*
 import io.appthreat.x2cpg.X2Cpg.withNewEmptyCpg
 import io.appthreat.x2cpg.X2CpgFrontend
-import io.appthreat.x2cpg.passes.callgraph.NaiveCallLinker
 import io.appthreat.x2cpg.passes.frontend.XTypeRecoveryConfig
 import io.appthreat.x2cpg.utils.{HashUtil, Report}
 import io.shiftleft.codepropertygraph.Cpg
@@ -80,8 +79,7 @@ object JsSrc2Cpg {
       new ConstClosurePass(cpg),
       new ImportResolverPass(cpg),
       new JavaScriptTypeRecoveryPass(cpg, typeRecoveryConfig),
-      new JavaScriptTypeHintCallLinker(cpg),
-      new NaiveCallLinker(cpg)
+      new JavaScriptTypeHintCallLinker(cpg)
     )
   }
 

platform/frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/passes/TypeRecoveryPassTests.scala

@@ -239,11 +239,6 @@ class TypeRecoveryPassTests extends DataFlowCodeToCpgSuite {
       x.methodFullName shouldBe "util.js::program:getIncrementalInteger"
     }
 
-    "resolve the method full name off of the imported 'util'" in {
-      val List(x) = cpg.file("foo.js").ast.isCall.nameExact("getIncrementalInteger").l
-      x.methodFullName shouldBe "util.js::program:getIncrementalInteger"
-    }
-
     "resolve the full name of the currying from the closure" in {
       val List(x) = cpg.file("util.js").ast.isCall.nameExact("anonymous").lineNumber(4).l
       x.name shouldBe "anonymous"

platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/CodeToCpg.scala

@@ -26,7 +26,7 @@ class CodeToCpg(cpg: Cpg, inputProvider: Iterable[InputProvider], schemaValidati
       diffGraph.absorb(astVisitor.getDiffGraph)
     } catch {
       case exception: Throwable =>
-        logger.warn(s"Failed to convert file ${inputPair.relFileName}", exception)
+        logger.debug(s"Failed to convert file ${inputPair.relFileName}", exception)
         Iterator.empty
     }
   }

platform/frontends/pysrc2cpg/src/test/scala/io/appthreat/pysrc2cpg/PySrc2CpgFixture.scala

@@ -5,7 +5,6 @@ import io.appthreat.dataflowengineoss.queryengine.EngineContext
 import io.appthreat.dataflowengineoss.semanticsloader.FlowSemantic
 import io.appthreat.x2cpg.X2Cpg
 import io.appthreat.x2cpg.passes.base.AstLinkerPass
-import io.appthreat.x2cpg.passes.callgraph.NaiveCallLinker
 import io.appthreat.x2cpg.testfixtures.{Code2CpgFixture, LanguageFrontend, TestCpg}
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve}
@@ -41,7 +40,6 @@ class PySrcTestCpg extends TestCpg with PythonFrontend {
     new DynamicTypeHintFullNamePass(this).createAndApply()
     new PythonTypeRecoveryPass(this).createAndApply()
     new PythonTypeHintCallLinker(this).createAndApply()
-    new NaiveCallLinker(this).createAndApply()
 
     // Some of passes above create new methods, so, we
     // need to run the ASTLinkerPass one more time

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/base/AstLinkerPass.scala

@@ -35,7 +35,7 @@ class AstLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil {
         case NodeTypes.TYPE_DECL       => typeDeclFullNameToNode(cpg, astParentFullName)
         case NodeTypes.NAMESPACE_BLOCK => namespaceBlockFullNameToNode(cpg, astParentFullName)
         case _ =>
-          logger.warn(
+          logger.debug(
             s"Invalid AST_PARENT_TYPE=$astParentFullName;" +
               s" astChild LABEL=${astChild.label};" +
               s" astChild FULL_NAME=$astChildFullName"

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/base/MethodDecoratorPass.scala

@@ -34,13 +34,13 @@ class MethodDecoratorPass(cpg: Cpg) extends CpgPass(cpg) {
 
         val method = parameterIn.astIn.headOption
         if (method.isEmpty) {
-          logger.warn("Parameter without method encountered: " + parameterIn.toString)
+          logger.debug("Parameter without method encountered: " + parameterIn.toString)
         } else {
           if (parameterIn.typeFullName == null) {
             val evalType = parameterIn.typ
             dstGraph.addEdge(parameterOut, evalType, EdgeTypes.EVAL_TYPE)
             if (!loggedMissingTypeFullName) {
-              logger.warn("Using deprecated CPG format with missing TYPE_FULL_NAME on METHOD_PARAMETER_IN nodes.")
+              logger.debug("Using deprecated CPG format with missing TYPE_FULL_NAME on METHOD_PARAMETER_IN nodes.")
               loggedMissingTypeFullName = true
             }
           }
@@ -50,7 +50,7 @@ class MethodDecoratorPass(cpg: Cpg) extends CpgPass(cpg) {
           dstGraph.addEdge(parameterIn, parameterOut, EdgeTypes.PARAMETER_LINK)
         }
       } else if (!loggedDeprecatedWarning) {
-        logger.warn("Using deprecated CPG format with PARAMETER_LINK edges")
+        logger.debug("Using deprecated CPG format with PARAMETER_LINK edges")
         loggedDeprecatedWarning = true
       }
     }

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/callgraph/StaticCallLinker.scala

@@ -39,7 +39,7 @@ class StaticCallLinker(cpg: Cpg) extends CpgPass(cpg) {
         linkStaticCall(call, dstGraph)
       case DispatchTypes.DYNAMIC_DISPATCH =>
       // Do nothing
-      case _ => logger.warn(s"Unknown dispatch type on dynamic CALL ${call.code}")
+      case _ => logger.debug(s"Unknown dispatch type on dynamic CALL ${call.code}")
     }
   }
 

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala

@@ -40,10 +40,10 @@ class CdgPass(cpg: Cpg) extends ForkJoinParallelCpgPass[Method](cpg) {
           val nodeLabel  = postDomFrontierNode.label
           val containsIn = postDomFrontierNode._containsIn
           if (containsIn == null || !containsIn.hasNext) {
-            logger.warn(s"Found CDG edge starting at $nodeLabel node. This is most likely caused by an invalid CFG.")
+            logger.debug(s"Found CDG edge starting at $nodeLabel node. This is most likely caused by an invalid CFG.")
           } else {
             val method = containsIn.next()
-            logger.warn(
+            logger.debug(
               s"Found CDG edge starting at $nodeLabel node. This is most likely caused by an invalid CFG." +
                 s" Method: ${method match {
                     case m: Method => m.fullName;

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/frontend/XConfigFileCreationPass.scala

@@ -25,7 +25,7 @@ abstract class XConfigFileCreationPass(cpg: Cpg) extends ConcurrentWriterCpgPass
 
   override def generateParts(): Array[File] =
     if (rootDir.isBlank) {
-      logger.warn("Unable to recover project directory for configuration file pass.")
+      logger.debug("Unable to recover project directory for configuration file pass.")
       Array.empty
     } else {
       Try(File(rootDir)) match {
@@ -50,7 +50,7 @@ abstract class XConfigFileCreationPass(cpg: Cpg) extends ConcurrentWriterCpgPass
         diffGraph.addNode(configNode)
 
       case Failure(error) =>
-        logger.warn(s"Unable to create config file node for ${file.canonicalPath}: $error")
+        logger.debug(s"Unable to create config file node for ${file.canonicalPath}: $error")
     }
   }
 

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/frontend/XTypeRecovery.scala

@@ -194,7 +194,7 @@ object XTypeRecovery {
           } else if (x == 1) {
             logger.debug("Intra-procedural type propagation enabled")
           } else if (x > 5) {
-            logger.warn(s"Large iteration count of $x will take a while to terminate")
+            logger.debug(s"Large iteration count of $x will take a while to terminate")
           }
           success
         }
@@ -695,7 +695,9 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode](
         val callCode = if (c.code.contains("(")) c.code.substring(c.code.indexOf("(")) else c.code
         XTypeRecovery.dummyMemberType(callCode, f.canonicalName, pathSep)
       case xs =>
-        logger.warn(s"Unhandled field structure ${xs.map(x => (x.label, x.code)).mkString(",")} @ ${debugLocation(fa)}")
+        logger.debug(
+          s"Unhandled field structure ${xs.map(x => (x.label, x.code)).mkString(",")} @ ${debugLocation(fa)}"
+        )
         wrapName("<unknown>")
     }
   }
@@ -724,7 +726,7 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode](
       val fieldName = getFieldName(fa)
       associateTypes(LocalVar(fieldName), fa, getLiteralType(l))
     } else {
-      logger.warn(s"Unhandled call assigned to literal point ${c.name} @ ${debugLocation(c)}")
+      logger.debug(s"Unhandled call assigned to literal point ${c.name} @ ${debugLocation(c)}")
       Set.empty
     }
   }
@@ -802,7 +804,7 @@ abstract class RecoverForXCompilationUnit[CompilationUnitType <: AstNode](
           Set(fieldName.stripSuffix(s"${XTypeRecovery.DummyMemberLoad}$pathSep${fi.canonicalName}"))
         )
       case _ =>
-        logger.warn(s"Unable to assign identifier '${i.name}' to field load '$fieldName' @ ${debugLocation(i)}")
+        logger.debug(s"Unable to assign identifier '${i.name}' to field load '$fieldName' @ ${debugLocation(i)}")
         Set.empty
     }
   }

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala

@@ -47,8 +47,11 @@ class CdxPass(atom: Cpg) extends CpgPass(atom) {
   private val BOM_JSON_FILE = ".*(bom|cdx).json"
 
   private def toPyModuleForm(str: String) = {
-    if (str.nonEmpty && str.count(_ == '.') > 0) {
-      s".*${str.split("\\.").take(2).mkString(Pattern.quote(File.separator))}.*"
+    if (str.nonEmpty) {
+      val tmpParts = str.split("\\.")
+      if (str.count(_ == '.') > 1) s"${tmpParts.take(2).mkString(Pattern.quote(File.separator))}.*"
+      else if (str.count(_ == '.') == 1) s"${tmpParts.head}.py:<module>.*"
+      else str
     } else if (str.nonEmpty) {
       s"$str${Pattern.quote(File.separator)}.*"
     } else {
@@ -110,6 +113,9 @@ class CdxPass(atom: Cpg) extends CpgPass(atom) {
                     atom.identifier.code(bpkg).newTagNode(compPurl).store()(dstGraph)
                     atom.identifier.code(bpkg).inCall.newTagNode(compPurl).store()(dstGraph)
                   }
+                  if (language == Languages.PYTHON || language == Languages.PYTHONSRC) {
+                    atom.call.where(_.methodFullName(bpkg)).argument.newTagNode(compPurl).store()(dstGraph)
+                  }
                 }
                 if (compType != "library") {
                   if (!containsRegex(bpkg)) {
@@ -127,6 +133,9 @@ class CdxPass(atom: Cpg) extends CpgPass(atom) {
                       atom.identifier.code(bpkg).newTagNode(compType).store()(dstGraph)
                       atom.identifier.code(bpkg).inCall.newTagNode(compType).store()(dstGraph)
                     }
+                    if (language == Languages.PYTHON || language == Languages.PYTHONSRC) {
+                      atom.call.where(_.methodFullName(bpkg)).argument.newTagNode(compType).store()(dstGraph)
+                    }
                   }
                 }
                 if (compType == "framework") {

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala

@@ -4,6 +4,7 @@ import io.circe.*
 import io.circe.parser.*
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.Languages
+import io.shiftleft.codepropertygraph.generated.nodes.Method
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.*
 
@@ -13,10 +14,17 @@ import java.util.regex.Pattern
   */
 class ChennaiTagsPass(atom: Cpg) extends CpgPass(atom) {
 
-  val language: String                   = atom.metaData.language.head
-  private val FRAMEWORK_ROUTE            = "framework-route"
-  private val PYTHON_ROUTES_CALL_REGEXES = Array("django/urls.py:<module>.(path|re_path)", ".*(route|web\\.).*")
+  val language: String         = atom.metaData.language.head
+  private val FRAMEWORK_ROUTE  = "framework-route"
+  private val FRAMEWORK_INPUT  = "framework-input"
+  private val FRAMEWORK_OUTPUT = "framework-output"
 
+  private val PYTHON_ROUTES_CALL_REGEXES =
+    Array("django/(conf/)?urls.py:<module>.(path|re_path|url).*", ".*(route|web\\.).*")
+  private val PYTHON_ROUTES_DECORATORS_REGEXES = Array(
+    ".*(route|endpoint|_request|require_http_methods|require_GET|require_POST|require_safe|_required)\\(.*"
+  )
+  private val HTTP_METHODS_REGEX = ".*(request|session)\\.(args|get|post|form).*"
   private def tagPythonRoutes(dstGraph: DiffGraphBuilder): Unit = {
     PYTHON_ROUTES_CALL_REGEXES.foreach { r =>
       atom.call
@@ -25,6 +33,29 @@ class ChennaiTagsPass(atom: Cpg) extends CpgPass(atom) {
         .isLiteral
         .newTagNode(FRAMEWORK_ROUTE)
         .store()(dstGraph)
+
+      PYTHON_ROUTES_DECORATORS_REGEXES.foreach { r =>
+        def decoratedMethods = atom.methodRef
+          .where(_.inCall.code(r).argument)
+          ._refOut
+          .collectAll[Method]
+        decoratedMethods.call.assignment
+          .code(HTTP_METHODS_REGEX)
+          .argument
+          .isIdentifier
+          .newTagNode(FRAMEWORK_INPUT)
+          .store()(dstGraph)
+        decoratedMethods
+          .newTagNode(FRAMEWORK_INPUT)
+          .store()(dstGraph)
+        decoratedMethods.parameter
+          .newTagNode(FRAMEWORK_INPUT)
+          .store()(dstGraph)
+      }
+      atom.ret
+        .where(_.method.tag.name(FRAMEWORK_INPUT))
+        .newTagNode(FRAMEWORK_OUTPUT)
+        .store()(dstGraph)
     }
   }
 

platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/utils/dependency/DependencyResolver.scala

@@ -34,7 +34,7 @@ object DependencyResolver {
       else if (isGradleBuildFile(buildFile))
         getCoordinatesForGradleProject(buildFile.getParent, defaultGradleConfigurationName)
       else {
-        logger.warn(s"Found unsupported build file $buildFile")
+        logger.debug(s"Found unsupported build file $buildFile")
         Nil
       }
     }.flatten
@@ -49,7 +49,7 @@ object DependencyResolver {
     val lines = ExternalCommand.run(s"gradle dependencies --configuration $configuration", projectDir.toString) match {
       case Success(lines) => lines
       case Failure(exception) =>
-        logger.warn(
+        logger.debug(
           s"Could not retrieve dependencies for Gradle project at path `$projectDir`\n" +
             exception.getMessage
         )
@@ -71,7 +71,7 @@ object DependencyResolver {
       } else if (isGradleBuildFile(buildFile)) {
         getDepsForGradleProject(params, buildFile.getParent)
       } else {
-        logger.warn(s"Found unsupported build file $buildFile")
+        logger.debug(s"Found unsupported build file $buildFile")
         Nil
       }
     }.flatten
@@ -90,7 +90,7 @@ object DependencyResolver {
     GradleDependencies.get(projectDir, gradleProjectName, gradleConfiguration) match {
       case Some(deps) => Some(deps)
       case None =>
-        logger.warn(s"Could not download Gradle dependencies for project at path `$projectDir`")
+        logger.debug(s"Could not download Gradle dependencies for project at path `$projectDir`")
         None
     }
   }