Improve performance of structural analysis by sacrificing some precis…

…ion (#29) * Improve performance of structural analysis by sacrificing some precision Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Test fix Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Microsoft specific default defines Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
AppThreat · Nov 1, 2023 · 7b10974 · 7b10974
1 parent 3452d89
commit 7b10974
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 20 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.5.4"
+ThisBuild / version      := "0.6.0"
 ThisBuild / scalaVersion := "3.3.1"
 
 val cpgVersion = "1.4.22"

diff --git a/...form/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstCreatorHelper.scala b/...form/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstCreatorHelper.scala
@@ -144,7 +144,11 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
       "enum",
       "struct",
       "interface",
-      "class"
+      "class",
+      "naked",
+      "export",
+      "module",
+      "import"
     )
 
   protected def cleanType(rawType: String, stripKeywords: Boolean = true): String = {
@@ -485,6 +489,10 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
   }
 
   protected def astForNode(node: IASTNode): Ast = {
+    if (config.includeFunctionBodies) astForNodeFull(node) else astForNodePartial(node)
+  }
+
+  protected def astForNodeFull(node: IASTNode): Ast = {
     node match {
       case expr: IASTExpression             => astForExpression(expr)
       case name: IASTName                   => astForIdentifier(name)
@@ -505,6 +513,16 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As
     }
   }
 
+  protected def astForNodePartial(node: IASTNode): Ast = {
+    node match {
+      case expr: IASTExpression           => astForExpression(expr)
+      case name: IASTName                 => astForIdentifier(name)
+      case decl: IASTDeclSpecifier        => astForIdentifier(decl)
+      case decl: ICPPASTDecltypeSpecifier => astForDecltypeSpecifier(decl)
+      case _                              => notHandledYet(node)
+    }
+  }
+
   protected def typeForDeclSpecifier(spec: IASTNode, stripKeywords: Boolean = true, index: Int = 0): String = {
     val tpe = spec match {
       case s: IASTSimpleDeclSpecifier if s.getParent.isInstanceOf[IASTParameterDeclaration] =>

diff --git a/...ntends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstForExpressionsCreator.scala b/...ntends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstForExpressionsCreator.scala
@@ -271,6 +271,12 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
     astForExpression(packExpansionExpression.getPattern)
 
   protected def astForExpression(expression: IASTExpression): Ast = {
+    if (config.includeFunctionBodies)
+      astForExpressionFull(expression)
+    else astForExpressionPartial(expression)
+  }
+
+  protected def astForExpressionFull(expression: IASTExpression): Ast = {
     val r = expression match {
       case lit: IASTLiteralExpression                  => astForLiteral(lit)
       case un: IASTUnaryExpression                     => astForUnaryExpression(un)
@@ -295,6 +301,19 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
     asChildOfMacroCall(expression, r)
   }
 
+  protected def astForExpressionPartial(expression: IASTExpression): Ast = {
+    val r = expression match {
+      case call: IASTFunctionCallExpression            => astForCallExpression(call)
+      case typeId: IASTTypeIdExpression                => astForTypeIdExpression(typeId)
+      case fieldRef: IASTFieldReference                => astForFieldReference(fieldRef)
+      case newExpression: ICPPASTNewExpression         => astForNewExpression(newExpression)
+      case typeIdInit: IASTTypeIdInitializerExpression => astForTypeIdInitExpression(typeIdInit)
+      case c: ICPPASTSimpleTypeConstructorExpression   => astForConstructorExpression(c)
+      case _                                           => notHandledYet(expression)
+    }
+    asChildOfMacroCall(expression, r)
+  }
+
   private def astForIdExpression(idExpression: IASTIdExpression): Ast = idExpression.getName match {
     case name: CPPASTQualifiedName => astForQualifiedName(name)
     case _                         => astForIdentifier(idExpression)

diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/CdtParser.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/CdtParser.scala
@@ -45,6 +45,22 @@ class CdtParser(config: Config) extends ParseProblemsLogger with PreprocessorSta
   private val includePaths     = parserConfig.userIncludePaths
   private val log              = new DefaultLogService
 
+  private var stayCpp: Boolean = false;
+
+  private val cScannerInfo: ExtendedScannerInfo = new ExtendedScannerInfo(
+    definedSymbols,
+    (includePaths ++ parserConfig.systemIncludePathsC).map(_.toString).toArray,
+    parserConfig.macroFiles.map(_.toString).toArray,
+    parserConfig.includeFiles.map(_.toString).toArray
+  )
+
+  private val cppScannerInfo: ExtendedScannerInfo = new ExtendedScannerInfo(
+    definedSymbols,
+    (includePaths ++ parserConfig.systemIncludePathsCPP).map(_.toString).toArray,
+    parserConfig.macroFiles.map(_.toString).toArray,
+    parserConfig.includeFiles.map(_.toString).toArray
+  )
+
   // Setup indexing
   var index: Option[IIndex] = Option(EmptyCIndex.INSTANCE)
   if (config.useProjectIndex) {
@@ -71,17 +87,11 @@ class CdtParser(config: Config) extends ParseProblemsLogger with PreprocessorSta
     }
   }
 
-  private def createScannerInfo(file: Path): ExtendedScannerInfo = {
-    val additionalIncludes =
-      if (FileDefaults.isCPPFile(file.toString)) parserConfig.systemIncludePathsCPP
-      else parserConfig.systemIncludePathsC
-    new ExtendedScannerInfo(
-      definedSymbols,
-      (includePaths ++ additionalIncludes).map(_.toString).toArray,
-      parserConfig.macroFiles.map(_.toString).toArray,
-      parserConfig.includeFiles.map(_.toString).toArray
-    )
-  }
+  private def createScannerInfo(file: Path): ExtendedScannerInfo =
+    if (stayCpp || FileDefaults.isCPPFile(file.toString)) {
+      stayCpp = true
+      cppScannerInfo
+    } else cScannerInfo
 
   private def parseInternal(file: Path): ParseResult = {
     val realPath = File(file)

diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/DefaultDefines.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/DefaultDefines.scala
@@ -2,8 +2,34 @@ package io.appthreat.c2cpg.parser
 
 object DefaultDefines {
   val DEFAULT_CALL_CONVENTIONS: Map[String, String] = Map(
-    "__fastcall" -> "__attribute((fastcall))",
-    "__cdecl"    -> "__attribute((cdecl))",
-    "__pascal"   -> "__attribute((pascal))"
+    "__fastcall"   -> "__attribute((fastcall))",
+    "__cdecl"      -> "__attribute((cdecl))",
+    "__pascal"     -> "__attribute((pascal))",
+    "__vectorcall" -> "__attribute((vectorcall))",
+    "__clrcall"    -> "__attribute((clrcall))",
+    "__stdcall"    -> "__attribute((stdcall))",
+    "__thiscall"   -> "__attribute((thiscall))",
+    "__declspec"   -> "__attribute((declspec))",
+    "__restrict"   -> "__attribute((restrict))",
+    "__sptr"       -> "__attribute((sptr))",
+    "__uptr"       -> "__attribute((uptr))",
+    "__syscall"    -> "__attribute((syscall))",
+    "__oldcall"    -> "__attribute((oldcall))",
+    "__unaligned"  -> "__attribute((unaligned))",
+    "__w64"        -> "__attribute((w64))",
+    "__asm"        -> "__attribute((asm))",
+    "__based"      -> "__attribute((based))",
+    "__interface"  -> "__attribute((interface))",
+    "__event"      -> "__attribute((event))",
+    "__hook"       -> "__attribute((hook))",
+    "__unhook"     -> "__attribute((unhook))",
+    "__raise"      -> "__attribute((raise))",
+    "__try"        -> "__attribute((try))",
+    "__except"     -> "__attribute((except))",
+    "__finally"    -> "__attribute((finally))",
+    "__m128"       -> "__attribute((m128))",
+    "__m128d"      -> "__attribute((m128d))",
+    "__m128i"      -> "__attribute((m128i))",
+    "__m64"        -> "__attribute((m64))"
   )
 }
diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/AstCreationPass.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/AstCreationPass.scala
@@ -33,6 +33,7 @@ class AstCreationPass(cpg: Cpg, config: Config, report: Report = new Report())
         FileDefaults.SOURCE_FILE_EXTENSIONS ++ FileDefaults.HEADER_FILE_EXTENSIONS,
         config.withDefaultIgnoredFilesRegex(DefaultIgnoredFolders)
       )
+      .sortWith(_.compareToIgnoreCase(_) > 0)
       .toArray
 
   override def runOnPart(diffGraph: DiffGraphBuilder, filename: String): Unit = {

diff --git a/...tends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/ast/HeaderAstCreationPassTests.scala b/...tends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/ast/HeaderAstCreationPassTests.scala
@@ -48,9 +48,9 @@ class HeaderAstCreationPassTests extends CCodeToCpgSuite {
           // second time for the actual implementation in the source file
           // We do not de-duplicate this as line/column numbers differ
           m1.fullName shouldBe "main"
-          m1.filename shouldBe "main.c"
+          m1.filename shouldBe "main.h"
           m2.fullName shouldBe "main"
-          m2.filename shouldBe "main.h"
+          m2.filename shouldBe "main.c"
           printf.fullName shouldBe "printf"
       }
     }

diff --git a/.../frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/ast/ProgramStructureTests.scala b/.../frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/ast/ProgramStructureTests.scala
@@ -24,7 +24,7 @@ class ProgramStructureTests extends CCodeToCpgSuite {
 
     "create one NamespaceBlock per file" in {
       val cpg                        = code("", "foo.c").moreCode("", "woo.c")
-      val expectedFilenames          = Seq("foo.c", "woo.c")
+      val expectedFilenames          = Seq("woo.c", "foo.c")
       val expectedNamespaceFullNames = expectedFilenames.map(f => s"$f:${NamespaceTraversal.globalNamespaceName}")
       val allNamespaceBlockFullNames = cpg.namespaceBlock.fullNameNot(NamespaceTraversal.globalNamespaceName).fullName.l
       allNamespaceBlockFullNames.zip(expectedNamespaceFullNames).foreach { case (actual, expected) =>

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "appthreat-chen"
-version = "0.5.4"
+version = "0.6.0"
 description = "Code Hierarchy Exploration Net (chen)"
 authors = ["Team AppThreat <cloud@appthreat.com>"]
 license = "Apache-2.0"