diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/C2Cpg.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/C2Cpg.scala index 5c6a0c914224..39dd01408530 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/C2Cpg.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/C2Cpg.scala @@ -1,6 +1,5 @@ package io.joern.c2cpg -import io.joern.c2cpg.datastructures.CGlobal import io.joern.c2cpg.passes.{AstCreationPass, PreprocessorPass, TypeDeclNodePass} import io.joern.c2cpg.utils.Report import io.shiftleft.codepropertygraph.Cpg @@ -20,8 +19,9 @@ class C2Cpg extends X2CpgFrontend[Config] { def createCpg(config: Config): Try[Cpg] = { withNewEmptyCpg(config.outputPath, config) { (cpg, config) => new MetaDataPass(cpg, Languages.NEWC, config.inputPath).createAndApply() - new AstCreationPass(cpg, config, report).createAndApply() - TypeNodePass.withRegisteredTypes(CGlobal.typesSeen(), cpg).createAndApply() + val astCreationPass = new AstCreationPass(cpg, config, report) + astCreationPass.createAndApply() + TypeNodePass.withRegisteredTypes(astCreationPass.typesSeen(), cpg).createAndApply() new TypeDeclNodePass(cpg)(config.schemaValidation).createAndApply() report.print() } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreator.scala index 21d498f15462..32e84c419f22 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreator.scala @@ -4,6 +4,7 @@ import io.joern.c2cpg.Config import io.joern.x2cpg.datastructures.Scope import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, AstNodeBuilder as X2CpgAstNodeBuilder} +import io.joern.x2cpg.datastructures.Global import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal @@ -18,6 +19,7 @@ import scala.collection.mutable */ class AstCreator( val filename: String, + val global: Global, val config: Config, val cdtAst: IASTTranslationUnit, val file2OffsetTable: ConcurrentHashMap[String, Array[Int]] diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala index 55fb28eede8b..fe266da422ae 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/astcreation/AstCreatorHelper.scala @@ -1,6 +1,5 @@ package io.joern.c2cpg.astcreation -import io.joern.c2cpg.datastructures.CGlobal import io.shiftleft.codepropertygraph.generated.nodes.{ExpressionNew, NewNode} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.joern.x2cpg.{Ast, SourceFiles, ValidationMode} @@ -10,7 +9,6 @@ import io.shiftleft.utils.IOUtils import org.apache.commons.lang.StringUtils import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.core.dom.ast.c.{ICASTArrayDesignator, ICASTDesignatedInitializer, ICASTFieldDesignator} -import org.eclipse.cdt.core.dom.ast.c.ICASTTypedefNameSpecifier import org.eclipse.cdt.core.dom.ast.cpp.* import org.eclipse.cdt.core.dom.ast.gnu.c.ICASTKnRFunctionDeclarator import org.eclipse.cdt.internal.core.dom.parser.c.CASTArrayRangeDesignator @@ -124,7 +122,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As protected def registerType(typeName: String): String = { val fixedTypeName = fixQualifiedName(StringUtils.normalizeSpace(typeName)) - CGlobal.usedTypes.putIfAbsent(fixedTypeName, true) + global.usedTypes.putIfAbsent(fixedTypeName, true) fixedTypeName } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/datastructures/CGlobal.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/datastructures/CGlobal.scala deleted file mode 100644 index e78f8f2609ed..000000000000 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/datastructures/CGlobal.scala +++ /dev/null @@ -1,16 +0,0 @@ -package io.joern.c2cpg.datastructures - -import io.joern.c2cpg.astcreation.Defines -import io.joern.x2cpg.datastructures.Global - -import scala.jdk.CollectionConverters._ - -object CGlobal extends Global { - - def typesSeen(): List[String] = this.synchronized { - val types = usedTypes.keys().asScala.filterNot(_ == Defines.anyTypeName).toList - usedTypes.clear() - types - } - -} diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/AstCreationPass.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/AstCreationPass.scala index 9d45763d7e9a..42440916501d 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/AstCreationPass.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/passes/AstCreationPass.scala @@ -3,16 +3,18 @@ package io.joern.c2cpg.passes import io.joern.c2cpg.C2Cpg.DefaultIgnoredFolders import io.joern.c2cpg.Config import io.joern.c2cpg.astcreation.AstCreator +import io.joern.c2cpg.astcreation.Defines import io.joern.c2cpg.parser.{CdtParser, FileDefaults} import io.joern.c2cpg.utils.{Report, TimeUtils} import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.passes.ConcurrentWriterCpgPass import io.joern.x2cpg.SourceFiles +import io.joern.x2cpg.datastructures.Global import java.nio.file.Paths import java.util.concurrent.ConcurrentHashMap -import java.util.regex.Pattern import scala.util.matching.Regex +import scala.jdk.CollectionConverters.* class AstCreationPass(cpg: Cpg, config: Config, report: Report = new Report()) extends ConcurrentWriterCpgPass[String](cpg) { @@ -20,6 +22,10 @@ class AstCreationPass(cpg: Cpg, config: Config, report: Report = new Report()) private val file2OffsetTable: ConcurrentHashMap[String, Array[Int]] = new ConcurrentHashMap() private val parser: CdtParser = new CdtParser(config) + private val global = new Global() + + def typesSeen(): List[String] = global.usedTypes.keys().asScala.filterNot(_ == Defines.anyTypeName).toList + override def generateParts(): Array[String] = SourceFiles .determine( config.inputPath, @@ -39,8 +45,9 @@ class AstCreationPass(cpg: Cpg, config: Config, report: Report = new Report()) parseResult match { case Some(translationUnit) => report.addReportInfo(relPath, fileLOC, parsed = true) - val localDiff = - new AstCreator(relPath, config, translationUnit, file2OffsetTable)(config.schemaValidation).createAst() + val localDiff = new AstCreator(relPath, global, config, translationUnit, file2OffsetTable)( + config.schemaValidation + ).createAst() diffGraph.absorb(localDiff) true case None => diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/JsSrc2Cpg.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/JsSrc2Cpg.scala index 5205471ee673..2d0af92f0ec4 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/JsSrc2Cpg.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/JsSrc2Cpg.scala @@ -3,7 +3,6 @@ package io.joern.jssrc2cpg import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.jssrc2cpg.JsSrc2Cpg.postProcessingPasses -import io.joern.jssrc2cpg.datastructures.JsGlobal import io.joern.jssrc2cpg.passes.* import io.joern.jssrc2cpg.utils.AstGenRunner import io.joern.x2cpg.X2Cpg.withNewEmptyCpg @@ -30,7 +29,7 @@ class JsSrc2Cpg extends X2CpgFrontend[Config] { val astCreationPass = new AstCreationPass(cpg, astGenResult, config, report)(config.schemaValidation) astCreationPass.createAndApply() - JavaScriptTypeNodePass.withRegisteredTypes(JsGlobal.typesSeen(), cpg).createAndApply() + JavaScriptTypeNodePass.withRegisteredTypes(astCreationPass.typesSeen(), cpg).createAndApply() new JavaScriptMetaDataPass(cpg, hash, config.inputPath).createAndApply() new DependenciesPass(cpg, config).createAndApply() new ConfigPass(cpg, config, report).createAndApply() diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala index ece6a14922ce..005307d290e1 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala @@ -9,6 +9,7 @@ import io.joern.jssrc2cpg.passes.Defines import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.utils.NodeBuilders.{newMethodReturnNode, newModifierNode} import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, AstNodeBuilder as X2CpgAstNodeBuilder} +import io.joern.x2cpg.datastructures.Global import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes, NodeTypes} import io.shiftleft.codepropertygraph.generated.nodes.NewBlock import io.shiftleft.codepropertygraph.generated.nodes.NewFile @@ -22,8 +23,9 @@ import ujson.Value import scala.collection.mutable -class AstCreator(val config: Config, val parserResult: ParseResult)(implicit withSchemaValidation: ValidationMode) - extends AstCreatorBase(parserResult.filename) +class AstCreator(val config: Config, val global: Global, val parserResult: ParseResult)(implicit + withSchemaValidation: ValidationMode +) extends AstCreatorBase(parserResult.filename) with AstForExpressionsCreator with AstForPrimitivesCreator with AstForTypesCreator diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala index 9ad8d58fa06b..c512e98a2dc3 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala @@ -38,7 +38,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As } protected def registerType(typeFullName: String): Unit = { - JsGlobal.usedTypes.putIfAbsent(typeFullName, true) + global.usedTypes.putIfAbsent(typeFullName, true) } private def nodeType(node: Value): BabelNode = fromString(node("type").str) diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/datastructures/JsGlobal.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/datastructures/JsGlobal.scala deleted file mode 100644 index 3e5c29d0bbdd..000000000000 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/datastructures/JsGlobal.scala +++ /dev/null @@ -1,16 +0,0 @@ -package io.joern.jssrc2cpg.datastructures - -import io.joern.jssrc2cpg.passes.Defines -import io.joern.x2cpg.datastructures.Global - -import scala.jdk.CollectionConverters._ - -object JsGlobal extends Global { - - def typesSeen(): List[String] = this.synchronized { - val types = usedTypes.keys().asScala.filterNot(_ == Defines.Any).toList - usedTypes.clear() - types - } - -} diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala index 89105940e87b..8b7bb7526c56 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/AstCreationPass.scala @@ -5,6 +5,7 @@ import io.joern.jssrc2cpg.astcreation.AstCreator import io.joern.jssrc2cpg.parser.BabelJsonParser import io.joern.jssrc2cpg.utils.AstGenRunner.AstGenRunnerResult import io.joern.x2cpg.ValidationMode +import io.joern.x2cpg.datastructures.Global import io.joern.x2cpg.utils.{Report, TimeUtils} import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.passes.ConcurrentWriterCpgPass @@ -13,6 +14,7 @@ import org.slf4j.{Logger, LoggerFactory} import java.nio.file.Paths import scala.util.{Failure, Success, Try} +import scala.jdk.CollectionConverters._ class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: Config, report: Report = new Report())( implicit withSchemaValidation: ValidationMode @@ -20,6 +22,10 @@ class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: private val logger: Logger = LoggerFactory.getLogger(classOf[AstCreationPass]) + private val global = new Global() + + def typesSeen(): List[String] = global.usedTypes.keys().asScala.filterNot(_ == Defines.Any).toList + override def generateParts(): Array[(String, String)] = astGenRunnerResult.parsedFiles.toArray override def finish(): Unit = { @@ -43,7 +49,7 @@ class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: val fileLOC = IOUtils.readLinesInFile(Paths.get(parseResult.fullPath)).size report.addReportInfo(parseResult.filename, fileLOC, parsed = true) Try { - val localDiff = new AstCreator(config, parseResult).createAst() + val localDiff = new AstCreator(config, global, parseResult).createAst() diffGraph.absorb(localDiff) } match { case Failure(exception) => diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractPassTest.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractPassTest.scala index deab87c9fc4a..dde8c68b6514 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractPassTest.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/passes/AbstractPassTest.scala @@ -3,7 +3,6 @@ package io.joern.jssrc2cpg.passes import better.files.File import io.joern.jssrc2cpg.utils.AstGenRunner import io.joern.jssrc2cpg.Config -import io.joern.jssrc2cpg.datastructures.JsGlobal import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.X2Cpg.newEmptyCpg import io.shiftleft.codepropertygraph.Cpg @@ -23,10 +22,11 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers with Inside { val cpg = newEmptyCpg() val file = dir / filename file.write(code) - val config = Config(tsTypes = tsTypes).withInputPath(dir.toString).withOutputPath(dir.toString) - val astGenResult = new AstGenRunner(config).execute(dir) - new AstCreationPass(cpg, astGenResult, config).createAndApply() - JavaScriptTypeNodePass.withRegisteredTypes(JsGlobal.typesSeen(), cpg).createAndApply() + val config = Config(tsTypes = tsTypes).withInputPath(dir.toString).withOutputPath(dir.toString) + val astGenResult = new AstGenRunner(config).execute(dir) + val astCreationPass = new AstCreationPass(cpg, astGenResult, config) + astCreationPass.createAndApply() + JavaScriptTypeNodePass.withRegisteredTypes(astCreationPass.typesSeen(), cpg).createAndApply() f(cpg) file.delete() } @@ -43,7 +43,7 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers with Inside { val astGenResult = new AstGenRunner(config).execute(dir) val astCreationPass = new AstCreationPass(cpg, astGenResult, config) astCreationPass.createAndApply() - JavaScriptTypeNodePass.withRegisteredTypes(JsGlobal.typesSeen(), cpg).createAndApply() + JavaScriptTypeNodePass.withRegisteredTypes(astCreationPass.typesSeen(), cpg).createAndApply() f(cpg) file.delete() } @@ -62,7 +62,7 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers with Inside { val astGenResult = new AstGenRunner(config).execute(dir) val astCreationPass = new AstCreationPass(cpg, astGenResult, config) astCreationPass.createAndApply() - JavaScriptTypeNodePass.withRegisteredTypes(JsGlobal.typesSeen(), cpg).createAndApply() + JavaScriptTypeNodePass.withRegisteredTypes(astCreationPass.typesSeen(), cpg).createAndApply() f(cpg) file1.delete() file2.delete() diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/SwiftSrc2Cpg.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/SwiftSrc2Cpg.scala index 675c4197605b..5c8827a78979 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/SwiftSrc2Cpg.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/SwiftSrc2Cpg.scala @@ -2,7 +2,6 @@ package io.joern.swiftsrc2cpg import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} -import io.joern.swiftsrc2cpg.datastructures.SwiftGlobal import io.joern.swiftsrc2cpg.passes.* import io.joern.swiftsrc2cpg.utils.AstGenRunner import io.joern.x2cpg.X2Cpg.withNewEmptyCpg @@ -29,7 +28,7 @@ class SwiftSrc2Cpg extends X2CpgFrontend[Config] { val astCreationPass = new AstCreationPass(cpg, astGenResult, config, report)(config.schemaValidation) astCreationPass.createAndApply() - SwiftTypeNodePass.withRegisteredTypes(SwiftGlobal.typesSeen(), cpg).createAndApply() + SwiftTypeNodePass.withRegisteredTypes(astCreationPass.typesSeen(), cpg).createAndApply() new SwiftMetaDataPass(cpg, hash, config.inputPath).createAndApply() new BuiltinTypesPass(cpg).createAndApply() new DependenciesPass(cpg, config).createAndApply() diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala index 74d86bf7a12b..f4c2bc554230 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala @@ -8,6 +8,7 @@ import io.joern.swiftsrc2cpg.passes.Defines import io.joern.x2cpg.datastructures.Stack.* import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, AstNodeBuilder as X2CpgAstNodeBuilder} +import io.joern.x2cpg.datastructures.Global import io.joern.x2cpg.utils.NodeBuilders.newModifierNode import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import io.shiftleft.codepropertygraph.generated.NodeTypes @@ -22,8 +23,9 @@ import overflowdb.BatchedUpdate.DiffGraphBuilder import scala.collection.mutable -class AstCreator(val config: Config, val parserResult: ParseResult)(implicit withSchemaValidation: ValidationMode) - extends AstCreatorBase(parserResult.filename) +class AstCreator(val config: Config, val global: Global, val parserResult: ParseResult)(implicit + withSchemaValidation: ValidationMode +) extends AstCreatorBase(parserResult.filename) with AstForSwiftTokenCreator with AstForSyntaxCreator with AstForExprSyntaxCreator diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala index ad5ca2063e09..31418e23a7de 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreatorHelper.scala @@ -13,6 +13,17 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewTypeDecl import scala.collection.mutable +object AstCreatorHelper { + + implicit class OptionSafeAst(val ast: Ast) extends AnyVal { + def withArgEdge(src: NewNode, dst: Option[NewNode]): Ast = dst match { + case Some(value) => ast.withArgEdge(src, value) + case None => ast + } + } + +} + trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: AstCreator => protected def notHandledYet(node: SwiftNode): Ast = { @@ -28,7 +39,7 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As } protected def registerType(typeFullName: String): Unit = { - SwiftGlobal.usedTypes.putIfAbsent((typeFullName), true) + global.usedTypes.putIfAbsent(typeFullName, true) } protected def generateUnusedVariableName( diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForExprSyntaxCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForExprSyntaxCreator.scala index 111819cccb96..930a99b06326 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForExprSyntaxCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForExprSyntaxCreator.scala @@ -3,6 +3,7 @@ package io.joern.swiftsrc2cpg.astcreation import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.* import io.joern.x2cpg.Ast import io.joern.x2cpg.ValidationMode +import io.shiftleft.codepropertygraph.generated.ControlStructureTypes import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.Operators @@ -40,8 +41,20 @@ trait AstForExprSyntaxCreator(implicit withSchemaValidation: ValidationMode) { private def astForForceUnwrapExprSyntax(node: ForceUnwrapExprSyntax): Ast = notHandledYet(node) private def astForFunctionCallExprSyntax(node: FunctionCallExprSyntax): Ast = notHandledYet(node) private def astForGenericSpecializationExprSyntax(node: GenericSpecializationExprSyntax): Ast = notHandledYet(node) - private def astForIfExprSyntax(node: IfExprSyntax): Ast = notHandledYet(node) - private def astForInOutExprSyntax(node: InOutExprSyntax): Ast = notHandledYet(node) + + private def astForIfExprSyntax(node: IfExprSyntax): Ast = { + val code = this.code(node) + val ifNode = controlStructureNode(node, ControlStructureTypes.IF, code) + val conditionAst = astForNode(node.conditions) + val thenAst = astForNode(node.body) + val elseAst = node.elseBody match { + case Some(value) => astForNode(value) + case None => Ast() + } + controlStructureAst(ifNode, Some(conditionAst), Seq(thenAst, elseAst)) + } + + private def astForInOutExprSyntax(node: InOutExprSyntax): Ast = notHandledYet(node) private def astForInfixOperatorExprSyntax(node: InfixOperatorExprSyntax): Ast = { val op = code(node.operator) match { case "=" => Operators.assignment @@ -105,8 +118,20 @@ trait AstForExprSyntaxCreator(implicit withSchemaValidation: ValidationMode) { private def astForSubscriptCallExprSyntax(node: SubscriptCallExprSyntax): Ast = notHandledYet(node) private def astForSuperExprSyntax(node: SuperExprSyntax): Ast = notHandledYet(node) private def astForSwitchExprSyntax(node: SwitchExprSyntax): Ast = notHandledYet(node) - private def astForTernaryExprSyntax(node: TernaryExprSyntax): Ast = notHandledYet(node) - private def astForTryExprSyntax(node: TryExprSyntax): Ast = notHandledYet(node) + + private def astForTernaryExprSyntax(node: TernaryExprSyntax): Ast = { + val name = Operators.conditional + val call = callNode(node, code(node), name, name, DispatchTypes.STATIC_DISPATCH) + + val condAst = astForNodeWithFunctionReference(node.condition) + val posAst = astForNodeWithFunctionReference(node.thenExpression) + val negAst = astForNodeWithFunctionReference(node.elseExpression) + + val children = List(condAst, posAst, negAst) + callAst(call, children) + } + + private def astForTryExprSyntax(node: TryExprSyntax): Ast = notHandledYet(node) private def astForTupleExprSyntax(node: TupleExprSyntax): Ast = { astForNode(node.elements) diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForStmtSyntaxCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForStmtSyntaxCreator.scala index bc8be7882137..f0b74a053894 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForStmtSyntaxCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForStmtSyntaxCreator.scala @@ -1,5 +1,6 @@ package io.joern.swiftsrc2cpg.astcreation +import io.joern.swiftsrc2cpg.astcreation.AstCreatorHelper.OptionSafeAst import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.* import io.joern.x2cpg.Ast import io.joern.x2cpg.ValidationMode @@ -24,20 +25,31 @@ trait AstForStmtSyntaxCreator(implicit withSchemaValidation: ValidationMode) { val code = this.code(node) // In Swift, a repeat-while loop is semantically the same as a C do-while loop val doNode = controlStructureNode(node, ControlStructureTypes.DO, code) - val conditionAst = astForNode(node.condition) + val conditionAst = astForNodeWithFunctionReference(node.condition) val bodyAst = astForNode(node.body) setOrderExplicitly(conditionAst, 1) setOrderExplicitly(bodyAst, 2) controlStructureAst(doNode, Some(conditionAst), Seq(bodyAst), placeConditionLast = true) } - private def astForReturnStmtSyntax(node: ReturnStmtSyntax): Ast = notHandledYet(node) - private def astForThenStmtSyntax(node: ThenStmtSyntax): Ast = notHandledYet(node) - private def astForThrowStmtSyntax(node: ThrowStmtSyntax): Ast = notHandledYet(node) + private def astForReturnStmtSyntax(node: ReturnStmtSyntax): Ast = { + val cpgReturn = returnNode(node, code(node)) + node.expression match { + case Some(value) => + val expr = astForNodeWithFunctionReference(value) + Ast(cpgReturn).withChild(expr).withArgEdge(cpgReturn, expr.root) + case None => + Ast(cpgReturn) + } + + } + + private def astForThenStmtSyntax(node: ThenStmtSyntax): Ast = notHandledYet(node) + private def astForThrowStmtSyntax(node: ThrowStmtSyntax): Ast = notHandledYet(node) private def astForWhileStmtSyntax(node: WhileStmtSyntax): Ast = { val code = this.code(node) - val conditionAst = astForNode(node.conditions) + val conditionAst = astForNodeWithFunctionReference(node.conditions) val bodyAst = astForNode(node.body) setOrderExplicitly(conditionAst, 1) setOrderExplicitly(bodyAst, 2) diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCollectionCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCollectionCreator.scala index 1050d9e33abd..285c7579eb5a 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCollectionCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCollectionCreator.scala @@ -12,7 +12,7 @@ trait AstForSyntaxCollectionCreator(implicit withSchemaValidation: ValidationMod private def astForListSyntaxChildren(node: SwiftNode, children: Seq[SwiftNode]): Ast = { children.toList match { case Nil => Ast() - case head :: Nil => astForNode(head) + case head :: Nil => astForNodeWithFunctionReference(head) case elements => val blockNode_ = blockNode(node, "", Defines.Any) scope.pushNewBlockScope(blockNode_) diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCreator.scala index 0be7b966b4c5..72c2c916c911 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCreator.scala @@ -20,7 +20,7 @@ trait AstForSyntaxCreator(implicit withSchemaValidation: ValidationMode) { this: private def astForAttributeSyntax(node: AttributeSyntax): Ast = { val argumentAsts = node.arguments match { case Some(argument) => - val argumentAst = astForNode(argument) + val argumentAst = astForNodeWithFunctionReference(argument) val parameter = NewAnnotationParameter().code("argument") val assign = NewAnnotationParameterAssign().code(code(argument)) val assignChildren = List(Ast(parameter), argumentAst) @@ -165,7 +165,7 @@ trait AstForSyntaxCreator(implicit withSchemaValidation: ValidationMode) { this: // And a TypeRef is returned afterwards. node.label match { case Some(_) => notHandledYet(node) - case None => astForNode(node.expression) + case None => astForNodeWithFunctionReference(node.expression) } } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/datastructures/SwiftGlobal.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/datastructures/SwiftGlobal.scala deleted file mode 100644 index a3427622a2ed..000000000000 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/datastructures/SwiftGlobal.scala +++ /dev/null @@ -1,16 +0,0 @@ -package io.joern.swiftsrc2cpg.datastructures - -import io.joern.swiftsrc2cpg.passes.Defines -import io.joern.x2cpg.datastructures.Global - -import scala.jdk.CollectionConverters._ - -object SwiftGlobal extends Global { - - def typesSeen(): List[String] = this.synchronized { - val types = usedTypes.keys().asScala.filterNot(_ == Defines.Any).toList - usedTypes.clear() - types - } - -} diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/AstCreationPass.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/AstCreationPass.scala index 51d8dee86c3d..a190d764b5bb 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/AstCreationPass.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/passes/AstCreationPass.scala @@ -5,6 +5,7 @@ import io.joern.swiftsrc2cpg.astcreation.AstCreator import io.joern.swiftsrc2cpg.parser.SwiftJsonParser import io.joern.swiftsrc2cpg.utils.AstGenRunner.AstGenRunnerResult import io.joern.x2cpg.ValidationMode +import io.joern.x2cpg.datastructures.Global import io.joern.x2cpg.utils.{Report, TimeUtils} import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.passes.ConcurrentWriterCpgPass @@ -12,9 +13,8 @@ import io.shiftleft.utils.IOUtils import org.slf4j.{Logger, LoggerFactory} import java.nio.file.Paths -import java.util.concurrent.ConcurrentHashMap -import scala.jdk.CollectionConverters.EnumerationHasAsScala import scala.util.{Failure, Success, Try} +import scala.jdk.CollectionConverters.* class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: Config, report: Report = new Report())( implicit withSchemaValidation: ValidationMode @@ -22,13 +22,17 @@ class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: private val logger: Logger = LoggerFactory.getLogger(classOf[AstCreationPass]) + private val global = new Global() + + def typesSeen(): List[String] = global.usedTypes.keys().asScala.filterNot(_ == Defines.Any).toList + override def generateParts(): Array[(String, String)] = astGenRunnerResult.parsedFiles.toArray override def finish(): Unit = { astGenRunnerResult.skippedFiles.foreach { skippedFile => - val (rootPath, fileName) = skippedFile - val filePath = Paths.get(fileName) - val fileLOC = IOUtils.readLinesInFile(filePath).size + val (_, fileName) = skippedFile + val filePath = Paths.get(fileName) + val fileLOC = IOUtils.readLinesInFile(filePath).size report.addReportInfo(fileName, fileLOC) } } @@ -40,7 +44,7 @@ class AstCreationPass(cpg: Cpg, astGenRunnerResult: AstGenRunnerResult, config: val fileLOC = IOUtils.readLinesInFile(Paths.get(parseResult.fullPath)).size report.addReportInfo(parseResult.filename, fileLOC, parsed = true) Try { - val localDiff = new AstCreator(config, parseResult).createAst() + val localDiff = new AstCreator(config, global, parseResult).createAst() diffGraph.absorb(localDiff) } match { case Failure(exception) => diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/AbstractPassTest.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/AbstractPassTest.scala index c5e30bf8dcef..5cefa56efcc1 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/AbstractPassTest.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/AbstractPassTest.scala @@ -3,7 +3,6 @@ package io.joern.swiftsrc2cpg.passes.ast import better.files.File import io.joern.swiftsrc2cpg.utils.AstGenRunner import io.joern.swiftsrc2cpg.Config -import io.joern.swiftsrc2cpg.datastructures.SwiftGlobal import io.joern.swiftsrc2cpg.passes.AstCreationPass import io.joern.swiftsrc2cpg.passes.SwiftTypeNodePass import io.joern.x2cpg.ValidationMode @@ -25,10 +24,11 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers with Inside { val cpg = newEmptyCpg() val file = dir / filename file.write(code) - val config = Config().withInputPath(dir.toString).withOutputPath(dir.toString) - val astGenResult = new AstGenRunner(config).execute(dir) - new AstCreationPass(cpg, astGenResult, config).createAndApply() - SwiftTypeNodePass.withRegisteredTypes(SwiftGlobal.typesSeen(), cpg).createAndApply() + val config = Config().withInputPath(dir.toString).withOutputPath(dir.toString) + val astGenResult = new AstGenRunner(config).execute(dir) + val astCreationPass = new AstCreationPass(cpg, astGenResult, config) + astCreationPass.createAndApply() + SwiftTypeNodePass.withRegisteredTypes(astCreationPass.typesSeen(), cpg).createAndApply() f(cpg) file.delete() } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ExpressionTests.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ExpressionTests.scala index 515bacf89d8d..fd692fe54193 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ExpressionTests.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/passes/ast/ExpressionTests.scala @@ -7,9 +7,37 @@ import io.shiftleft.semanticcpg.language._ class ExpressionTests extends AbstractPassTest { "ExpressionTests" should { - "testTernary" ignore AstFixture("a ? b : c") { cpg => ??? } + "testTernary" in AstFixture("a ? b : c") { cpg => + inside(cpg.method.name("").ast.isCall.l) { case List(call: Call) => + call.code shouldBe "a ? b : c" + call.name shouldBe Operators.conditional + inside(call.argument.l) { case List(cond, trueCase, falseCase) => + cond.code shouldBe "a" + trueCase.code shouldBe "b" + falseCase.code shouldBe "c" + } + call.lineNumber shouldBe Some(1) + call.columnNumber shouldBe Some(1) + } + } - "testSequence1" ignore AstFixture("a ? b : c ? d : e") { cpg => ??? } + "testSequence1" in AstFixture("a ? b : c ? d : e") { cpg => + inside(cpg.method.name("").ast.isCall.l) { case List(call: Call, nestedCall: Call) => + call.code shouldBe "a ? b : c ? d : e" + call.name shouldBe Operators.conditional + inside(call.argument.l) { case List(cond, trueCase, falseCase: Call) => + cond.code shouldBe "a" + trueCase.code shouldBe "b" + falseCase.code shouldBe "c ? d : e" + falseCase shouldBe nestedCall + inside(falseCase.argument.l) { case List(cond, trueCase, falseCase) => + cond.code shouldBe "c" + trueCase.code shouldBe "d" + falseCase.code shouldBe "e" + } + } + } + } "testSequence2" ignore AstFixture("A as? B + C -> D is E as! F ? G = 42 : H") { cpg => ??? } @@ -125,11 +153,24 @@ class ExpressionTests extends AbstractPassTest { ??? } - "testIfExprInReturn" ignore AstFixture(""" + "testIfExprInReturn" in AstFixture(""" |func foo() { | return if .random() { 0 } else { 1 } |} - |""".stripMargin) { cpg => ??? } + |""".stripMargin) { cpg => + inside(cpg.method.name("foo").ast.isReturn.astChildren.isControlStructure.l) { + case List(controlStruct: ControlStructure) => + controlStruct.code should startWith("if .random() {") + controlStruct.controlStructureType shouldBe ControlStructureTypes.IF + inside(controlStruct.condition.l) { case List(cndNode) => + cndNode.code shouldBe ".random()" + } + controlStruct.whenTrue.code.l shouldBe List("0") + controlStruct.whenFalse.code.l shouldBe List("1") + controlStruct.lineNumber shouldBe Some(3) + controlStruct.columnNumber shouldBe Some(10) + } + } "testSwitchExprInReturn" ignore AstFixture(""" |func foo() {