From 26560421a33979c8511e34528b321f8793086750 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Fri, 21 Jul 2023 13:10:55 +0530 Subject: [PATCH 1/6] remove the logic to treat anything uppercase as Constant member --- .../AstForExpressionsCreator.scala | 2 +- .../passes/RubyTypeRecoveryTests.scala | 27 +++++++++++++++++-- .../rubysrc2cpg/passes/ast/ModuleTests.scala | 3 ++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index 7345bdd1c8d2..c05e83ce2548 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -242,7 +242,7 @@ trait AstForExpressionsCreator { this: AstCreator => */ val variableName = ctx.getText - val isSelfFieldAccess = variableName.startsWith("@") || variableName.isAllUpperCase + val isSelfFieldAccess = variableName.startsWith("@") if (isSelfFieldAccess) { // Very basic field detection fieldReferences.updateWith(classStack.top) { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala index 42bc648e23b4..f808d51d948a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala @@ -11,6 +11,8 @@ import io.joern.x2cpg.passes.frontend.ImportsPass.{ } import io.shiftleft.semanticcpg.language.* +import scala.collection.immutable.List + object RubyTypeRecoveryTests { def getPackageTable: PackageTable = { val packageTable = PackageTable() @@ -100,10 +102,10 @@ class RubyTypeRecoveryTests maxCall.methodFullName shouldBe "__builtin.puts" } + // TODO Need to fix this and stream line the implemented w.r.t python "conservatively present either option when an imported function uses the same name as a builtin" ignore { val List(absCall) = cpg.call("sleep").l - absCall.methodFullName shouldBe "main.rb::program.sleep" - absCall.dynamicTypeHintFullName shouldBe Seq("main.rb::program.sleep") + absCall.dynamicTypeHintFullName shouldBe Seq("__buitlin.sleep", "main.rb::program.sleep") } } @@ -212,4 +214,25 @@ class RubyTypeRecoveryTests errorCall.methodFullName shouldBe "logger::program.Logger.error" } } + + "recovery of type for call having a method with same name" should { + lazy val cpg = code(""" + |require "dbi" + | + |def connect + | puts "I am here" + |end + | + |d = DBI.connect("DBI:Mysql:TESTDB:localhost", "testuser", "test123") + |""".stripMargin) + + "have a correct type for call `connect`" in { + cpg.call("connect").methodFullName.l shouldBe List("dbi::program.DBI.connect") + // cpg.call("connect").dynamicTypeHintFullName.l shouldBe List("lskdj") + } + + "have a correct type for identifier `d`" in { + cpg.identifier("d").typeFullName.l shouldBe List("dbi::program.DBI.connect.") + } + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/ModuleTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/ModuleTests.scala index dfcc8f8f98f3..06370ef830ac 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/ModuleTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/ModuleTests.scala @@ -123,7 +123,8 @@ class ModuleTests extends RubyCode2CpgFixture { | MY_CONSTANT = 0 |end |""".stripMargin) - "member variables structure in place" in { + // TODO Ignoring below test case and the function where this is implemented treats every UpperCase node as Constant which is incorrect and causing conflicts elsewhere + "member variables structure in place" ignore { val List(moduleInit) = cpg.method(XDefines.StaticInitMethodName).l moduleInit.fullName shouldBe s"Test0.rb::program.MyNamespace.${XDefines.StaticInitMethodName}" val List(myconstant) = moduleInit.call.nameExact(Operators.fieldAccess).fieldAccess.l From 95c5824f461aec0ab44d1c59e10e1624dd6e0b75 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Mon, 24 Jul 2023 19:26:51 +0530 Subject: [PATCH 2/6] add - dummy assignment nodes for builtins and method definitions --- .../rubysrc2cpg/astcreation/AstCreator.scala | 74 +++++++++++++------ .../astcreation/AstCreatorHelper.scala | 22 ++++++ .../AstForExpressionsCreator.scala | 2 +- .../astcreation/AstForStatementsCreator.scala | 3 - .../passes/RubyTypeRecoveryTests.scala | 8 +- .../rubysrc2cpg/passes/ast/CallCpgTests.scala | 2 - .../passes/ast/MethodOneTests.scala | 2 +- .../ast/SimpleAstCreationPassTest.scala | 74 ++++++------------- .../querying/AssignmentTests.scala | 2 +- .../querying/ControlStructureTests.scala | 2 +- .../rubysrc2cpg/querying/FunctionTests.scala | 6 +- .../querying/IdentifierTests.scala | 5 +- .../rubysrc2cpg/querying/MiscTests.scala | 2 +- 13 files changed, 112 insertions(+), 92 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index 48666d16b86b..5dc8d700e713 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -48,8 +48,8 @@ class AstCreator( */ protected val methodNameAsIdentifierStack = mutable.Stack[Ast]() - protected val methodAliases = mutable.HashMap[String, String]() - protected val methodNames = mutable.HashMap[String, String]() + protected val methodAliases = mutable.HashMap[String, String]() + protected val methodNameToMethod = mutable.HashMap[String, nodes.NewMethod]() protected val methodNamesWithYield = mutable.HashSet[String]() @@ -86,6 +86,8 @@ class AstCreator( */ protected val blockChildHash = mutable.HashMap[Int, Int]() + protected val builtInCallNames = mutable.HashSet[String]() + protected def createIdentifierWithScope( ctx: ParserRuleContext, name: String, @@ -135,19 +137,57 @@ class AstCreator( } scope.popScope() - val thisParam = parameterInNode(programCtx, "this", "this", 0, false, EvaluationStrategies.BY_VALUE).typeFullName( - classStack.reverse.mkString(pathSep) - ) - val thisParamAst = Ast(thisParam) - val methodRetNode = NewMethodReturn() .lineNumber(None) .columnNumber(None) .typeFullName(Defines.Any) + // For all the builtIn's encountered create assignment ast + val lineColNum = 1 + val builtInMethodAst = builtInCallNames.map { builtInCallName => + val identifierNode = NewIdentifier() + .code(builtInCallName) + .name(builtInCallName) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + .typeFullName(Defines.Any) + val typeRefNode = NewTypeRef() + .code(prefixAsBuiltin(builtInCallName)) + .typeFullName(prefixAsBuiltin(builtInCallName)) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + astForAssignment(identifierNode, typeRefNode, Some(lineColNum), Some(lineColNum)) + }.toList + + val methodRefAssignmentAsts = methodNameToMethod.values.map { methodNode => + // Create a methodRefNode and assign it to the identifier version of the method, which will help in type propogation to resolve calls + val methodRefNode = NewMethodRef() + .code("def " + methodNode.name + "(...)") + .methodFullName(methodNode.fullName) + .typeFullName(methodNode.fullName) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + + val methodNameIdentifier = NewIdentifier() + .code(methodNode.name) + .name(methodNode.name) + .typeFullName(Defines.Any) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + + val methodRefAssignmentAst = + astForAssignment(methodNameIdentifier, methodRefNode, methodNode.lineNumber, methodNode.columnNumber) + methodRefAssignmentAst + }.toList + val blockNode = NewBlock().typeFullName(Defines.Any) val programAst = - methodAst(programMethod, Seq(thisParamAst), blockAst(blockNode, statementAsts.toList), methodRetNode) + methodAst( + programMethod, + Seq(Ast()), + blockAst(blockNode, statementAsts.toList ++ builtInMethodAst ++ methodRefAssignmentAsts), + methodRetNode + ) val fileNode = NewFile().name(filename).order(1) val namespaceBlock = globalNamespaceBlock() @@ -864,9 +904,6 @@ class AstCreator( if (ctx.block() != null) { val blockAst = Seq(astForBlock(ctx.block())) Seq(callAst(callNode, parenAst ++ blockAst)) - } else if (methodNames.contains(getActualMethodName(callNode.name))) { - val thisNode = identifierNode(ctx, "this", "this", classStack.reverse.mkString(pathSep)) - Seq(callAst(callNode, parenAst, Some(Ast(thisNode)))) } else Seq(callAst(callNode, parenAst)) } @@ -923,16 +960,11 @@ class AstCreator( "" } val name = s"${getActualMethodName(ctx.getText)}$nameSuffix" - val methodFullName = packageContext.packageTable - .getMethodFullNameUsingName(packageStack.toList, name) - .headOption match { - case None if methodNames.contains(name) => methodNames.get(name).get - case None if isBuiltin(name) => prefixAsBuiltin(name) // TODO: Probably not super precise - case Some(externalDependencyResolution) => DynamicCallUnknownFullName - case None => DynamicCallUnknownFullName - } + // Add the call name to the global builtIn callNames set + if (isBuiltin(name)) + builtInCallNames.add(name) - callAst(callNode(ctx, code, name, methodFullName, DispatchTypes.STATIC_DISPATCH)) + callAst(callNode(ctx, code, name, DynamicCallUnknownFullName, DispatchTypes.STATIC_DISPATCH)) } def astForMethodOnlyIdentifier(ctx: MethodOnlyIdentifierContext): Seq[Ast] = { @@ -1313,7 +1345,7 @@ class AstCreator( * TODO find out how they should be used. Need to do this iff it adds any value */ - methodNames.put(methodNode.name, methodFullName) + methodNameToMethod.put(methodNode.name, methodNode) val blockNode = NewBlock().typeFullName(Defines.Any) /* Before creating ast, we traverse the method params and identifiers and link them*/ diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala index 1c9cded07053..6cdac0767efa 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala @@ -1,5 +1,9 @@ package io.joern.rubysrc2cpg.astcreation +import io.joern.x2cpg.{Ast, Defines} +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} +import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeNew, NewCall, NewNode} + trait AstCreatorHelper { this: AstCreator => import GlobalTypes._ @@ -8,6 +12,24 @@ trait AstCreatorHelper { this: AstCreator => def prefixAsBuiltin(x: String): String = s"$builtinPrefix$pathSep$x" + def astForAssignment(lhs: NewNode, rhs: NewNode, lineNumber: Option[Integer], colNumber: Option[Integer]): Ast = { + + val code = codeOf(lhs) + " = " + codeOf(rhs) + val callNode = NewCall() + .name(Operators.assignment) + .code(code) + .dispatchType(DispatchTypes.STATIC_DISPATCH) + .lineNumber(lineNumber) + .columnNumber(colNumber) + .methodFullName(Operators.assignment) + + callAst(callNode, Seq(Ast(lhs), Ast(rhs))) + } + + protected def codeOf(node: NewNode): String = { + node.asInstanceOf[AstNodeNew].code + } + } object GlobalTypes { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala index c05e83ce2548..b4e1db201018 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala @@ -254,7 +254,7 @@ trait AstForExpressionsCreator { this: AstCreator => } else if (definitelyIdentifier || scope.lookupVariable(variableName).isDefined) { val node = createIdentifierWithScope(ctx, variableName, variableName, Defines.Any, List()) Ast(node) - } else if (methodNames.contains(variableName)) { + } else if (methodNameToMethod.contains(variableName)) { astForCallNode(ctx, variableName) } else if (ModifierTypes.ALL.contains(variableName.toUpperCase)) { lastModifier = Option(variableName.toUpperCase) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala index 6afe067ee2de..b7a836a568ec 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForStatementsCreator.scala @@ -256,9 +256,6 @@ trait AstForStatementsCreator { resolveRequireOrLoadPath(argsAsts, callNode) } else if (callNode.name == "require_relative") { resolveRelativePath(filename, argsAsts, callNode) - } else if (methodNames.contains(getActualMethodName(callNode.name))) { - val thisNode = identifierNode(ctx, "this", "this", classStack.reverse.mkString(pathSep)) - Seq(callAst(callNode, argsAsts, Some(Ast(thisNode)))) } else { Seq(callAst(callNode, argsAsts)) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala index 7baf39dfcf53..f0466f62bb6e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala @@ -102,10 +102,9 @@ class RubyTypeRecoveryTests maxCall.methodFullName shouldBe "__builtin.puts" } - // TODO Need to fix this and stream line the implemented w.r.t python - "conservatively present either option when an imported function uses the same name as a builtin" ignore { + "conservatively present either option when an imported function uses the same name as a builtin" in { val List(absCall) = cpg.call("sleep").l - absCall.dynamicTypeHintFullName shouldBe Seq("__buitlin.sleep", "main.rb::program.sleep") + absCall.dynamicTypeHintFullName.l shouldBe Seq("__builtin.sleep", "main.rb::program.sleep") } } @@ -233,7 +232,7 @@ class RubyTypeRecoveryTests } } - "recovery of type for call having a method with same name" ignore { + "recovery of type for call having a method with same name" should { lazy val cpg = code(""" |require "dbi" | @@ -246,7 +245,6 @@ class RubyTypeRecoveryTests "have a correct type for call `connect`" in { cpg.call("connect").methodFullName.l shouldBe List("dbi::program.DBI.connect") - cpg.call("connect").dynamicTypeHintFullName.l shouldBe List("dbi::program.DBI.connect") } "have a correct type for identifier `d`" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/CallCpgTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/CallCpgTests.scala index 85c0bb14b042..9fa2a07c0f8c 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/CallCpgTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/CallCpgTests.scala @@ -112,9 +112,7 @@ class CallCpgTests extends RubyCode2CpgFixture { callNode.signature shouldBe "" callNode.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH callNode.lineNumber shouldBe Some(14) - callNode.astChildren.head.code shouldBe "this" callNode.astChildren.last.code shouldBe "books_and_articles_we_love" - callNode.argument.head.code shouldBe "this" callNode.argument.last.code shouldBe "books_and_articles_we_love" } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/MethodOneTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/MethodOneTests.scala index 32266b6f1981..d761fc1d6a00 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/MethodOneTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/MethodOneTests.scala @@ -43,7 +43,7 @@ class MethodOneTests extends RubyCode2CpgFixture { } "should allow traversing to method" in { - cpg.methodReturn.method.name.l shouldBe List("foo", ":program") + cpg.methodReturn.method.name.l shouldBe List("foo", ":program", ".assignment") } "should allow traversing to file" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/SimpleAstCreationPassTest.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/SimpleAstCreationPassTest.scala index ed1abd3ccd37..b24028ac8872 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/SimpleAstCreationPassTest.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/ast/SimpleAstCreationPassTest.scala @@ -13,15 +13,17 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { "have correct structure for a single command call" in { val cpg = code("""puts 123""") - val List(commandCall) = cpg.call.l - val List(arg) = commandCall.argument.isLiteral.l + val List(call1, call2) = cpg.call.l + val List(arg) = call1.argument.isLiteral.l - commandCall.code shouldBe "puts 123" - commandCall.lineNumber shouldBe Some(1) + call1.code shouldBe "puts 123" + call1.lineNumber shouldBe Some(1) arg.code shouldBe "123" arg.lineNumber shouldBe Some(1) arg.columnNumber shouldBe Some(5) + + call2.name shouldBe ".assignment" // call node for builtin typeRef assignment } "have correct structure for an unsigned, decimal integer literal" in { @@ -178,8 +180,8 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { } "have correct structure for `self` identifier" in { - val cpg = code("puts self") - val List(self) = cpg.identifier.l + val cpg = code("puts self") + val List(self, _) = cpg.identifier.l self.typeFullName shouldBe Defines.Object self.code shouldBe "self" self.lineNumber shouldBe Some(1) @@ -187,8 +189,8 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { } "have correct structure for `__FILE__` identifier" in { - val cpg = code("puts __FILE__") - val List(file) = cpg.identifier.l + val cpg = code("puts __FILE__") + val List(file, _) = cpg.identifier.l file.typeFullName shouldBe "__builtin.String" file.code shouldBe "__FILE__" file.lineNumber shouldBe Some(1) @@ -196,8 +198,8 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { } "have correct structure for `__LINE__` identifier" in { - val cpg = code("puts __LINE__") - val List(line) = cpg.identifier.l + val cpg = code("puts __LINE__") + val List(line, _) = cpg.identifier.l line.typeFullName shouldBe "__builtin.Integer" line.code shouldBe "__LINE__" line.lineNumber shouldBe Some(1) @@ -205,8 +207,8 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { } "have correct structure for `__ENCODING__` identifier" in { - val cpg = code("puts __ENCODING__") - val List(encoding) = cpg.identifier.l + val cpg = code("puts __ENCODING__") + val List(encoding, _) = cpg.identifier.l encoding.typeFullName shouldBe Defines.Encoding encoding.code shouldBe "__ENCODING__" encoding.lineNumber shouldBe Some(1) @@ -279,7 +281,7 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { "have correct structure for a single-line regular expression literal passed as argument to a command" in { val cpg = code("puts /x/") - val List(callNode) = cpg.call.l + val List(callNode, _) = cpg.call.l callNode.code shouldBe "puts /x/" callNode.name shouldBe "puts" callNode.lineNumber shouldBe Some(1) @@ -380,8 +382,8 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { } "have correct structure for a call node" in { - val cpg = code("puts \"something\"") - val List(callNode) = cpg.call.l + val cpg = code("puts \"something\"") + val List(callNode, _) = cpg.call.l callNode.code shouldBe "puts \"something\"" callNode.lineNumber shouldBe Some(1) callNode.columnNumber shouldBe Some(0) @@ -841,32 +843,6 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { literalArg.lineNumber shouldBe Some(1) } - "have correct base for a call" in { - val cpg = code(""" - |def foo(x) - | puts x - |end - | - |foo 123 - |foo(132) - |""".stripMargin) - - val callWithoutParen = cpg.call("foo").lineNumber(6).l - val callWithParen = cpg.call("foo").lineNumber(7).l - - val List(base1) = callWithoutParen.argument.isIdentifier.l - base1.typeFullName shouldBe "Test0.rb::program" - - val List(argument1) = callWithoutParen.argument.isLiteral.l - argument1.code shouldBe "123" - - val List(base2) = callWithParen.argument.isIdentifier.l - base2.typeFullName shouldBe "Test0.rb::program" - - val List(argument2) = callWithParen.argument.isLiteral.l - argument2.code shouldBe "132" - } - "have generated call nodes for regex interpolation" in { val cpg = code("/x#{Regexp.quote(foo)}b#{x+'z'}a/") val List(literalNode) = cpg.literal.l @@ -910,7 +886,7 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { "have correct structure for proc definiton with procParameters and empty block" in { val cpg = code("-> (x,y) {}") - cpg.parameter.size shouldBe 3 + cpg.parameter.size shouldBe 2 } "have correct structure for proc definiton with procParameters and non-empty block" in { @@ -922,18 +898,16 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { | b |end |}""".stripMargin) - cpg.parameter.size shouldBe 3 - val List(paramOne, paramTwo, paramThree) = cpg.parameter.l - paramOne.name shouldBe "this" - paramTwo.name shouldBe "x" - paramThree.name shouldBe "y" + cpg.parameter.size shouldBe 2 + val List(paramOne, paramTwo) = cpg.parameter.l + paramOne.name shouldBe "x" + paramTwo.name shouldBe "y" cpg.ifBlock.size shouldBe 1 } "have correct structure for proc definition with no parameters and empty block" in { val cpg = code("-> {}") - cpg.parameter.size shouldBe 1 - cpg.parameter.head.code shouldBe "this" + cpg.parameter.size shouldBe 0 } "have correct structure for proc definition with additional context" in { @@ -941,7 +915,7 @@ class SimpleAstCreationPassTest extends RubyCode2CpgFixture { "scope :get_all_doctors, -> { (select('id, first_name').where('role = :user_role', user_role: User.roles[:doctor])) }" ) cpg.parameter.size shouldBe 6 - cpg.call.size shouldBe 6 + cpg.call.size shouldBe 7 } } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AssignmentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AssignmentTests.scala index 0a3c735e1620..5eef34ef2fdb 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AssignmentTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/AssignmentTests.scala @@ -19,7 +19,7 @@ class AssignmentTests extends RubyCode2CpgFixture { |""".stripMargin) "recognize all assignment nodes" in { - cpg.assignment.size shouldBe 5 + cpg.assignment.size shouldBe 6 // One assignment is for `puts = typeRef(__builtin.puts)` } "have call nodes for .assignment as method name" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ControlStructureTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ControlStructureTests.scala index 2e9f588685df..3d394441990b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ControlStructureTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ControlStructureTests.scala @@ -15,7 +15,7 @@ class ControlStructureTests extends RubyCode2CpgFixture { "recognise all identifier nodes" in { cpg.identifier.name("n").size shouldBe 1 - cpg.identifier.size shouldBe 1 + cpg.identifier.size shouldBe 2 // 1 identifier node is for `puts = typeDef(__builtin.puts)` } "recognize all call nodes" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FunctionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FunctionTests.scala index 7401e265f65f..48f887c032f7 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FunctionTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FunctionTests.scala @@ -35,7 +35,7 @@ class FunctionTests extends RubyCode2CpgFixture { cpg.identifier.name("age").size shouldBe 1 cpg.fieldAccess.fieldIdentifier.canonicalName("name").size shouldBe 2 cpg.fieldAccess.fieldIdentifier.canonicalName("age").size shouldBe 4 - cpg.identifier.size shouldBe 11 + cpg.identifier.size shouldBe 15 // 4 identifier node is for `puts = typeDef(__builtin.puts)` and methodRef's assignment } "recognize all call nodes" in { @@ -77,10 +77,10 @@ class FunctionTests extends RubyCode2CpgFixture { } "recognize all call nodes" in { - cpg.call.name(Operators.assignment).size shouldBe 3 + cpg.call.name(Operators.assignment).size shouldBe 6 // 3 identifier node is for methodRef's assignment cpg.call.name("to_s").size shouldBe 2 cpg.call.name("new").size shouldBe 1 - cpg.call.size shouldBe 11 + cpg.call.size shouldBe 14 // 3 identifier node is for methodRef's assignment } "recognize all identifier nodes" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/IdentifierTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/IdentifierTests.scala index 4c344c63c7ba..7dd2f48fa3c0 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/IdentifierTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/IdentifierTests.scala @@ -35,8 +35,7 @@ class IdentifierTests extends RubyCode2CpgFixture { cpg.identifier.name("num3").size shouldBe 1 cpg.identifier.name("sum").size shouldBe 2 cpg.identifier.name("ret").size shouldBe 2 - cpg.identifier.name("this").size shouldBe 2 - cpg.identifier.size shouldBe 16 + cpg.identifier.size shouldBe 16 // 2 identifier node is for methodRef's assigment } "identify a single call node" in { @@ -107,7 +106,7 @@ class IdentifierTests extends RubyCode2CpgFixture { "recognise all identifier nodes" in { cpg.identifier .name("create_conflict") - .size shouldBe 3 + .size shouldBe 4 // 1 identifier node is for methodRef's assignment } "recognise all call nodes" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MiscTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MiscTests.scala index 5f3cca9522f2..ea3b7e75f29e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MiscTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MiscTests.scala @@ -31,7 +31,7 @@ class MiscTests extends RubyCode2CpgFixture { cpg.identifier.name("beginbool").size shouldBe 1 cpg.identifier.name("endbool").size shouldBe 1 cpg.call.name("puts").size shouldBe 1 - cpg.identifier.size shouldBe 6 + cpg.identifier.size shouldBe 7 // 1 identifier node is for `puts = typeDef(__builtin.puts)` } } From 858c587acab08393eb680e94653cdac28f65ad1e Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Thu, 27 Jul 2023 20:19:12 +0530 Subject: [PATCH 3/6] merged --- .../rubysrc2cpg/astcreation/AstCreator.scala | 194 ++++++++++++------ .../astcreation/AstCreatorHelper.scala | 75 +++++++ .../astcreation/AstForTypesCreator.scala | 2 + .../passes/RubyTypeRecoveryPass.scala | 7 + .../rubysrc2cpg/dataflow/DataFlowTests.scala | 32 ++- .../rubysrc2cpg/querying/FunctionTests.scala | 10 +- 6 files changed, 245 insertions(+), 75 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index c195e44f317d..ed247ce66031 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -51,6 +51,8 @@ class AstCreator( protected val methodAliases = mutable.HashMap[String, String]() protected val methodNames = mutable.HashMap[String, String]() + protected val typeDeclNameToTypeDecl = mutable.HashMap[String, nodes.NewTypeDecl]() + protected val methodNamesWithYield = mutable.HashSet[String]() /* @@ -148,9 +150,75 @@ class AstCreator( .columnNumber(None) .typeFullName(Defines.Any) + // For all the builtIn's encountered create assignment ast + val lineColNum = 1 + val builtInMethodAst = builtInCallNames.map { builtInCallName => + val identifierNode = NewIdentifier() + .code(builtInCallName) + .name(builtInCallName) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + .typeFullName(Defines.Any) + val typeRefNode = NewTypeRef() + .code(prefixAsBuiltin(builtInCallName)) + .typeFullName(prefixAsBuiltin(builtInCallName)) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + astForAssignment(identifierNode, typeRefNode, Some(lineColNum), Some(lineColNum)) + }.toList + + val methodRefAssignmentAsts = methodNameToMethod.values.map { methodNode => + // Create a methodRefNode and assign it to the identifier version of the method, which will help in type propogation to resolve calls + val methodRefNode = NewMethodRef() + .code("def " + methodNode.name + "(...)") + .methodFullName(methodNode.fullName) + .typeFullName(methodNode.fullName) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + + val methodNameIdentifier = NewIdentifier() + .code(methodNode.name) + .name(methodNode.name) + .typeFullName(Defines.Any) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + + val methodRefAssignmentAst = + astForAssignment(methodNameIdentifier, methodRefNode, methodNode.lineNumber, methodNode.columnNumber) + methodRefAssignmentAst + }.toList + + val typeRefAssignmentAst = typeDeclNameToTypeDecl.values.map { typeDeclNode => + + val typeRefNode = NewTypeRef() + .code("class " + typeDeclNode.name + "(...)") + .typeFullName(typeDeclNode.fullName) + .lineNumber(typeDeclNode.lineNumber) + .columnNumber(typeDeclNode.columnNumber) + + val typeDeclNameIdentifier = NewIdentifier() + .code(typeDeclNode.name) + .name(typeDeclNode.name) + .typeFullName(Defines.Any) + .lineNumber(lineColNum) + .columnNumber(lineColNum) + + val typeRefAssignmentAst = + astForAssignment(typeDeclNameIdentifier, typeRefNode, typeDeclNode.lineNumber, typeDeclNode.columnNumber) + typeRefAssignmentAst + } + val blockNode = NewBlock().typeFullName(Defines.Any) val programAst = - methodAst(programMethod, Seq(thisParamAst), blockAst(blockNode, statementAsts.toList), methodRetNode) + methodAst( + programMethod, + Seq(Ast()), + blockAst( + blockNode, + statementAsts.toList ++ builtInMethodAst ++ methodRefAssignmentAsts ++ typeRefAssignmentAst + ), + methodRetNode + ) val fileNode = NewFile().name(filename).order(1) val namespaceBlock = globalNamespaceBlock() @@ -1088,65 +1156,43 @@ class AstCreator( private def astForParametersContext(ctx: ParametersContext): Seq[Ast] = { if (ctx == null) return Seq() - val localVarList = ListBuffer[Option[TerminalNode]]() - // NOT differentiating between the productions here since either way we get parameters - val mandatoryParameters = ctx - .parameter() - .asScala - .filter(ctx => Option(ctx.mandatoryParameter()).isDefined) - .map(ctx => Option(ctx.mandatoryParameter().LOCAL_VARIABLE_IDENTIFIER())) - val optionalParameters = ctx - .parameter() - .asScala - .filter(ctx => Option(ctx.optionalParameter()).isDefined) - .map(ctx => Option(ctx.optionalParameter().LOCAL_VARIABLE_IDENTIFIER())) - val arrayParameter = ctx - .parameter() - .asScala - .filter(ctx => Option(ctx.arrayParameter()).isDefined) - .map(ctx => Option(ctx.arrayParameter().LOCAL_VARIABLE_IDENTIFIER())) - val procParameter = ctx - .parameter() - .asScala - .filter(ctx => Option(ctx.procParameter()).isDefined) - .map(ctx => Option(ctx.procParameter().LOCAL_VARIABLE_IDENTIFIER())) - - localVarList.addAll(mandatoryParameters) - localVarList.addAll(optionalParameters) - localVarList.addAll(arrayParameter) - localVarList.addAll(procParameter) - - localVarList.map { - case localVar @ Some(paramContext) => { - val varSymbol = paramContext.getSymbol - createIdentifierWithScope(ctx, varSymbol.getText, varSymbol.getText, Defines.Any, Seq[String](Defines.Any)) - val param = NewMethodParameterIn() - .name(varSymbol.getText) - .code(varSymbol.getText) - .lineNumber(varSymbol.getLine) - .typeFullName(Defines.Any) - .columnNumber(varSymbol.getCharPositionInLine) - if (Option(arrayParameter).isDefined) { - param.isVariadic = true - } - Ast(param) - } - case localVar @ _ => { - val identifierName = getUnusedVariableNames(usedVariableNames, Defines.TempIdentifier) - val parameterName = getUnusedVariableNames(usedVariableNames, Defines.TempParameter) - createIdentifierWithScope(ctx, identifierName, identifierName, Defines.Any, Seq[String](Defines.Any)) - val param = NewMethodParameterIn() - .name(parameterName) - .code(parameterName) - .lineNumber(None) - .typeFullName(Defines.Any) - .columnNumber(None) - if (Option(arrayParameter).isDefined) { - param.isVariadic = true - } - Ast(param) - } - }.toSeq + + // the parameterTupleList holds the parameter terminal node and is the parameter a variadic parameter + val parameterTupleList = ctx.parameter().asScala.map { + case procCtx if procCtx.procParameter() != null => + (Option(procCtx.procParameter().LOCAL_VARIABLE_IDENTIFIER()), false) + case optCtx if optCtx.optionalParameter() != null => + (Option(optCtx.optionalParameter().LOCAL_VARIABLE_IDENTIFIER()), false) + case manCtx if manCtx.mandatoryParameter() != null => + (Option(manCtx.mandatoryParameter().LOCAL_VARIABLE_IDENTIFIER()), false) + case arrCtx if arrCtx.arrayParameter() != null => + (Option(arrCtx.arrayParameter().LOCAL_VARIABLE_IDENTIFIER()), arrCtx.arrayParameter().STAR() != null) + case _ => (None, false) + } + + parameterTupleList.zipWithIndex.map { case (paraTuple, paraIndex) => + paraTuple match + case (Some(paraValue), isVariadic) => + val varSymbol = paraValue.getSymbol + createIdentifierWithScope(ctx, varSymbol.getText, varSymbol.getText, Defines.Any, Seq[String](Defines.Any)) + Ast( + createMethodParameterIn( + varSymbol.getText, + lineNumber = Some(varSymbol.getLine), + colNumber = Some(varSymbol.getCharPositionInLine), + order = paraIndex + 1, + index = paraIndex + 1 + ).isVariadic(isVariadic) + ) + case _ => + Ast( + createMethodParameterIn( + getUnusedVariableNames(usedVariableNames, Defines.TempParameter), + order = paraIndex + 1, + index = paraIndex + 1 + ) + ) + }.toList } // TODO: Rewrite for simplicity and take into account more than parameter names. @@ -1281,9 +1327,25 @@ class AstCreator( def astForMethodDefinitionContext(ctx: MethodDefinitionContext): Seq[Ast] = { scope.pushNewScope(()) - val astMethodParamSeq = astForMethodParameterPartContext(ctx.methodParameterPart()) - val astMethodName = astForMethodNamePartContext(ctx.methodNamePart()) - val callNode = astMethodName.head.nodes.filter(node => node.isInstanceOf[NewCall]).head.asInstanceOf[NewCall] + val astMethodName = astForMethodNamePartContext(ctx.methodNamePart()) + val callNode = astMethodName.head.nodes.filter(node => node.isInstanceOf[NewCall]).head.asInstanceOf[NewCall] + + // Create thisParameter if this is an instance method + // TODO may need to revisit to make this more robust + val astMethodParamSeq = ctx.methodNamePart() match { + case _: SimpleMethodNamePartContext if !classStack.top.endsWith(":program") => + val thisParameterNode = createMethodParameterIn( + "this", + typeFullName = callNode.methodFullName, + lineNumber = callNode.lineNumber, + colNumber = callNode.columnNumber, + index = 0, + order = 0 + ) + Seq(Ast(thisParameterNode)) ++ astForMethodParameterPartContext(ctx.methodParameterPart()) + case _ => astForMethodParameterPartContext(ctx.methodParameterPart()) + } + // there can be only one call node val astBody = astForBodyStatementContext(ctx.bodyStatement(), true) scope.popScope() @@ -1362,12 +1424,12 @@ class AstCreator( .filter(_.isInstanceOf[NewMethodParameterIn]) .asInstanceOf[Seq[NewMethodParameterIn]] ) - .foreach(paramNode => { + .foreach { paramNode => val linkIdentifiers = identifiers.filter(_.name == paramNode.name) - identifiers.foreach { identifier => + linkIdentifiers.foreach { identifier => diffGraph.addEdge(identifier, paramNode, EdgeTypes.REF) } - }) + } Seq( methodAst( diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala index f8bc0a5cedd3..4c55a9168f27 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala @@ -1,5 +1,15 @@ package io.joern.rubysrc2cpg.astcreation +import io.joern.x2cpg.{Ast, Defines} +import io.joern.rubysrc2cpg.passes.Defines as RubyDefines +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, Operators, nodes} +import io.shiftleft.codepropertygraph.generated.nodes.{ + AstNodeNew, + NewCall, + NewFieldIdentifier, + NewMethodParameterIn, + NewNode +} import scala.collection.mutable trait AstCreatorHelper { this: AstCreator => @@ -10,12 +20,77 @@ trait AstCreatorHelper { this: AstCreator => def prefixAsBuiltin(x: String): String = s"$builtinPrefix$pathSep$x" + def astForAssignment(lhs: NewNode, rhs: NewNode, lineNumber: Option[Integer], colNumber: Option[Integer]): Ast = { + + val code = codeOf(lhs) + " = " + codeOf(rhs) + val callNode = NewCall() + .name(Operators.assignment) + .code(code) + .dispatchType(DispatchTypes.STATIC_DISPATCH) + .lineNumber(lineNumber) + .columnNumber(colNumber) + .methodFullName(Operators.assignment) + + callAst(callNode, Seq(Ast(lhs), Ast(rhs))) + } + + protected def createFieldAccess( + baseNode: NewNode, + fieldName: String, + lineNumber: Option[Integer], + colNumber: Option[Integer] + ) = { + val fieldIdNode = NewFieldIdentifier() + .code(fieldName) + .canonicalName(fieldName) + .lineNumber(lineNumber) + .columnNumber(colNumber) + + val baseNodeCopy = baseNode.copy + val code = codeOf(baseNode) + "." + codeOf(fieldIdNode) + val callNode = NewCall() + .code(code) + .name(Operators.fieldAccess) + .methodFullName(Operators.fieldAccess) + .dispatchType(DispatchTypes.STATIC_DISPATCH) + .lineNumber(lineNumber) + .columnNumber(colNumber) + + callAst(callNode, Seq(Ast(baseNodeCopy), Ast(fieldIdNode))) + } + + protected def createMethodParameterIn( + name: String, + lineNumber: Option[Integer] = None, + colNumber: Option[Integer] = None, + typeFullName: String = RubyDefines.Any, + order: Int = -1, + index: Int = -1 + ) = { + NewMethodParameterIn() + .name(name) + .code(name) + .lineNumber(lineNumber) + .typeFullName(typeFullName) + .columnNumber(colNumber) + .order(order) + .index(index) + } + + protected def codeOf(node: NewNode): String = { + node.asInstanceOf[AstNodeNew].code + } + def getUnusedVariableNames(usedVariableNames: mutable.HashMap[String, Int], variableName: String): String = { val counter = usedVariableNames.get(variableName).map(_ + 1).getOrElse(0) val currentVariableName = s"${variableName}_$counter" usedVariableNames.put(variableName, counter) currentVariableName } + + protected def addReceiverEdge(dstNode: nodes.NewNode, srcNode: nodes.NewNode): Unit = { + diffGraph.addEdge(srcNode, dstNode, EdgeTypes.RECEIVER) + } } object GlobalTypes { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala index 1b75db709b62..8c43241875ab 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForTypesCreator.scala @@ -51,6 +51,8 @@ trait AstForTypesCreator { this: AstCreator => val typeDeclNode = NewTypeDecl() .name(className) .fullName(fullName) + + typeDeclNameToTypeDecl.put(className, typeDeclNode) Seq(Ast(typeDeclNode).withChildren(bodyAst)) } else { Seq.empty diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPass.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPass.scala index 55e48ca41b3a..ab7c40cbf306 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPass.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryPass.scala @@ -121,4 +121,11 @@ private class RecoverForRubyFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder, symbolTable.append(c, callTypes) } + override protected def visitIdentifierAssignedToTypeRef(i: Identifier, t: TypeRef, rec: Option[String]): Set[String] = + t.typ.referencedTypeDecl + .map(_.fullName.stripSuffix("")) + .map(td => symbolTable.append(CallAlias(i.name, rec), Set(td))) + .headOption + .getOrElse(super.visitIdentifierAssignedToTypeRef(i, t, rec)) + } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala index 5e22c70f8425..6fbd3ec8e788 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala @@ -1,6 +1,7 @@ package io.joern.rubysrc2cpg.dataflow import io.joern.dataflowengineoss.language.* +import io.joern.rubysrc2cpg.RubySrc2Cpg import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.shiftleft.semanticcpg.language.* @@ -319,8 +320,8 @@ class DataFlowTests extends RubyCode2CpgFixture(withPostProcessing = true, withD sink.reachableByFlows(src).l.size shouldBe 2 } } - // TODO: - "Data flow through class member" ignore { + + "Data flow through class member" should { val cpg = code(""" |class MyClass | @instanceVariable @@ -1781,8 +1782,7 @@ class DataFlowTests extends RubyCode2CpgFixture(withPostProcessing = true, withD } } - // TODO: Need to be fixed. - "Across the file data flow test" ignore { + "Across the file data flow test" should { val cpg = code( """ |def foo(arg) @@ -1815,7 +1815,8 @@ class DataFlowTests extends RubyCode2CpgFixture(withPostProcessing = true, withD sink.reachableByFlows(src).size shouldBe 1 } - "be found for sink in nested block" in { + // TODO: Need to be fixed. + "be found for sink in nested block" ignore { val src = cpg.identifier("x").lineNumber(3).l val sink = cpg.call.name("puts").argument(1).lineNumber(7).l sink.reachableByFlows(src).size shouldBe 1 @@ -2519,3 +2520,24 @@ class DataFlowTests extends RubyCode2CpgFixture(withPostProcessing = true, withD sink.reachableByFlows(source).size shouldBe 2 } } + +class trail extends RubyCode2CpgFixture(withPostProcessing = true, withDataFlow = true) { + "Data flow through array constructor splattingOnlyIndexingArguments" should { + val cpg = code(""" + |def foo(*splat_args) + |array = [*splat_args] + |puts array + |end + | + |x = 1 + |y = 2 + |y = foo(x,y) + |""".stripMargin) + + "find flows to the sink" in { + val source = cpg.identifier.name("x").l + val sink = cpg.call.name("puts").l + sink.reachableByFlows(source).l.size shouldBe 2 + } + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FunctionTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FunctionTests.scala index 7401e265f65f..1dc874d167a1 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FunctionTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/FunctionTests.scala @@ -26,7 +26,7 @@ class FunctionTests extends RubyCode2CpgFixture { | end |end | - |p = Person. new + |p = Person.new |p.greet |""".stripMargin) @@ -35,7 +35,7 @@ class FunctionTests extends RubyCode2CpgFixture { cpg.identifier.name("age").size shouldBe 1 cpg.fieldAccess.fieldIdentifier.canonicalName("name").size shouldBe 2 cpg.fieldAccess.fieldIdentifier.canonicalName("age").size shouldBe 4 - cpg.identifier.size shouldBe 11 + cpg.identifier.size shouldBe 12 // 4 identifier node is for `puts = typeDef(__builtin.puts)` and methodRef's assignment, 1 node for class Person = typeDef } "recognize all call nodes" in { @@ -77,10 +77,12 @@ class FunctionTests extends RubyCode2CpgFixture { } "recognize all call nodes" in { - cpg.call.name(Operators.assignment).size shouldBe 3 + cpg.call + .name(Operators.assignment) + .size shouldBe 7 // 3 identifier node is for methodRef's assignment, 1 identifier node for TypeRef's assignment cpg.call.name("to_s").size shouldBe 2 cpg.call.name("new").size shouldBe 1 - cpg.call.size shouldBe 11 + cpg.call.size shouldBe 12 // 3 identifier node is for methodRef's assignment, 1 identifier node for TypeRef's assignment } "recognize all identifier nodes" in { From 08b5f49b82e94e53f783049cfcf03316d9733735 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Thu, 27 Jul 2023 20:32:55 +0530 Subject: [PATCH 4/6] remove dummy test case --- .../rubysrc2cpg/dataflow/DataFlowTests.scala | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala index 6fbd3ec8e788..e732378f14fb 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/dataflow/DataFlowTests.scala @@ -2520,24 +2520,3 @@ class DataFlowTests extends RubyCode2CpgFixture(withPostProcessing = true, withD sink.reachableByFlows(source).size shouldBe 2 } } - -class trail extends RubyCode2CpgFixture(withPostProcessing = true, withDataFlow = true) { - "Data flow through array constructor splattingOnlyIndexingArguments" should { - val cpg = code(""" - |def foo(*splat_args) - |array = [*splat_args] - |puts array - |end - | - |x = 1 - |y = 2 - |y = foo(x,y) - |""".stripMargin) - - "find flows to the sink" in { - val source = cpg.identifier.name("x").l - val sink = cpg.call.name("puts").l - sink.reachableByFlows(source).l.size shouldBe 2 - } - } -} From fd87195225f3b37d07060c9455cc207c77698902 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Tue, 1 Aug 2023 18:40:44 +0530 Subject: [PATCH 5/6] scalafmt --- .../scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index 8cb17658d10f..41cb3ca593ca 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -1341,7 +1341,7 @@ class AstCreator( astForMethodNamePartContext(ctxMethodNamePart) case None => astForMethodIdentifierContext(ctx.methodIdentifier(), ctx.getText) - val callNode = astMethodName.head.nodes.filter(node => node.isInstanceOf[NewCall]).head.asInstanceOf[NewCall] + val callNode = astMethodName.head.nodes.filter(node => node.isInstanceOf[NewCall]).head.asInstanceOf[NewCall] // Create thisParameter if this is an instance method // TODO may need to revisit to make this more robust From 11e24ff60934923b93368bdb8ece7b2e6be98ec0 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Tue, 1 Aug 2023 18:45:07 +0530 Subject: [PATCH 6/6] removed unused method --- .../io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala index df6f72a6a663..5718b7035f1e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala @@ -86,10 +86,6 @@ trait AstCreatorHelper { this: AstCreator => usedVariableNames.put(variableName, counter) currentVariableName } - - protected def addReceiverEdge(dstNode: nodes.NewNode, srcNode: nodes.NewNode): Unit = { - diffGraph.addEdge(srcNode, dstNode, EdgeTypes.RECEIVER) - } } object GlobalTypes {