From 9d79faa6b92bb0d7a2783bb49eb7350f6ed0a25b Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Fri, 15 Dec 2023 13:30:18 +0200 Subject: [PATCH] [c#] Classes, Methods, and Method Decorators (#3962) * Generating type declarations with modifiers * Generating method nodes with modifiers, a block, parameters, and a method return Resolves #3854 Resolves #3855 Resolves #3951 --- .../astcreation/AstCreator.scala | 6 +- .../astcreation/AstCreatorHelper.scala | 33 ++++- .../AstForDeclarationsCreator.scala | 124 +++++++++++++++--- .../csharpsrc2cpg/parser/DotNetJsonAst.scala | 48 ++++--- .../querying/ast/MethodTests.scala | 41 ++++++ .../querying/ast/TypeDeclTests.scala | 50 +++++++ 6 files changed, 267 insertions(+), 35 deletions(-) create mode 100644 joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/MethodTests.scala create mode 100644 joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/TypeDeclTests.scala diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala index 28770f7dd912..c2bc00681248 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreator.scala @@ -1,6 +1,6 @@ package io.joern.csharpsrc2cpg.astcreation -import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.NamespaceDeclaration +import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.* import io.joern.csharpsrc2cpg.parser.{DotNetNodeInfo, ParserKeys} import io.joern.x2cpg.astgen.{AstGenNodeBuilder, ParserResult} import io.joern.x2cpg.datastructures.Stack.Stack @@ -52,6 +52,10 @@ class AstCreator(val relativeFileName: String, val parserResult: ParserResult)(i protected def astForNode(nodeInfo: DotNetNodeInfo): Ast = { nodeInfo.node match { case NamespaceDeclaration => astForNamespaceDeclaration(nodeInfo) + case ClassDeclaration => astForClassDeclaration(nodeInfo) + case MethodDeclaration => astForMethodDeclaration(nodeInfo) + case UsingDirective => notHandledYet(nodeInfo) + case Block => notHandledYet(nodeInfo) case _ => notHandledYet(nodeInfo) } } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala index b0ba35608ab6..d631dcdcd980 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstCreatorHelper.scala @@ -1,8 +1,9 @@ package io.joern.csharpsrc2cpg.astcreation -import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.DotNetParserNode +import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.* import io.joern.csharpsrc2cpg.parser.{DotNetJsonAst, DotNetNodeInfo, ParserKeys} import io.joern.x2cpg.{Ast, ValidationMode} +import io.shiftleft.codepropertygraph.generated.nodes.{NewMethod, NewNamespaceBlock, NewTypeDecl} import ujson.Value import scala.util.Try @@ -59,4 +60,34 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As private def nodeType(node: Value): DotNetParserNode = DotNetJsonAst.fromString(node(ParserKeys.Kind).str, this.relativeFileName) + protected def astFullName(node: DotNetNodeInfo): String = { + methodAstParentStack.headOption match + case Some(head: NewNamespaceBlock) => s"${head.fullName}.${nameFromNode(node)}" + case Some(head: NewMethod) => s"${head.fullName}.${nameFromNode(node)}" + case Some(head: NewTypeDecl) => s"${head.fullName}.${nameFromNode(node)}" + case _ => nameFromNode(node) + } + + protected def nameFromNode(identifierNode: DotNetNodeInfo): String = { + identifierNode.node match + case IdentifierName | Parameter => nameFromIdentifier(identifierNode) + case QualifiedName => nameFromQualifiedName(identifierNode) + case _: DeclarationExpr => nameFromDeclaration(identifierNode) + case _ => "" + } + + protected def nameFromIdentifier(identifier: DotNetNodeInfo): String = { + identifier.json(ParserKeys.Identifier).obj(ParserKeys.Value).str + } + + protected def nameFromDeclaration(node: DotNetNodeInfo): String = { + node.json(ParserKeys.Identifier).obj(ParserKeys.Value).str + } + + protected def nameFromQualifiedName(qualifiedName: DotNetNodeInfo): String = { + val rhs = nameFromNode(createDotNetNodeInfo(qualifiedName.json(ParserKeys.Right))) + val lhs = nameFromNode(createDotNetNodeInfo(qualifiedName.json(ParserKeys.Left))) + s"$lhs.$rhs" + } + } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala index fce022f12506..e45eb42f85ed 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/astcreation/AstForDeclarationsCreator.scala @@ -1,19 +1,21 @@ package io.joern.csharpsrc2cpg.astcreation -import io.joern.csharpsrc2cpg.parser.DotNetJsonAst.{IdentifierName, QualifiedName} import io.joern.csharpsrc2cpg.parser.{DotNetNodeInfo, ParserKeys} import io.joern.x2cpg.datastructures.Stack.StackWrapper +import io.joern.x2cpg.utils.NodeBuilders.newModifierNode import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.nodes.NewNamespaceBlock +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.{ModifierTypes, NodeTypes} +import io.shiftleft.proto.cpg.Cpg.EvaluationStrategies + +import scala.util.Try trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => protected def astForNamespaceDeclaration(namespace: DotNetNodeInfo): Ast = { val nameNode = createDotNetNodeInfo(namespace.json(ParserKeys.Name)) - val fullName = methodAstParentStack.headOption match - case Some(head: NewNamespaceBlock) => s"${head.fullName}.${stringFromIdentifierNode(nameNode)}" - case _ => stringFromIdentifierNode(nameNode) - val name = fullName.split('.').filterNot(_.isBlank).lastOption.getOrElse(fullName) + val fullName = astFullName(nameNode) + val name = fullName.split('.').filterNot(_.isBlank).lastOption.getOrElse(fullName) val namespaceBlock = NewNamespaceBlock() .name(name) .code(code(nameNode)) @@ -28,21 +30,109 @@ trait AstForDeclarationsCreator(implicit withSchemaValidation: ValidationMode) { .withChildren(memberAsts) } - protected def stringFromIdentifierNode(identifierNode: DotNetNodeInfo): String = { - identifierNode.node match - case IdentifierName => stringFromIdentifierName(identifierNode) - case QualifiedName => stringFromQualifiedName(identifierNode) - case _ => "" + protected def astForClassDeclaration(classDecl: DotNetNodeInfo): Ast = { + val name = nameFromIdentifier(classDecl) + val fullName = astFullName(classDecl) + val typeDecl = typeDeclNode(classDecl, name, fullName, relativeFileName, code(classDecl)) + methodAstParentStack.push(typeDecl) + val modifiers = astForModifiers(classDecl) + val members = astForMembers(classDecl.json(ParserKeys.Members).arr.map(createDotNetNodeInfo).toSeq) + methodAstParentStack.pop() + Ast(typeDecl) + .withChildren(modifiers) + .withChildren(members) + } + + protected def astForMethodDeclaration(methodDecl: DotNetNodeInfo): Ast = { + val name = nameFromIdentifier(methodDecl) + val params = methodDecl + .json(ParserKeys.ParameterList) + .obj(ParserKeys.Parameters) + .arr + .map(createDotNetNodeInfo) + .zipWithIndex + .map(astForParameter) + .toSeq + val body = astForMethodBody(createDotNetNodeInfo(methodDecl.json(ParserKeys.Body))) + val methodReturn = nodeToMethodReturn(createDotNetNodeInfo(methodDecl.json(ParserKeys.ReturnType))) + val signature = + methodSignature(methodReturn, params.flatMap(_.nodes.collectFirst { case x: NewMethodParameterIn => x })) + val fullName = s"${astFullName(methodDecl)}:$signature" + val methodNode_ = methodNode(methodDecl, name, code(methodDecl), fullName, Option(signature), relativeFileName) + val modifiers = astForModifiers(methodDecl).flatMap(_.nodes).collect { case x: NewModifier => x } + methodAst(methodNode_, params, body, methodReturn, modifiers) + } + + private def methodSignature(methodReturn: NewMethodReturn, params: Seq[NewMethodParameterIn]): String = { + s"${methodReturn.typeFullName}(${params.map(_.typeFullName).mkString(",")})" + } + + private def astForParameter(paramNode: DotNetNodeInfo, idx: Int): Ast = { + val name = nameFromNode(paramNode) + val isVariadic = false // TODO + val typeFullName = Option("ANY") // TODO + val evaluationStrategy = EvaluationStrategies.BY_SHARING.name // TODO + val param = parameterInNode(paramNode, name, code(paramNode), idx, isVariadic, evaluationStrategy, typeFullName) + Ast(param) + } + + private def astForMethodBody(body: DotNetNodeInfo): Ast = { + val block = blockNode(body) + val statements = List.empty // TODO + blockAst(block, statements) + } + + private def nodeToMethodReturn(methodReturn: DotNetNodeInfo): NewMethodReturn = { + methodReturnNode( + methodReturn, + Try(methodReturn.json(ParserKeys.Value).str) + .orElse(Try(methodReturn.json(ParserKeys.Keyword).obj(ParserKeys.Value).str)) + .getOrElse("ANY") + ) } - protected def stringFromIdentifierName(identifierName: DotNetNodeInfo): String = { - identifierName.json(ParserKeys.Identifier).obj(ParserKeys.Value).str + /** Parses the modifier array and handles implicit defaults. + * @see + * https://learn.microsoft.com/en-us/dotnet/csharp/programming-guide/classes-and-structs/access-modifiers + */ + private def astForModifiers(declaration: DotNetNodeInfo): Seq[Ast] = { + val allModifiers = declaration.json(ParserKeys.Modifiers).arr.flatMap(astForModifier).toList + val accessModifiers = allModifiers + .flatMap(_.nodes) + .collect { case x: NewModifier => x.modifierType } intersect List( + ModifierTypes.PUBLIC, + ModifierTypes.PRIVATE, + ModifierTypes.INTERNAL, + ModifierTypes.PROTECTED + ) + accessModifiers match + // Internal is default for top-level definitions + case Nil + if methodAstParentStack.isEmpty || !methodAstParentStack + .take(2) + .map(_.label()) + .distinct + .contains(NodeTypes.METHOD) => + Ast(newModifierNode(ModifierTypes.INTERNAL)) :: allModifiers + // Private is default for nested definitions + case Nil + if methodAstParentStack.headOption.exists(x => x.isInstanceOf[NewMethod] || x.isInstanceOf[NewTypeDecl]) => + Ast(newModifierNode(ModifierTypes.PRIVATE)) :: allModifiers + case _ => allModifiers } - protected def stringFromQualifiedName(qualifiedName: DotNetNodeInfo): String = { - val rhs = stringFromIdentifierNode(createDotNetNodeInfo(qualifiedName.json(ParserKeys.Right))) - val lhs = stringFromIdentifierNode(createDotNetNodeInfo(qualifiedName.json(ParserKeys.Left))) - s"$lhs.$rhs" + private def astForModifier(modifier: ujson.Value): Option[Ast] = { + Option { + modifier(ParserKeys.Value).str match + case "public" => newModifierNode(ModifierTypes.PUBLIC) + case "private" => newModifierNode(ModifierTypes.PRIVATE) + case "internal" => newModifierNode(ModifierTypes.INTERNAL) + case "static" => newModifierNode(ModifierTypes.STATIC) + case "readonly" => newModifierNode(ModifierTypes.READONLY) + case x => + logger.warn(s"Unhandled modifier name '$x'") + null + }.map(Ast(_)) } } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/parser/DotNetJsonAst.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/parser/DotNetJsonAst.scala index d9365567f58a..a69dcfd0f947 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/parser/DotNetJsonAst.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/parser/DotNetJsonAst.scala @@ -34,7 +34,17 @@ object DotNetJsonAst { object ClassDeclaration extends DeclarationExpr - object UsingDirective extends DeclarationExpr + object MethodDeclaration extends DeclarationExpr + + object UsingDirective extends BaseExpr + + object Parameter extends BaseExpr + + sealed trait TypeIdentifier extends BaseExpr + + object PredefinedType extends TypeIdentifier + + object Block extends BaseExpr sealed trait IdentifierNode extends BaseExpr @@ -46,19 +56,25 @@ object DotNetJsonAst { object ParserKeys { - val FileName = "FileName" - val AstRoot = "AstRoot" - val MetaData = "MetaData" - val Kind = "Kind" - val LineStart = "LineStart" - val LineEnd = "LineEnd" - val ColumnStart = "ColumnStart" - val ColumnEnd = "ColumnEnd" - val Usings = "Usings" - val Members = "Members" - val Name = "Name" - val Value = "Value" - val Identifier = "Identifier" - val Right = "Right" - val Left = "Left" + val FileName = "FileName" + val AstRoot = "AstRoot" + val Body = "Body" + val MetaData = "MetaData" + val Kind = "Kind" + val LineStart = "LineStart" + val LineEnd = "LineEnd" + val ColumnStart = "ColumnStart" + val ColumnEnd = "ColumnEnd" + val Keyword = "Keyword" + val Usings = "Usings" + val Members = "Members" + val Modifiers = "Modifiers" + val Name = "Name" + val Parameters = "Parameters" + val ParameterList = "ParameterList" + val Value = "Value" + val Identifier = "Identifier" + val ReturnType = "ReturnType" + val Right = "Right" + val Left = "Left" } diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/MethodTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/MethodTests.scala new file mode 100644 index 000000000000..3d22a4df7f82 --- /dev/null +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/MethodTests.scala @@ -0,0 +1,41 @@ +package io.joern.csharpsrc2cpg.querying.ast + +import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture +import io.shiftleft.codepropertygraph.generated.ModifierTypes +import io.shiftleft.semanticcpg.language.* + +class MethodTests extends CSharpCode2CpgFixture { + + "a basic class declaration with method" should { + val cpg = code(basicBoilerplate(), "Program.cs") + + "generate a method node with type decl parent" in { + val x = cpg.method.nameExact("Main").head + x.fullName should startWith("HelloWorld.Program.Main:void") + // TODO: Extract types from parameters and these should work +// x.fullName shouldBe "HelloWorld.Program.Main:void(string[])" +// x.signature shouldBe "void(string[])" + x.filename shouldBe "Program.cs" + + x.typeDecl match + case Some(typeDecl) => typeDecl.name shouldBe "Program" + case None => fail("No TYPE_DECL parent found!") + } + + "generate a method node with the correct modifiers" in { + val List(x, y) = cpg.method.nameExact("Main").modifier.l: @unchecked + x.modifierType shouldBe ModifierTypes.INTERNAL + y.modifierType shouldBe ModifierTypes.STATIC + } + + "generate a method node with a parameter" in { + val List(x) = cpg.method.nameExact("Main").parameter.l: @unchecked + x.name shouldBe "args" + } + + "generate a method node with a block" in { + cpg.method.nameExact("Main").body.l should not be empty + } + } + +} diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/TypeDeclTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/TypeDeclTests.scala new file mode 100644 index 000000000000..ec85b1962f68 --- /dev/null +++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/TypeDeclTests.scala @@ -0,0 +1,50 @@ +package io.joern.csharpsrc2cpg.querying.ast + +import io.joern.csharpsrc2cpg.testfixtures.CSharpCode2CpgFixture +import io.shiftleft.semanticcpg.language.* +import io.shiftleft.codepropertygraph.generated.ModifierTypes + +class TypeDeclTests extends CSharpCode2CpgFixture { + + "a basic class declaration" should { + val cpg = code("public class Container { }", "Container.cs") + + "generate a type declaration with the correct properties" in { + val x = cpg.typeDecl.nameExact("Container").head + x.fullName shouldBe "Container" + x.filename shouldBe "Container.cs" + x.aliasTypeFullName shouldBe None + x.inheritsFromTypeFullName shouldBe Seq.empty + } + + "generate a type declaration with the correct modifiers" in { + val x = cpg.typeDecl.nameExact("Container").head + x.modifier.modifierType.head shouldBe ModifierTypes.PUBLIC + } + } + + "a basic class declaration within a namespace" should { + val cpg = code( + """namespace SampleNamespace + |{ + | private class SampleClass { } + |} + |""".stripMargin, + "SampleClass.cs" + ) + + "generate a type declaration with the correct properties" in { + val x = cpg.typeDecl.nameExact("SampleClass").head + x.fullName shouldBe "SampleNamespace.SampleClass" + x.filename shouldBe "SampleClass.cs" + x.aliasTypeFullName shouldBe None + x.inheritsFromTypeFullName shouldBe Seq.empty + } + + "generate a type declaration with the correct modifiers" in { + val x = cpg.typeDecl.nameExact("SampleClass").head + x.modifier.modifierType.head shouldBe ModifierTypes.PRIVATE + } + } + +}