diff --git a/README.md b/README.md index 05c3d6c..c8ac0c4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for you ## Requirements -- Java 17 - 20 +- Java 17 - 21 - Python > 3.8.1 - Node.js > 16 (To run [atom](https://github.com/AppThreat/atom)) - Minimum 16GB RAM diff --git a/build.sbt b/build.sbt index 9950fb5..1efeb56 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "chen" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "1.0.2" +ThisBuild / version := "1.0.3" ThisBuild / scalaVersion := "3.3.1" val cpgVersion = "1.4.22" diff --git a/ci/Dockerfile b/ci/Dockerfile index dcecbfb..d1fa75d 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \ org.opencontainers.image.authors="Team AppThreat " \ org.opencontainers.image.source="https://github.com/appthreat/chen" \ org.opencontainers.image.url="https://github.com/appthreat/chen" \ - org.opencontainers.image.version="1.0.0" \ + org.opencontainers.image.version="1.0.x" \ org.opencontainers.image.vendor="appthreat" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.title="chen" \ diff --git a/codemeta.json b/codemeta.json index 6622126..4a7b14c 100644 --- a/codemeta.json +++ b/codemeta.json @@ -7,7 +7,7 @@ "downloadUrl": "https://github.com/AppThreat/chen", "issueTracker": "https://github.com/AppThreat/chen/issues", "name": "chen", - "version": "1.0.2", + "version": "1.0.3", "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.", "applicationCategory": "code-analysis", "keywords": [ diff --git a/console/src/main/scala/io/appthreat/console/Console.scala b/console/src/main/scala/io/appthreat/console/Console.scala index 6d36e99..be7a844 100644 --- a/console/src/main/scala/io/appthreat/console/Console.scala +++ b/console/src/main/scala/io/appthreat/console/Console.scala @@ -502,18 +502,22 @@ class Console[T <: Project]( atom.file.whereNot(_.name("")).foreach { f => val childTree = richTreeLib.Tree(f.name, highlight = true) f.method.foreach(m => - val mtree = childTree.add(m.fullName) + val addedMethods = mutable.Map.empty[String, Boolean] + val mtree = childTree.add(m.fullName) if includeCalls then m.call .filterNot(_.name.startsWith(" - mtree - .add( - c.methodFullName + (if c.callee(NoResolve).head.isExternal - then " :right_arrow_curving_up:" - else "") - ) + if !addedMethods.contains(c.methodFullName) then + mtree + .add( + c.methodFullName + (if c.callee(NoResolve).head.isExternal + then " :right_arrow_curving_up:" + else "") + ) + addedMethods += c.methodFullName -> true ) ) rootTree.add(childTree) diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/C2Cpg.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/C2Cpg.scala index 9174b9a..143e003 100644 --- a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/C2Cpg.scala +++ b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/C2Cpg.scala @@ -1,14 +1,18 @@ package io.appthreat.c2cpg import io.appthreat.c2cpg.datastructures.CGlobal -import io.appthreat.c2cpg.passes.{AstCreationPass, PreprocessorPass, TypeDeclNodePass} +import io.appthreat.c2cpg.passes.{ + AstCreationPass, + ConfigFileCreationPass, + PreprocessorPass, + TypeDeclNodePass +} import io.appthreat.c2cpg.utils.Report -import io.appthreat.c2cpg.passes.{AstCreationPass, PreprocessorPass, TypeDeclNodePass} -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.Languages -import io.appthreat.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass} import io.appthreat.x2cpg.X2Cpg.withNewEmptyCpg import io.appthreat.x2cpg.X2CpgFrontend +import io.appthreat.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass} +import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.Languages import scala.util.Try @@ -20,6 +24,7 @@ class C2Cpg extends X2CpgFrontend[Config]: withNewEmptyCpg(config.outputPath, config) { (cpg, config) => new MetaDataPass(cpg, Languages.NEWC, config.inputPath).createAndApply() new AstCreationPass(cpg, config, report).createAndApply() + new ConfigFileCreationPass(cpg).createAndApply() TypeNodePass.withRegisteredTypes(CGlobal.typesSeen(), cpg).createAndApply() new TypeDeclNodePass(cpg)(config.schemaValidation).createAndApply() report.print() diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstCreatorHelper.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstCreatorHelper.scala index c407523..24b07dd 100644 --- a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstCreatorHelper.scala +++ b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/AstCreatorHelper.scala @@ -1,6 +1,7 @@ package io.appthreat.c2cpg.astcreation import io.appthreat.c2cpg.datastructures.CGlobal +import io.appthreat.c2cpg.parser.FileDefaults import io.shiftleft.codepropertygraph.generated.nodes.{ExpressionNew, NewNode} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators} import io.appthreat.x2cpg.{Ast, SourceFiles, ValidationMode} @@ -280,6 +281,9 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode): cleanedName.split(Defines.qualifiedNameSeparator).lastOption.getOrElse(cleanedName) protected def fullName(node: IASTNode): String = + val filename = fileName(node) + val lineNo: Integer = line(node).getOrElse(-1) + val lineNoEnd: Integer = lineEnd(node).getOrElse(-1) val qualifiedName: String = node match case d: CPPASTIdExpression if d.getEvaluation.isInstanceOf[EvalBinding] => val evaluation = d.getEvaluation.asInstanceOf[EvalBinding] @@ -324,9 +328,21 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode): if ASTStringUtil.getSimpleName( f.getName ).isEmpty && f.getNestedDeclarator != null => - s"${fullName(f.getParent)}.${shortName(f.getNestedDeclarator)}" + val parentFullName = fullName(f.getParent) + val sn = shortName(f.getNestedDeclarator) + val fnWithParent = + if parentFullName.nonEmpty then s"${parentFullName}.${sn}" else sn + if FileDefaults.isHeaderFile(filename) then + s"$filename:$lineNo:$lineNoEnd:${fnWithParent}" + else fnWithParent case f: IASTFunctionDeclarator => - s"${fullName(f.getParent)}.${ASTStringUtil.getSimpleName(f.getName)}" + val parentFullName = fullName(f.getParent) + val sn = ASTStringUtil.getSimpleName(f.getName) + val fnWithParent = + if parentFullName.nonEmpty then s"${parentFullName}.${sn}" else sn + if FileDefaults.isHeaderFile(filename) then + s"$filename:$lineNo:$lineNoEnd:${fnWithParent}" + else fnWithParent case f: IASTFunctionDefinition if f.getDeclarator != null => s"${fullName(f.getParent)}.${ASTStringUtil.getQualifiedName(f.getDeclarator.getName)}" case f: IASTFunctionDefinition => diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/MacroHandler.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/MacroHandler.scala index 25a0aac..4890484 100644 --- a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/MacroHandler.scala +++ b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/astcreation/MacroHandler.scala @@ -158,7 +158,7 @@ trait MacroHandler(implicit withSchemaValidation: ValidationMode): val filename = fileName(macroDef) val lineNo: Integer = line(macroDef).getOrElse(-1) val lineNoEnd: Integer = lineEnd(macroDef).getOrElse(-1) - s"$filename:$lineNo:$lineNoEnd:$name:${argAsts.size}" + if name != "NULL" then s"$filename:$lineNo:$lineNoEnd:$name:${argAsts.size}" else name /** The CDT utility method is unfortunately in a class that is marked as deprecated, however, * this is because the CDT team would like to discourage its use but at the same time does not diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/FileDefaults.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/FileDefaults.scala index c45ff05..fe63847 100644 --- a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/FileDefaults.scala +++ b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/FileDefaults.scala @@ -12,7 +12,8 @@ object FileDefaults: val SOURCE_FILE_EXTENSIONS: Set[String] = Set(C_EXT, CC_EXT, CPP_EXT) - val HEADER_FILE_EXTENSIONS: Set[String] = Set(C_HEADER_EXT, CPP_HEADER_EXT, OTHER_HEADER_EXT) + val HEADER_FILE_EXTENSIONS: Set[String] = + Set(C_HEADER_EXT, CPP_HEADER_EXT, OTHER_HEADER_EXT, ".h.in") private val CPP_FILE_EXTENSIONS = Set(CC_EXT, CPP_EXT, CPP_HEADER_EXT, ".ccm", ".cxxm", ".c++m") diff --git a/platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/ast/HeaderAstCreationPassTests.scala b/platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/ast/HeaderAstCreationPassTests.scala index 2a5cd92..119003f 100644 --- a/platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/ast/HeaderAstCreationPassTests.scala +++ b/platform/frontends/c2cpg/src/test/scala/io/appthreat/c2cpg/passes/ast/HeaderAstCreationPassTests.scala @@ -42,15 +42,16 @@ class HeaderAstCreationPassTests extends CCodeToCpgSuite { // in main.h and included in main.c and we do scan both bar.fullName shouldBe "bar" bar.filename shouldBe "main.h" - foo.fullName shouldBe "foo" - foo.filename shouldBe "other.h" + + foo.fullName shouldBe "main" + foo.filename shouldBe "main.c" // main is include twice. First time for the header file, // second time for the actual implementation in the source file // We do not de-duplicate this as line/column numbers differ - m1.fullName shouldBe "main" + m1.fullName shouldBe "main.h:2:2:main" m1.filename shouldBe "main.h" - m2.fullName shouldBe "main" - m2.filename shouldBe "main.c" + m2.fullName shouldBe "other.h:2:2:foo" + m2.filename shouldBe "other.h" printf.fullName shouldBe "printf" } } diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala index e78e82c..8bf3432 100644 --- a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala +++ b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala @@ -44,7 +44,8 @@ class CdxPass(atom: Cpg) extends CpgPass(atom): private def PY_REQUEST_PATTERNS = Array(".*views.py:.*") - private def containsRegex(str: String) = Pattern.quote(str) == str || str.contains("*") + private def containsRegex(str: String) = + Pattern.quote(str) == str || str.contains("*") || str.contains("(") || str.contains(")") private val BOM_JSON_FILE = ".*(bom|cdx).json" @@ -121,7 +122,54 @@ class CdxPass(atom: Cpg) extends CpgPass(atom): then bpkg = toPyModuleForm(bpkg) if bpkg.nonEmpty && !donePkgs.contains(bpkg) then donePkgs.put(bpkg, true) - if !containsRegex(bpkg) then + // C/C++ + if language == Languages.NEWC || language == Languages.C + then + atom.method.fullNameExact(bpkg).callIn( + NoResolve + ).newTagNode( + compPurl + ).store()(dstGraph) + atom.method.fullNameExact(bpkg).callIn( + NoResolve + ).newTagNode( + "library-call" + ).store()(dstGraph) + atom.method.fullNameExact(bpkg).newTagNode( + compPurl + ).store()(dstGraph) + if !containsRegex(bpkg) then + atom.parameter.typeFullName(s"$bpkg.*").newTagNode( + compPurl + ).store()(dstGraph) + atom.parameter.typeFullName(s"$bpkg.*").newTagNode( + "framework-input" + ).store()(dstGraph) + atom.parameter.typeFullName(s"$bpkg.*").method.callIn( + NoResolve + ).newTagNode( + compPurl + ).store()(dstGraph) + else + atom.parameter.typeFullName( + s"${Pattern.quote(bpkg)}.*" + ).newTagNode( + compPurl + ).store()(dstGraph) + atom.parameter.typeFullName( + s"${Pattern.quote(bpkg)}.*" + ).newTagNode( + "framework-input" + ).store()(dstGraph) + atom.parameter.typeFullName( + s"${Pattern.quote(bpkg)}.*" + ).method.callIn( + NoResolve + ).newTagNode( + compPurl + ).store()(dstGraph) + end if + else if !containsRegex(bpkg) then atom.call.typeFullNameExact(bpkg).newTagNode( compPurl ).store()(dstGraph) @@ -246,4 +294,5 @@ class CdxPass(atom: Cpg) extends CpgPass(atom): } } } + end run end CdxPass diff --git a/pyproject.toml b/pyproject.toml index 18a7751..9a43748 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appthreat-chen" -version = "1.0.2" +version = "1.0.3" description = "Code Hierarchy Exploration Net (chen)" authors = ["Team AppThreat "] license = "Apache-2.0"