diff --git a/build.sbt b/build.sbt index 077208e..4e0e8b8 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "chen" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "0.0.21" +ThisBuild / version := "0.5.0" ThisBuild / scalaVersion := "3.3.1" val cpgVersion = "1.4.22" diff --git a/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/Jimple2Cpg.scala b/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/Jimple2Cpg.scala index d008884..ef87c51 100644 --- a/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/Jimple2Cpg.scala +++ b/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/Jimple2Cpg.scala @@ -8,7 +8,6 @@ import io.appthreat.x2cpg.X2Cpg.withNewEmptyCpg import io.appthreat.x2cpg.X2CpgFrontend import io.appthreat.x2cpg.datastructures.Global import io.appthreat.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass} -import io.appthreat.x2cpg.passes.taggers.CdxPass import io.shiftleft.codepropertygraph.Cpg import org.slf4j.LoggerFactory import soot.options.Options @@ -120,7 +119,6 @@ class Jimple2Cpg extends X2CpgFrontend[Config] { .createAndApply() DeclarationRefPass(cpg).createAndApply() new ConfigFileCreationPass(cpg).createAndApply() - new CdxPass(cpg).createAndApply() } override def createCpg(config: Config): Try[Cpg] = diff --git a/platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/PythonAstVisitor.scala b/platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/PythonAstVisitor.scala index b2bef5c..32a19ae 100644 --- a/platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/PythonAstVisitor.scala +++ b/platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/PythonAstVisitor.scala @@ -2019,7 +2019,9 @@ object PythonAstVisitor { // There is a corresponding list in policies which needs to be updated if this one is updated and vice versa. val builtinFunctionsV3: Iterable[String] = Iterable( "abs", + "aiter", "all", + "anext", "any", "ascii", "bin", @@ -2051,6 +2053,7 @@ object PythonAstVisitor { "locals", "map", "max", + "memoryview", "min", "next", "oct", diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala index 38640c8..8e551a1 100644 --- a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala +++ b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala @@ -7,6 +7,7 @@ import io.shiftleft.codepropertygraph.generated.Languages import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* +import java.io.File import java.util.regex.Pattern import scala.collection.mutable import scala.io.Source @@ -45,6 +46,16 @@ class CdxPass(atom: Cpg) extends CpgPass(atom) { private val BOM_JSON_FILE = ".*(bom|cdx).json" + private def toPyModuleForm(str: String) = { + if (str.nonEmpty && str.count(_ == '.') > 0) { + s".*${str.split("\\.").take(2).mkString(Pattern.quote(File.separator))}.*" + } else if (str.nonEmpty) { + s"$str${Pattern.quote(File.separator)}.*" + } else { + str + } + } + override def run(dstGraph: DiffGraphBuilder): Unit = { atom.configFile.name(BOM_JSON_FILE).content.foreach { cdxData => val cdxJson = parse(cdxData).getOrElse(Json.Null) @@ -72,12 +83,19 @@ class CdxPass(atom: Cpg) extends CpgPass(atom) { .filterNot(_.contains("mock")) .filterNot(_.endsWith(".lock")) .filterNot(_.endsWith(".json")) + .filterNot(_.endsWith(".txt")) .foreach { (pkg: String) => var bpkg = pkg.takeWhile(_ != '$') - if (language == Languages.JAVA || language == Languages.JAVASRC) + if (language == Languages.JAVA || language == Languages.JAVASRC) { bpkg = bpkg.split("\\.").take(PKG_NS_SIZE).mkString(".").concat(".*") - if (language == Languages.JSSRC || language == Languages.JAVASCRIPT) bpkg = s".*${bpkg}.*" - if (!donePkgs.contains(bpkg)) { + bpkg = bpkg.replace(File.separator, Pattern.quote(File.separator)) + } + if (language == Languages.JSSRC || language == Languages.JAVASCRIPT) { + bpkg = s".*${bpkg}.*" + bpkg = bpkg.replace(File.separator, Pattern.quote(File.separator)) + } + if (language == Languages.PYTHON || language == Languages.PYTHONSRC) bpkg = toPyModuleForm(bpkg) + if (bpkg.nonEmpty && !donePkgs.contains(bpkg)) { donePkgs.put(bpkg, true) if (!containsRegex(bpkg)) { atom.call.typeFullNameExact(bpkg).newTagNode(compPurl).store()(dstGraph) diff --git a/pyproject.toml b/pyproject.toml index b162137..895d8ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appthreat-chen" -version = "0.0.21" +version = "0.5.0" description = "Code Hierarchy Exploration Net (chen)" authors = ["Team AppThreat "] license = "Apache-2.0"