diff --git a/build.sbt b/build.sbt index b4989a0..0d22e83 100644 --- a/build.sbt +++ b/build.sbt @@ -1,9 +1,9 @@ name := "atom" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "1.5.0" +ThisBuild / version := "1.5.1" ThisBuild / scalaVersion := "3.3.1" -val chenVersion = "0.0.19" +val chenVersion = "0.0.20" lazy val atom = Projects.atom diff --git a/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala index bcb21a6..a07719d 100644 --- a/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala @@ -18,13 +18,13 @@ class DataFlowSlicing { private val nodeCache = new TrieMap[Long, SliceNode]() private var language: Option[String] = _ - def calculateDataFlowSlice(cpg: Cpg, config: DataFlowConfig): Option[DataFlowSlice] = { - language = cpg.metaData.language.headOption + def calculateDataFlowSlice(atom: Cpg, config: DataFlowConfig): Option[DataFlowSlice] = { + language = atom.metaData.language.headOption excludeOperatorCalls.set(config.excludeOperatorCalls) val dataFlowSlice = (config.fileFilter match { - case Some(fileRegex) => cpg.call.where(_.file.name(fileRegex)) - case None => cpg.call + case Some(fileRegex) => atom.call.where(_.file.name(fileRegex)) + case None => atom.call }) .where(c => c.callee.isExternal) .flatMap { diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala index 6bda489..7b070f1 100644 --- a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala @@ -5,6 +5,7 @@ import io.appthreat.dataflowengineoss.language.* import io.appthreat.dataflowengineoss.queryengine.{EngineConfig, EngineContext} import io.appthreat.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.Languages import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* @@ -17,17 +18,41 @@ object ReachableSlicing { val engineConfig = EngineConfig() implicit val context: EngineContext = EngineContext(semantics, engineConfig) private implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder - private val API_TAG = "api" + private def API_TAG = "api" + private def FRAMEWORK_TAG = "framework" def calculateReachableSlice(atom: Cpg, config: ReachablesConfig): ReachableSlice = { + val language = atom.metaData.language.head def source = atom.tag.name(config.sourceTag).parameter def sink = atom.ret.where(_.tag.name(config.sinkTag)) var flowsList = sink.reachableByFlows(source).map(toSlice).toList - // If we did not identify any flows from input to output, fallback to looking for - // flows between two apis - if (flowsList.isEmpty) { - flowsList = - atom.tag.name(API_TAG).parameter.reachableByFlows(atom.tag.name(API_TAG).parameter).map(toSlice).toList + flowsList ++= + atom.tag + .name(FRAMEWORK_TAG) + .method + .parameter + .reachableByFlows(atom.tag.name(config.sourceTag).parameter) + .map(toSlice) + .toList + flowsList ++= + atom.tag.name(API_TAG).parameter.reachableByFlows(atom.tag.name(API_TAG).parameter).map(toSlice).toList + // For JavaScript, we need flows between arguments of call nodes to track callbacks and middlewares + if (language == Languages.JSSRC || language == Languages.JAVASCRIPT) { + def jsCallSource = atom.tag.name(config.sourceTag).call.argument.isIdentifier + def jsFrameworkIdentifier = atom.tag.name(FRAMEWORK_TAG).identifier + def jsFrameworkParameter = atom.tag.name(FRAMEWORK_TAG).parameter + def jsSink = atom.tag.name(config.sinkTag).call.argument.isIdentifier + flowsList ++= jsSink + .reachableByFlows(jsCallSource, jsFrameworkIdentifier, jsFrameworkParameter) + .map(toSlice) + .toList + flowsList ++= atom.tag + .name(FRAMEWORK_TAG) + .call + .argument + .reachableByFlows(jsFrameworkParameter) + .map(toSlice) + .toList } ReachableSlice(flowsList) } @@ -39,7 +64,7 @@ object ReachableSlicing { private def toSlice(path: Path) = { val tableRows = ArrayBuffer[SliceNode]() val addedPaths = mutable.Set[String]() - var purls = mutable.Set[String]() + val purls = mutable.Set[String]() path.elements.foreach { astNode => val lineNumber = astNode.lineNumber.getOrElse("").toString val fileName = astNode.file.name.headOption.getOrElse("").replace("", "") @@ -58,6 +83,7 @@ object ReachableSlicing { ) astNode match { case _: MethodReturn => + case _: Block => case methodParameterIn: MethodParameterIn => val methodName = methodParameterIn.method.name if (tags.isEmpty && methodParameterIn.method.tag.nonEmpty) { @@ -104,7 +130,9 @@ object ReachableSlicing { if (!addedPaths.contains(s"${fileName}#${lineNumber}") && identifier.inCall.nonEmpty) { sliceNode = sliceNode.copy( name = identifier.name, - code = if (identifier.inCall.nonEmpty) identifier.inCall.head.code else identifier.code, + code = + if (identifier.inCall.nonEmpty) identifier.inCall.head.code + else identifier.code, parentMethodName = methodName, parentMethodSignature = identifier.method.signature, parentPackageName = identifier.method.location.packageName, @@ -122,9 +150,10 @@ object ReachableSlicing { case call: Call => if (!call.code.startsWith(" cpg.file.nameExact(fileName).method - case None => cpg.method + case Some(fileName) => atom.file.nameExact(fileName).method + case None => atom.method }).withMethodNameFilter.withMethodParameterFilter.withMethodAnnotationFilter.declaration - def typeMap = TrieMap.from(cpg.typeDecl.map(f => (f.name, f.fullName)).toMap) - val slices = usageSlices(cpg, () => getDeclarations, typeMap) - val language = cpg.metaData.language.headOption - val userDefTypes = userDefinedTypes(cpg) + def typeMap = TrieMap.from(atom.typeDecl.map(f => (f.name, f.fullName)).toMap) + val slices = usageSlices(atom, () => getDeclarations, typeMap) + val language = atom.metaData.language.headOption + val userDefTypes = userDefinedTypes(atom) if (language.get == Languages.NEWC || language.get == Languages.C) - ProgramUsageSlice(slices ++ importsAsSlices(cpg), userDefTypes) + ProgramUsageSlice(slices ++ importsAsSlices(atom), userDefTypes) else ProgramUsageSlice(slices, userDefTypes) } import io.shiftleft.semanticcpg.codedumper.CodeDumper.dump - private def usageSlices(cpg: Cpg, getDeclIdentifiers: () => Traversal[Declaration], typeMap: TrieMap[String, String])( - implicit config: UsagesConfig - ): List[MethodUsageSlice] = { - val language = cpg.metaData.language.headOption - val root = cpg.metaData.root.headOption + private def usageSlices( + atom: Cpg, + getDeclIdentifiers: () => Traversal[Declaration], + typeMap: TrieMap[String, String] + )(implicit config: UsagesConfig): List[MethodUsageSlice] = { + val language = atom.metaData.language.headOption + val root = atom.metaData.root.headOption getDeclIdentifiers() .to(LazyList) .filterNot(a => a.name.equals("*")) .filter(a => !a.name.startsWith("_tmp_") && atLeastNCalls(a, config.minNumCalls)) - .map(a => exec.submit(new TrackUsageTask(cpg, a, typeMap))) + .map(a => exec.submit(new TrackUsageTask(atom, a, typeMap))) .flatMap(TimedGet) .groupBy { case (scope, _) => scope } .view @@ -85,8 +87,8 @@ object UsageSlicing { .toList } - private def importsAsSlices(cpg: Cpg): List[MethodUsageSlice] = { - cpg.imports.l.map(im => { + private def importsAsSlices(atom: Cpg): List[MethodUsageSlice] = { + atom.imports.l.map(im => { MethodUsageSlice( code = if (im.code.nonEmpty) im.code.replaceAll("\\s*", "") else "", fullName = im.importedEntity.get, @@ -142,12 +144,12 @@ object UsageSlicing { /** Discovers internally defined types. * - * @param cpg + * @param atom * the CPG to query for types. * @return * a list of user defined types. */ - def userDefinedTypes(cpg: Cpg): List[UserDefinedType] = { + def userDefinedTypes(atom: Cpg): List[UserDefinedType] = { def generateUDT(typeDecl: TypeDecl): UserDefinedType = { UserDefinedType( @@ -173,14 +175,14 @@ object UsageSlicing { ) } - cpg.typeDecl + atom.typeDecl .filterNot(t => t.isExternal || t.name.matches("(:program||||||)")) .map(generateUDT) .filter(udt => udt.fields.nonEmpty || udt.procedures.nonEmpty) .l } - private class TrackUsageTask(cpg: Cpg, tgt: Declaration, typeMap: TrieMap[String, String])(implicit + private class TrackUsageTask(atom: Cpg, tgt: Declaration, typeMap: TrieMap[String, String])(implicit config: UsagesConfig ) extends Callable[Option[(Method, ObjectUsageSlice)]] { @@ -290,7 +292,7 @@ object UsageSlicing { * an API call if present. */ private def exprToObservedCall(baseCall: Call): Option[ObservedCall] = { - val language = cpg.metaData.language.headOption + val language = atom.metaData.language.headOption val isMemberInvocation = baseCall.name.equals(Operators.fieldAccess) val isConstructor = baseCall.name.equals(Operators.alloc) || baseCall.ast.isCall.nameExact(Operators.alloc).nonEmpty @@ -353,7 +355,7 @@ object UsageSlicing { baseCall.argumentOut .flatMap { case x: Call if !DefComponent.unresolvedCallPattern.matcher(x.methodFullName).matches() => - cpg.method.fullNameExact(x.methodFullName).methodReturn.typeFullName.headOption + atom.method.fullNameExact(x.methodFullName).methodReturn.typeFullName.headOption case x: Call => x.callee(resolver).methodReturn.typeFullName.headOption case _ => None diff --git a/wrapper/nodejs/package-lock.json b/wrapper/nodejs/package-lock.json index 975daf1..d4dde26 100644 --- a/wrapper/nodejs/package-lock.json +++ b/wrapper/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@appthreat/atom", - "version": "1.5.0", + "version": "1.5.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@appthreat/atom", - "version": "1.5.0", + "version": "1.5.1", "license": "Apache-2.0", "dependencies": { "@babel/parser": "^7.23.0", @@ -18,7 +18,7 @@ "atom": "index.js" }, "devDependencies": { - "eslint": "^8.51.0" + "eslint": "^8.52.0" }, "engines": { "node": ">=16.0.0" @@ -92,21 +92,21 @@ } }, "node_modules/@eslint/js": { - "version": "8.51.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.51.0.tgz", - "integrity": "sha512-HxjQ8Qn+4SI3/AFv6sOrDB+g6PpUTDwSJiQqOrnneEk8L71161srI9gjzzZvYVbzHiVg/BvcH95+cK/zfIt4pg==", + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.52.0.tgz", + "integrity": "sha512-mjZVbpaeMZludF2fsWLD0Z9gCref1Tk4i9+wddjRvpUNqqcndPkBD09N/Mapey0b3jaXbLm2kICwFv2E64QinA==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.11", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.11.tgz", - "integrity": "sha512-N2brEuAadi0CcdeMXUkhbZB84eskAc8MEX1By6qEchoVywSgXPIjou4rYsl0V3Hj0ZnuGycGCjdNgockbzeWNA==", + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", + "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", "dev": true, "dependencies": { - "@humanwhocodes/object-schema": "^1.2.1", + "@humanwhocodes/object-schema": "^2.0.1", "debug": "^4.1.1", "minimatch": "^3.0.5" }, @@ -128,9 +128,9 @@ } }, "node_modules/@humanwhocodes/object-schema": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", - "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", + "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", "dev": true }, "node_modules/@nodelib/fs.scandir": { @@ -168,6 +168,12 @@ "node": ">= 8" } }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true + }, "node_modules/acorn": { "version": "8.10.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", @@ -384,18 +390,19 @@ } }, "node_modules/eslint": { - "version": "8.51.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.51.0.tgz", - "integrity": "sha512-2WuxRZBrlwnXi+/vFSJyjMqrNjtJqiasMzehF0shoLaW7DzS3/9Yvrmq5JiT66+pNjiX4UBnLDiKHcWAr/OInA==", + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.52.0.tgz", + "integrity": "sha512-zh/JHnaixqHZsolRB/w9/02akBk9EPrOs9JwcTP2ek7yL5bVvXuRariiaAjjoJ5DvuwQ1WAE/HsMz+w17YgBCg==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.2", - "@eslint/js": "8.51.0", - "@humanwhocodes/config-array": "^0.11.11", + "@eslint/js": "8.52.0", + "@humanwhocodes/config-array": "^0.11.13", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", diff --git a/wrapper/nodejs/package.json b/wrapper/nodejs/package.json index 3799db0..a673b9f 100644 --- a/wrapper/nodejs/package.json +++ b/wrapper/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "@appthreat/atom", - "version": "1.5.0", + "version": "1.5.1", "description": "Create atom (⚛) representation for your application, packages and libraries", "exports": "./index.js", "type": "module", @@ -14,7 +14,7 @@ "yargs": "^17.7.2" }, "devDependencies": { - "eslint": "^8.51.0" + "eslint": "^8.52.0" }, "bin": { "atom": "./index.js",