Skip to content

Commit

Permalink
kotlin2cpg: add nodes for gradle dependencies (#3092)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ursachec
ursachec authored Jul 12, 2023
1 parent dd5d8d8 commit 18d33be
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 6 deletions.
34 changes: 31 additions & 3 deletions joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Kotlin2Cpg.scala
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package io.joern.kotlin2cpg

import better.files.File

import java.nio.file.{Files, Paths}
import org.jetbrains.kotlin.psi.KtFile
import org.slf4j.LoggerFactory

import scala.util.Try
import scala.jdk.CollectionConverters.{CollectionHasAsScala, EnumerationHasAsScala}

import io.joern.kotlin2cpg.files.SourceFilesPicker
import io.joern.kotlin2cpg.passes.{AstCreationPass, ConfigPass}
import io.joern.kotlin2cpg.passes.{AstCreationPass, ConfigPass, DependenciesFromMavenCoordinatesPass}
import io.joern.kotlin2cpg.compiler.{CompilerAPI, ErrorLoggingMessageCollector}
import io.joern.kotlin2cpg.types.{ContentSourcesPicker, DefaultTypeInfoProvider}
import io.joern.kotlin2cpg.utils.PathUtils
Expand All @@ -20,7 +21,7 @@ import io.joern.kotlin2cpg.interop.JavasrcInterop
import io.joern.kotlin2cpg.jar4import.UsesService
import io.shiftleft.codepropertygraph.Cpg
import io.shiftleft.codepropertygraph.generated.Languages
import io.shiftleft.semanticcpg.language._
import io.shiftleft.semanticcpg.language.*
import io.shiftleft.utils.IOUtils

object Kotlin2Cpg {
Expand Down Expand Up @@ -85,6 +86,11 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService {
Seq()
}

val mavenCoordinates = if (config.generateNodesForDependencies) {
logger.info(s"Fetching maven coordinates.")
fetchMavenCoordinates(sourceDir, config)
} else Seq()

val jarsAtConfigClassPath = findJarsIn(config.classpath)
if (config.classpath.nonEmpty) {
if (jarsAtConfigClassPath.nonEmpty) {
Expand Down Expand Up @@ -143,6 +149,9 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService {
val configCreator = new ConfigPass(configFiles, cpg)
configCreator.createAndApply()

val dependenciesFromMavenCoordinatesPass = new DependenciesFromMavenCoordinatesPass(mavenCoordinates, cpg)
dependenciesFromMavenCoordinatesPass.createAndApply()

val hasAtLeastOneMethodNode = cpg.method.take(1).nonEmpty
if (!hasAtLeastOneMethodNode) {
logger.warn("Resulting CPG does not contain any METHOD nodes.")
Expand Down Expand Up @@ -181,6 +190,25 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService {
}
}

private def fetchMavenCoordinates(sourceDir: String, config: Config): Seq[String] = {
val gradleParams = Map(
GradleConfigKeys.ProjectName -> config.gradleProjectName,
GradleConfigKeys.ConfigurationName -> config.gradleConfigurationName
).collect { case (key, Some(value)) => (key, value) }

val resolverParams = DependencyResolverParams(Map.empty, gradleParams)
DependencyResolver.getCoordinates(Paths.get(sourceDir), resolverParams) match {
case Some(coordinates) =>
logger.info(s"Found ${coordinates.size} maven coordinates.")
coordinates.toSeq
case None =>
logger.warn(s"Could not fetch coordinates for project at path $sourceDir")
println("Could not fetch coordinates when explicitly asked to. Exiting.")
System.exit(1)
Seq()
}
}

private def findJarsIn(dirs: Set[String]) = {
val jarExtension = ".jar"
dirs.foldLeft(Seq[String]())((acc, classpathEntry) => {
Expand Down
12 changes: 10 additions & 2 deletions joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ final case class Config(
gradleProjectName: Option[String] = None,
gradleConfigurationName: Option[String] = None,
jar4importServiceUrl: Option[String] = None,
includeJavaSourceFiles: Boolean = false
includeJavaSourceFiles: Boolean = false,
generateNodesForDependencies: Boolean = false
) extends X2CpgConfig[Config] {

def withClasspath(classpath: Set[String]): Config = {
Expand Down Expand Up @@ -43,6 +44,10 @@ final case class Config(
def withIncludeJavaSourceFiles(value: Boolean): Config = {
this.copy(includeJavaSourceFiles = value).withInheritedFields(this)
}

def withGenerateNodesForDependencies(value: Boolean): Config = {
this.copy(generateNodesForDependencies = value).withInheritedFields(this)
}
}

private object Frontend {
Expand Down Expand Up @@ -75,7 +80,10 @@ private object Frontend {
.action((value, c) => c.withGradleConfigurationName(value)),
opt[Unit]("include-java-sources")
.text("Include Java sources in the resulting CPG")
.action((_, c) => c.withIncludeJavaSourceFiles(true))
.action((_, c) => c.withIncludeJavaSourceFiles(true)),
opt[Unit]("generate-nodes-for-dependencies")
.text("Generate nodes for the dependencies of the target project")
.action((_, c) => c.withGenerateNodesForDependencies(true))
)
}
}
Expand Down
41 changes: 41 additions & 0 deletions ...2cpg/src/main/scala/io/joern/kotlin2cpg/passes/DependenciesFromMavenCoordinatesPass.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package io.joern.kotlin2cpg.passes

import io.shiftleft.codepropertygraph.Cpg
import io.shiftleft.passes.CpgPass
import io.shiftleft.semanticcpg.language._
import io.shiftleft.codepropertygraph.generated.nodes.{NewDependency}
import org.slf4j.{Logger, LoggerFactory}

import scala.util.matching.Regex

// This pass takes a list of strings representing maven coordinates in order to add DEPENDENCY nodes to the graph.
/*
example of a sequence of coordinates that are valid for the pass:
```
org.jetbrains.kotlin:kotlin-stdlib:1.7.22
org.jetbrains.kotlin:kotlin-stdlib-common:1.7.22
org.jetbrains:annotations:13.0
org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.7.22
org.apache.logging.log4j:log4j-api:2.15.0
org.springframework.boot:spring-boot-starter:3.0.5
org.springframework.boot:spring-boot:3.0.5
org.springframework:spring-core:6.0.7
org.springframework:spring-jcl:6.0.7
org.springframework:spring-context:6.0.7
```
*/
class DependenciesFromMavenCoordinatesPass(coordinates: Seq[String], cpg: Cpg) extends CpgPass(cpg) {
override def run(dstGraph: DiffGraphBuilder): Unit = {

coordinates.foreach { coordinate =>
val keyValPattern: Regex = "^([^:]+):([^:]+):([^:]+)$".r
for (patternMatch <- keyValPattern.findAllMatchIn(coordinate)) {
val groupId = patternMatch.group(1)
val name = patternMatch.group(2)
val version = patternMatch.group(3)
val node = NewDependency().name(name).version(version).dependencyGroupId(groupId)
dstGraph.addNode(node)
}
}
}
}
88 changes: 87 additions & 1 deletion ...i/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package io.joern.x2cpg.utils.dependency

import better.files.File
import io.joern.x2cpg.utils.ExternalCommand
import io.joern.x2cpg.utils.dependency.GradleConfigKeys.GradleConfigKey
import org.slf4j.LoggerFactory

import java.nio.file.Path
import scala.util.{Failure, Success}

object GradleConfigKeys extends Enumeration {
type GradleConfigKey = Value
Expand All @@ -18,9 +20,93 @@ case class DependencyResolverParams(
object DependencyResolver {
private val logger = LoggerFactory.getLogger(getClass)
private val defaultGradleProjectName = "app"
private val defaultGradleConfigurationName = "releaseCompileClasspath"
private val defaultGradleConfigurationName = "compileClasspath"
private val MaxSearchDepth: Int = 4

def getCoordinates(
projectDir: Path,
params: DependencyResolverParams = new DependencyResolverParams
): Option[collection.Seq[String]] = {
val coordinates = findSupportedBuildFiles(projectDir).flatMap { buildFile =>
if (isMavenBuildFile(buildFile))
// TODO: implement
None
else if (isGradleBuildFile(buildFile))
getCoordinatesForGradleProject(buildFile.getParent, defaultGradleConfigurationName)
else {
logger.warn(s"Found unsupported build file $buildFile")
Nil
}
}.flatten

Option.when(coordinates.nonEmpty)(coordinates)
}

private def getCoordinatesForGradleProject(
projectDir: Path,
configuration: String
): Option[collection.Seq[String]] = {
val lines = ExternalCommand.run(s"gradle dependencies --configuration $configuration", projectDir.toString) match {
case Success(lines) => lines
case Failure(exception) =>
logger.warn(
s"Could not retrieve dependencies for Gradle project at path `$projectDir`\n" +
exception.getMessage
)
Seq()
}

/*
on the following regex, for the following input:
```
| | +--- org.springframework.boot:spring-boot-starter-logging:3.0.5
| | | +--- ch.qos.logback:logback-classic:1.4.6
| | | | +--- ch.qos.logback:logback-core:1.4.6
| | | | \--- org.slf4j:slf4j-api:2.0.4 -> 2.0.7
| | | +--- org.apache.logging.log4j:log4j-to-slf4j:2.19.0
| | | | +--- org.slf4j:slf4j-api:1.7.36 -> 2.0.7
| | | | \--- org.apache.logging.log4j:log4j-api:2.19.0 -> 2.15.0
| | | \--- org.slf4j:jul-to-slf4j:2.0.7
```
the resulting matches and their groups are:
```
org.springframework.boot:spring-boot-starter-logging:3.0.5
^g1 ------------------------------------------------^^g2 ^
ch.qos.logback:logback-classic:1.4.6
^g1 --------------------------^^g2 ^
ch.qos.logback:logback-core:1.4.6
^g1 -----------------------^^g2 ^
org.slf4j:slf4j-api:2.0.4 -> 2.0.7
^g1 ---------------^^g2 ^^g3^^g4 ^
org.apache.logging.log4j:log4j-to-slf4j:2.19.0
^g1 -----------------------------------^^g2 -^
org.slf4j:slf4j-api:1.7.36 -> 2.0.7
^g1 ---------------^^g2 -^^g3^^g4 ^
org.apache.logging.log4j:log4j-api:2.19.0 -> 2.15.0
^g1 ------------------------------^^g2 -^^g3^^g4 -^
org.slf4j:jul-to-slf4j:2.0.7
^g1 ------------------^^g2 ^
```
*/
val pattern = """^[| ]?[+\\]\s*[-]*\s*([^:]+:[^:]+:)([^\s]+)(\s+->\s+)?([^\s]+)?""".r
val coordinates = lines
.flatMap { l =>
pattern.findFirstMatchIn(l) match {
case Some(m) =>
if (Option(m.group(4)).isEmpty)
Some(m.group(1) + m.group(2))
else
Some(m.group(1) + m.group(4))
case _ => None
}
}
.distinct
.toList

logger.info("Got {} Maven coordinates", coordinates.size)
Some(coordinates)
}

def getDependencies(
projectDir: Path,
params: DependencyResolverParams = new DependencyResolverParams
Expand Down

0 comments on commit 18d33be

Please sign in to comment.