From d53ee115b55f45684dcc9564b53e02b038107259 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 18:24:42 +0530 Subject: [PATCH 1/9] remove ImportResolverPass from CreateCpg --- .../io/joern/rubysrc2cpg/RubySrc2Cpg.scala | 62 ++++++++++--------- .../passes/RubyTypeRecoveryTests.scala | 3 +- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 608c385eef1f..81221859869d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -27,31 +27,50 @@ import scala.util.{Failure, Success, Try} class RubySrc2Cpg extends X2CpgFrontend[Config] { - private val logger = LoggerFactory.getLogger(this.getClass) - val global = new Global() override def createCpg(config: Config): Try[Cpg] = { withNewEmptyCpg(config.outputPath, config: Config) { (cpg, config) => - val packageTableInfo = new PackageTable() + new MetaDataPass(cpg, Languages.RUBYSRC, config.inputPath).createAndApply() new ConfigFileCreationPass(cpg).createAndApply() - if (config.enableDependencyDownload && !scala.util.Properties.isWin) { - val tempDir = File.newTemporaryDirectory() - try { - downloadDependency(config.inputPath, tempDir.toString()) - new AstPackagePass(cpg, tempDir.toString(), global, packageTableInfo, config.inputPath).createAndApply() - } finally { - tempDir.delete() - } - } - val astCreationPass = new AstCreationPass(config.inputPath, cpg, global, packageTableInfo) + val astCreationPass = new AstCreationPass(config.inputPath, cpg, global, new PackageTable()) astCreationPass.createAndApply() TypeNodePass.withRegisteredTypes(astCreationPass.allUsedTypes(), cpg).createAndApply() - new ImportResolverPass(cpg, packageTableInfo).createAndApply() } } +} + +object RubySrc2Cpg { + + val packageTableInfo = new PackageTable() + + private val logger = LoggerFactory.getLogger(this.getClass) + def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] = { + + val global = new Global() + if (config.isDefined && config.get.enableDependencyDownload && !scala.util.Properties.isWin) { + val tempDir = File.newTemporaryDirectory() + try { + downloadDependency(config.get.inputPath, tempDir.toString()) + new AstPackagePass(cpg, tempDir.toString(), global, packageTableInfo, config.get.inputPath).createAndApply() + } finally { + tempDir.delete() + } + } + + List( + // TODO commented below two passes, as waiting on Dependency download PR to get merged + new ImportResolverPass(cpg, packageTableInfo), + new RubyTypeRecoveryPass(cpg), + new RubyTypeHintCallLinker(cpg), + new NaiveCallLinker(cpg), + // Some of passes above create new methods, so, we + // need to run the ASTLinkerPass one more time + new AstLinkerPass(cpg) + ) + } private def downloadDependency(inputPath: String, tempPath: String): Unit = { if (Files.isRegularFile(Paths.get(s"${inputPath}${java.io.File.separator}Gemfile"))) { ExternalCommand.run(s"bundle config set --local path ${tempPath}", inputPath) match { @@ -69,20 +88,5 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { } } } -} - -object RubySrc2Cpg { - - def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] = - List( - // TODO commented below two passes, as waiting on Dependency download PR to get merged - new RubyTypeRecoveryPass(cpg), - new RubyTypeHintCallLinker(cpg), - new NaiveCallLinker(cpg), - - // Some of passes above create new methods, so, we - // need to run the ASTLinkerPass one more time - new AstLinkerPass(cpg) - ) } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala index 59644525be71..f28e68813af8 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala @@ -1,6 +1,6 @@ package io.joern.rubysrc2cpg.passes -import io.joern.rubysrc2cpg.Config +import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.shiftleft.semanticcpg.language.* @@ -10,7 +10,6 @@ class RubyTypeRecoveryTests extends RubyCode2CpgFixture { "Type information for nodes with external dependency" should { "be present in (Case 1)" ignore { - val cpg = code( """ |require "sendgrid-ruby" From a0627cc2ca5f4cba658d92950c479e85bc88f148 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 18:27:26 +0530 Subject: [PATCH 2/9] refactor --- .../src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 81221859869d..5449abdf0510 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -48,12 +48,12 @@ object RubySrc2Cpg { private val logger = LoggerFactory.getLogger(this.getClass) def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] = { - val global = new Global() if (config.isDefined && config.get.enableDependencyDownload && !scala.util.Properties.isWin) { val tempDir = File.newTemporaryDirectory() try { downloadDependency(config.get.inputPath, tempDir.toString()) - new AstPackagePass(cpg, tempDir.toString(), global, packageTableInfo, config.get.inputPath).createAndApply() + new AstPackagePass(cpg, tempDir.toString(), new Global(), packageTableInfo, config.get.inputPath) + .createAndApply() } finally { tempDir.delete() } From 952dfc1ba267fa67c48ff49e4ef2230b25891b09 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 18:28:36 +0530 Subject: [PATCH 3/9] refactor --- .../io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala index f28e68813af8..a7e3cb03a388 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/passes/RubyTypeRecoveryTests.scala @@ -1,6 +1,6 @@ package io.joern.rubysrc2cpg.passes -import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} +import io.joern.rubysrc2cpg.Config import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.shiftleft.semanticcpg.language.* From 2dcf81fefdd788ace8401755d8f7fe6ebd748edf Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 18:59:28 +0530 Subject: [PATCH 4/9] add - make provision for packageTable in Test --- .../joern/rubysrc2cpg/utils/PackageTable.scala | 5 +++++ .../testfixtures/RubyCode2CpgFixture.scala | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/PackageTable.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/PackageTable.scala index fdb3a7166a0e..6454337abcf9 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/PackageTable.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/utils/PackageTable.scala @@ -29,4 +29,9 @@ class PackageTable() { case Some(value) => value.toList case None => List[MethodTableModel]() } + + def set(table: PackageTable): Unit = { + methodTableMap.addAll(table.methodTableMap) + } + def clear(): Unit = methodTableMap.clear } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala index 360cc14a41c6..1ec95d5a36c9 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala @@ -2,6 +2,7 @@ package io.joern.rubysrc2cpg.testfixtures import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.dataflowengineoss.queryengine.EngineContext +import io.joern.rubysrc2cpg.utils.PackageTable import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend, TestCpg} @@ -24,13 +25,19 @@ trait RubyFrontend extends LanguageFrontend { } -class DefaultTestCpgWithRuby(withPostProcessing: Boolean, withDataFlow: Boolean) +class DefaultTestCpgWithRuby(withPostProcessing: Boolean, withDataFlow: Boolean, packageTable: Option[PackageTable]) extends DefaultTestCpg with RubyFrontend { override def applyPasses(): Unit = { X2Cpg.applyDefaultOverlays(this) if (withPostProcessing) { + packageTable match { + case Some(table) => + RubySrc2Cpg.packageTableInfo.clear() + RubySrc2Cpg.packageTableInfo.set(table) + case None => + } RubySrc2Cpg.postProcessingPasses(this).foreach(_.createAndApply()) } @@ -43,8 +50,11 @@ class DefaultTestCpgWithRuby(withPostProcessing: Boolean, withDataFlow: Boolean) } -class RubyCode2CpgFixture(withPostProcessing: Boolean = false, withDataFlow: Boolean = false) - extends Code2CpgFixture(() => new DefaultTestCpgWithRuby(withPostProcessing, withDataFlow)) { +class RubyCode2CpgFixture( + withPostProcessing: Boolean = false, + withDataFlow: Boolean = false, + packageTable: Option[PackageTable] = None +) extends Code2CpgFixture(() => new DefaultTestCpgWithRuby(withPostProcessing, withDataFlow, packageTable)) { implicit val resolver: ICallResolver = NoResolve implicit lazy val engineContext: EngineContext = EngineContext() From 7d51b836a33874f57b92027c5219f4540e5831bd Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 20:45:16 +0530 Subject: [PATCH 5/9] add --- .../src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 5449abdf0510..4024505370ac 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -34,7 +34,7 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { new MetaDataPass(cpg, Languages.RUBYSRC, config.inputPath).createAndApply() new ConfigFileCreationPass(cpg).createAndApply() - val astCreationPass = new AstCreationPass(config.inputPath, cpg, global, new PackageTable()) + val astCreationPass = new AstCreationPass(config.inputPath, cpg, global, RubySrc2Cpg.packageTableInfo) astCreationPass.createAndApply() TypeNodePass.withRegisteredTypes(astCreationPass.allUsedTypes(), cpg).createAndApply() } From 5418ea627c0179e4b950d69b1e779286828216af Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 20:48:38 +0530 Subject: [PATCH 6/9] PR comments --- .../io/joern/rubysrc2cpg/RubySrc2Cpg.scala | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 4024505370ac..2f42fedd97f8 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -1,6 +1,7 @@ package io.joern.rubysrc2cpg import better.files.File +import io.joern.rubysrc2cpg.RubySrc2Cpg.logger import io.joern.rubysrc2cpg.passes.{ AstCreationPass, AstPackagePass, @@ -34,11 +35,40 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { new MetaDataPass(cpg, Languages.RUBYSRC, config.inputPath).createAndApply() new ConfigFileCreationPass(cpg).createAndApply() + if (config.enableDependencyDownload && !scala.util.Properties.isWin) { + val tempDir = File.newTemporaryDirectory() + try { + downloadDependency(config.inputPath, tempDir.toString()) + new AstPackagePass(cpg, tempDir.toString(), new Global(), RubySrc2Cpg.packageTableInfo, config.inputPath) + .createAndApply() + } finally { + tempDir.delete() + } + } + val astCreationPass = new AstCreationPass(config.inputPath, cpg, global, RubySrc2Cpg.packageTableInfo) astCreationPass.createAndApply() TypeNodePass.withRegisteredTypes(astCreationPass.allUsedTypes(), cpg).createAndApply() } } + + private def downloadDependency(inputPath: String, tempPath: String): Unit = { + if (Files.isRegularFile(Paths.get(s"${inputPath}${java.io.File.separator}Gemfile"))) { + ExternalCommand.run(s"bundle config set --local path ${tempPath}", inputPath) match { + case Success(configOutput) => + logger.info(s"Gem config successfully done: $configOutput") + case Failure(exception) => + logger.error(s"Error while configuring Gem Path: ${exception.getMessage}") + } + val command = s"bundle install" + ExternalCommand.run(command, inputPath) match { + case Success(bundleOutput) => + logger.info(s"Dependency installed successfully: $bundleOutput") + case Failure(exception) => + logger.error(s"Error while downloading dependency: ${exception.getMessage}") + } + } + } } object RubySrc2Cpg { @@ -48,17 +78,6 @@ object RubySrc2Cpg { private val logger = LoggerFactory.getLogger(this.getClass) def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] = { - if (config.isDefined && config.get.enableDependencyDownload && !scala.util.Properties.isWin) { - val tempDir = File.newTemporaryDirectory() - try { - downloadDependency(config.get.inputPath, tempDir.toString()) - new AstPackagePass(cpg, tempDir.toString(), new Global(), packageTableInfo, config.get.inputPath) - .createAndApply() - } finally { - tempDir.delete() - } - } - List( // TODO commented below two passes, as waiting on Dependency download PR to get merged new ImportResolverPass(cpg, packageTableInfo), @@ -71,22 +90,5 @@ object RubySrc2Cpg { new AstLinkerPass(cpg) ) } - private def downloadDependency(inputPath: String, tempPath: String): Unit = { - if (Files.isRegularFile(Paths.get(s"${inputPath}${java.io.File.separator}Gemfile"))) { - ExternalCommand.run(s"bundle config set --local path ${tempPath}", inputPath) match { - case Success(configOutput) => - logger.info(s"Gem config successfully done: $configOutput") - case Failure(exception) => - logger.error(s"Error while configuring Gem Path: ${exception.getMessage}") - } - val command = s"bundle install" - ExternalCommand.run(command, inputPath) match { - case Success(bundleOutput) => - logger.info(s"Dependency installed successfully: $bundleOutput") - case Failure(exception) => - logger.error(s"Error while downloading dependency: ${exception.getMessage}") - } - } - } } From 63088b1aa7d268629d910b9a566cbefa0704be1c Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 20:49:54 +0530 Subject: [PATCH 7/9] PR comments --- .../src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 2f42fedd97f8..53ccb8c49822 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -75,9 +75,7 @@ object RubySrc2Cpg { val packageTableInfo = new PackageTable() - private val logger = LoggerFactory.getLogger(this.getClass) - def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] = { - + def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] = List( // TODO commented below two passes, as waiting on Dependency download PR to get merged new ImportResolverPass(cpg, packageTableInfo), @@ -89,6 +87,5 @@ object RubySrc2Cpg { // need to run the ASTLinkerPass one more time new AstLinkerPass(cpg) ) - } } From 4b6f5e7c74eb2d4b1a2761abe130dc9ee5a66991 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 20:51:33 +0530 Subject: [PATCH 8/9] PR comments --- .../src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 53ccb8c49822..1dc80a65c5d0 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -28,7 +28,8 @@ import scala.util.{Failure, Success, Try} class RubySrc2Cpg extends X2CpgFrontend[Config] { - val global = new Global() + val global = new Global() + private val logger = LoggerFactory.getLogger(this.getClass) override def createCpg(config: Config): Try[Cpg] = { withNewEmptyCpg(config.outputPath, config: Config) { (cpg, config) => @@ -39,7 +40,7 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { val tempDir = File.newTemporaryDirectory() try { downloadDependency(config.inputPath, tempDir.toString()) - new AstPackagePass(cpg, tempDir.toString(), new Global(), RubySrc2Cpg.packageTableInfo, config.inputPath) + new AstPackagePass(cpg, tempDir.toString(), global, RubySrc2Cpg.packageTableInfo, config.inputPath) .createAndApply() } finally { tempDir.delete() From f8c8aef8c50e884535509eaebb77b814142a01c7 Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Wed, 12 Jul 2023 20:52:06 +0530 Subject: [PATCH 9/9] PR comments --- .../src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 1dc80a65c5d0..2646dc1af900 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -1,7 +1,6 @@ package io.joern.rubysrc2cpg import better.files.File -import io.joern.rubysrc2cpg.RubySrc2Cpg.logger import io.joern.rubysrc2cpg.passes.{ AstCreationPass, AstPackagePass,