From 445d90ad003d5061c820a91b3cb4c66c7eeb4d5d Mon Sep 17 00:00:00 2001 From: Khemraj Rathore Date: Mon, 23 Dec 2024 20:31:44 +0530 Subject: [PATCH] fix failing code --- .../main/scala/io/joern/kotlin2cpg/Main.scala | 4 + .../io/joern/rubysrc2cpg/RubySrc2Cpg.scala | 64 ++++------- .../rubysrc2cpg/parser/AnltrAstPrinter.scala | 36 +++++++ .../testfixtures/RubyCode2CpgFixture.scala | 3 +- .../testfixtures/RubyParserFixture.scala | 102 ------------------ 5 files changed, 60 insertions(+), 149 deletions(-) create mode 100644 joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AnltrAstPrinter.scala delete mode 100644 joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyParserFixture.scala diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala index 1463dbb9b810..3ef661e84692 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala @@ -56,6 +56,10 @@ final case class Config( def withKeepTypeArguments(value: Boolean): Config = { copy(keepTypeArguments = value).withInheritedFields(this) } + + def withResolveTypes(value: Boolean): Config = { + copy(resolveTypes = value).withInheritedFields(this) + } } private object Frontend { diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 15ca2a39919b..56e7047beeb6 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -6,7 +6,7 @@ import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.StatementList import io.joern.rubysrc2cpg.datastructures.RubyProgramSummary import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* -import io.joern.rubysrc2cpg.parser.* +import io.joern.rubysrc2cpg.parser.{RubyJsonParser, RubyJsonToNodeCreator, RubyAstGenRunner} import io.joern.rubysrc2cpg.passes.{ AstCreationPass, ConfigFileCreationPass, @@ -19,6 +19,7 @@ import io.joern.x2cpg.frontendspecific.rubysrc2cpg.* import io.joern.x2cpg.passes.base.AstLinkerPass import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.joern.x2cpg.passes.frontend.{MetaDataPass, TypeNodePass, XTypeRecoveryConfig} +import io.joern.x2cpg.utils.ExternalCommand.ExternalCommandResult import io.joern.x2cpg.utils.{ConcurrentTaskUtil, ExternalCommand} import io.joern.x2cpg.{SourceFiles, X2CpgFrontend} import io.shiftleft.codepropertygraph.generated.{Cpg, Languages} @@ -49,11 +50,13 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { } private def newCreateCpgAction(cpg: Cpg, config: Config): Unit = { - Using.resource( - new parser.ResourceManagedParser(config.antlrCacheMemLimit, config.antlrDebug, config.antlrProfiling) - ) { parser => + File.usingTemporaryDirectory("rubysrc2cpgOut") { tmpDir => + val astGenResult = RubyAstGenRunner(config).execute(tmpDir) + val astCreators = ConcurrentTaskUtil - .runUsingThreadPool(RubySrc2Cpg.generateParserTasks(parser, config, cpg.metaData.root.headOption)) + .runUsingThreadPool( + RubySrc2Cpg.processAstGenRunnerResults(astGenResult.parsedFiles, config, cpg.metaData.root.headOption) + ) .flatMap { case Failure(exception) => logger.warn(s"Unable to parse Ruby file, skipping -", exception); None case Success(astCreator) => Option(astCreator) @@ -135,18 +138,18 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { private def downloadDependency(inputPath: String, tempPath: String): Unit = { if (Files.isRegularFile(Paths.get(s"${inputPath}${java.io.File.separator}Gemfile"))) { - ExternalCommand.run(s"bundle config set --local path ${tempPath}", inputPath) match { - case Success(configOutput) => - logger.info(s"Gem config successfully done: $configOutput") - case Failure(exception) => - logger.error(s"Error while configuring Gem Path: ${exception.getMessage}") + ExternalCommand.run(Seq("bundle", "config", "set", "--local", "path", tempPath), inputPath) match { + case ExternalCommandResult(0, stdOut, _) => + logger.info(s"Gem config successfully done") + case ExternalCommandResult(_, stdOut, _) => + logger.error(s"Error while configuring Gem Path: ${stdOut.mkString(System.lineSeparator())}") } - val command = s"bundle install" + val command = Seq("bundle", "install") ExternalCommand.run(command, inputPath) match { - case Success(bundleOutput) => - logger.info(s"Dependency installed successfully: $bundleOutput") - case Failure(exception) => - logger.error(s"Error while downloading dependency: ${exception.getMessage}") + case ExternalCommandResult(0, stdOut, _) => + logger.info(s"Dependency installed successfully") + case ExternalCommandResult(_, stdOut, _) => + logger.error(s"Error while downloading dependency: ${stdOut.mkString(System.lineSeparator())}") } } } @@ -177,37 +180,6 @@ object RubySrc2Cpg { } } - def generateParserTasks( - resourceManagedParser: parser.ResourceManagedParser, - config: Config, - projectRoot: Option[String] - ): Iterator[() => AstCreator] = { - SourceFiles - .determine( - config.inputPath, - RubySourceFileExtensions, - ignoredDefaultRegex = Option(config.defaultIgnoredFilesRegex), - ignoredFilesRegex = Option(config.ignoredFilesRegex), - ignoredFilesPath = Option(config.ignoredFiles) - ) - .map { fileName => () => - resourceManagedParser.parse(File(config.inputPath), fileName) match { - case Failure(exception) => throw exception - case Success(ctx) => - val fileContent = (File(config.inputPath) / fileName).contentAsString - new AstCreator( - fileName, - ctx, - projectRoot, - enableFileContents = !config.disableFileContent, - fileContent = fileContent, - rootNode = Option(new RubyNodeCreator().visit(ctx).asInstanceOf[StatementList]) - )(config.schemaValidation) - } - } - .iterator - } - /** Parses the generated AST Gen files in parallel and produces AstCreators from each. */ def processAstGenRunnerResults( diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AnltrAstPrinter.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AnltrAstPrinter.scala new file mode 100644 index 000000000000..24776624a3f0 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AnltrAstPrinter.scala @@ -0,0 +1,36 @@ +package io.joern.rubysrc2cpg.parser + +import org.antlr.v4.runtime.ParserRuleContext +import org.antlr.v4.runtime.tree.TerminalNode + +/** General purpose ANTLR parse tree printer. + */ +object AnltrAstPrinter { + private val indentationIncrement = 1 + + private def print(level: Int, sb: StringBuilder, context: ParserRuleContext): StringBuilder = { + val indentation = " ".repeat(level) + val contextName = context.getClass.getSimpleName.stripSuffix("Context") + val nextLevel = level + indentationIncrement + sb.append(s"$indentation$contextName\n") + Option(context.children).foreach(_.forEach { + case c: ParserRuleContext => print(nextLevel, sb, c) + case t: TerminalNode => print(nextLevel, sb, t) + }) + sb + } + + private def print(level: Int, sb: StringBuilder, terminal: TerminalNode): StringBuilder = { + val indentation = " ".repeat(level) + sb.append(s"$indentation${terminal.getText}\n") + sb + } + + /** Pretty-prints an entire `ParserRuleContext` together with its descendants. + * @param context + * the context to pretty-print + * @return + * an indented, multiline string representation + */ + def print(context: ParserRuleContext): String = print(0, new StringBuilder, context).toString() +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala index 72f35c500deb..b430000c1268 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyCode2CpgFixture.scala @@ -1,5 +1,6 @@ package io.joern.rubysrc2cpg.testfixtures +import java.io.File import io.joern.dataflowengineoss.DefaultSemantics import io.joern.dataflowengineoss.language.Path import io.joern.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics} @@ -10,7 +11,7 @@ import io.joern.x2cpg.ValidationMode import io.joern.x2cpg.testfixtures.* import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve} -import org.scalatest.Inside +import org.scalatest.{Inside, Tag} import java.nio.file.Files import scala.jdk.CollectionConverters.* diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyParserFixture.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyParserFixture.scala deleted file mode 100644 index 4518f99f2453..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/testfixtures/RubyParserFixture.scala +++ /dev/null @@ -1,102 +0,0 @@ -package io.joern.rubysrc2cpg.testfixtures - -import io.joern.rubysrc2cpg.Config -import io.joern.rubysrc2cpg.parser.{AstPrinter, ResourceManagedParser, RubyNodeCreator, RubyParser} -import io.joern.x2cpg.SourceFiles -import io.joern.x2cpg.utils.{ConcurrentTaskUtil, TestCodeWriter} -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpecLike -import org.slf4j.LoggerFactory -import better.files.File as BFile - -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path} -import scala.util.{Failure, Success, Using} - -class RubyParserFixture - extends RubyFrontend( - useDeprecatedFrontend = false, - withDownloadDependencies = false, - disableFileContent = true, - antlrDebugging = false, - antlrProfiling = false - ) - with TestCodeWriter - with AnyWordSpecLike - with Matchers { - private val RubySourceFileExtensions: Set[String] = Set(".rb") - private val logger = LoggerFactory.getLogger(this.getClass) - private var fileNameCounter = 0 - - def generateParserTasks( - resourceManagedParser: ResourceManagedParser, - config: Config, - inputPath: String - ): Iterator[() => RubyParser.ProgramContext] = { - SourceFiles - .determine( - inputPath, - RubySourceFileExtensions, - ignoredDefaultRegex = Option(config.defaultIgnoredFilesRegex), - ignoredFilesRegex = Option(config.ignoredFilesRegex), - ignoredFilesPath = Option(config.ignoredFiles) - ) - .map(fileName => - () => - resourceManagedParser.parse(BFile(config.inputPath), fileName) match { - case Failure(exception) => throw exception - case Success(ctx) => ctx - } - ) - .iterator - } - - def writeCode(code: String, extension: String): Path = { - val tmpDir = BFile.newTemporaryDirectory("x2cpgTestTmpDir").deleteOnExit() - val tmpPath = tmpDir.path - val codeFiles = { - val fileName = { - val filename = s"Test$fileNameCounter$extension" - fileNameCounter += 1 - filename - } - - val filePath = Path.of(fileName) - if (filePath.getParent != null) { - Files.createDirectories(tmpPath.resolve(filePath.getParent)) - } - val codeAsBytes = code.getBytes(StandardCharsets.UTF_8) - val codeFile = tmpPath.resolve(filePath) - Files.write(codeFile, codeAsBytes) - codeFilePreProcessing(codeFile) - codeFile - } - - tmpPath - } - - def parseCode(code: String): List[RubyParser.ProgramContext] = { - val tempPath = writeCode(code, ".rb") - - Using.resource(new ResourceManagedParser(config.antlrCacheMemLimit)) { parser => - ConcurrentTaskUtil.runUsingThreadPool(generateParserTasks(parser, config, tempPath.toString)).flatMap { - case Failure(exception) => logger.warn(s"Could not parse file, skipping - ", exception); None - case Success(ctx) => Option(ctx) - } - } - } - - def test(code: String, expected: String = null): Unit = { - val astPrinter = parseCode(code).headOption match { - case Some(head) => Option(AstPrinter().visit(head)) - case None => None - } - - astPrinter match { - case Some(ast) => - val compareTo = if (expected != null) expected else code - ast shouldBe compareTo - case None => fail("AST Printer failed") - } - } -}