Skip to content

Commit

Permalink
[rubysrc2cpg] Decouple parser and ASTCreator (#3283)
Browse files Browse the repository at this point in the history
* Fix bug in command line parsing

* Decouple parser and ASTCreator

* Also log error message

* Add missing import

---------

Co-authored-by: Fabian Yamaguchi <fabs@joern.io>
  • Loading branch information
fabsx00 and Fabian Yamaguchi authored Jul 28, 2023
1 parent 2894946 commit cacae03
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ private object Frontend {
programName("rubysrc2cpg"),
opt[Unit]("enableDependencyDownload")
.hidden()
.action((_, c) => c.withEnableDependencyDownload(false))
.action((_, c) => c.withEnableDependencyDownload(true))
.text("enable dependency download for Unix System only")
)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package io.joern.rubysrc2cpg.astcreation

import io.joern.rubysrc2cpg.parser.{RubyLexer, RubyParser}
import org.antlr.v4.runtime.{CharStreams, CommonTokenStream, ParserRuleContext, Token}

class AntlrParser {

def parse(filename: String): RubyParser.ProgramContext = {
val charStream = CharStreams.fromFileName(filename)
val lexer = new RubyLexer(charStream)
val tokenStream = new CommonTokenStream(lexer)
val parser = new RubyParser(tokenStream)
parser.program()
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import io.joern.x2cpg.datastructures.{Global, Scope}
import io.joern.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, Defines as XDefines}
import io.shiftleft.codepropertygraph.generated.*
import io.shiftleft.codepropertygraph.generated.nodes.*
import org.antlr.v4.runtime.{ParserRuleContext, Token}
import org.antlr.v4.runtime.tree.TerminalNode
import org.antlr.v4.runtime.{CharStreams, CommonTokenStream, ParserRuleContext, Token}
import org.slf4j.LoggerFactory
import overflowdb.{BatchedUpdate, NodeOrDetachedNode}

Expand All @@ -19,6 +19,7 @@ import scala.collection.immutable.Seq
import scala.collection.mutable
import scala.collection.mutable.ListBuffer
import scala.jdk.CollectionConverters.*
import scala.util.{Failure, Success, Try}

class AstCreator(
protected val filename: String,
Expand Down Expand Up @@ -105,12 +106,19 @@ class AstCreator(
methodAliases.getOrElse(name, name)
}
override def createAst(): BatchedUpdate.DiffGraphBuilder = {
val charStream = CharStreams.fromFileName(filename)
val lexer = new RubyLexer(charStream)
val tokenStream = new CommonTokenStream(lexer)
val parser = new RubyParser(tokenStream)
val programCtx = parser.program()
Try {
new AntlrParser().parse(filename)
} match {
case Success(programCtx) =>
createAstForProgramCtx(programCtx)
case Failure(exc) =>
logger.warn(s"Could not parse file: $filename, skipping")
logger.warn(exc.getMessage)
diffGraph
}
}

private def createAstForProgramCtx(programCtx: RubyParser.ProgramContext) = {
val name = ":program"
val fullName = s"$relativeFilename:$name"
val programMethod =
Expand Down

0 comments on commit cacae03

Please sign in to comment.