From fc7d004706fb4847bf8f59a48749c7287b601836 Mon Sep 17 00:00:00 2001 From: Andrei Dreyer Date: Fri, 23 Aug 2024 10:38:23 +0200 Subject: [PATCH] [ruby] Added parser tests from official parser testS (#4872) --- .../joern/rubysrc2cpg/parser/AstPrinter.scala | 12 ++- .../rubysrc2cpg/parser/ArrayParserTests.scala | 8 ++ .../parser/AssignmentParserTests.scala | 55 +++++++++++ .../parser/BeginStatementParserTests.scala | 14 ++- .../parser/BitwiseOperatorParserTests.scala | 15 +++ .../rubysrc2cpg/parser/BlockParserTests.scala | 33 +++++++ .../parser/BooleanParserTests.scala | 7 ++ .../parser/CaseConditionParserTests.scala | 9 ++ .../parser/ClassDefinitionParserTests.scala | 36 +++++++ .../parser/ControlStructureParserTests.scala | 27 ++++++ .../parser/DoBlockParserTests.scala | 95 ++++++++++++++++++- .../parser/FieldAccessParserTests.scala | 19 ++++ .../parser/HashLiteralParserTests.scala | 6 ++ .../parser/IndexAccessParserTests.scala | 2 + ...InvocationWithParenthesisParserTests.scala | 30 ++++++ ...ocationWithoutParenthesesParserTests.scala | 2 +- .../parser/MathOperatorParserTests.scala | 15 +++ .../parser/MethodDefinitionParserTests.scala | 73 +++++++++++++- .../parser/ProcDefinitionParserTests.scala | 25 +++++ .../rubysrc2cpg/parser/RangeParserTests.scala | 18 ++++ .../rubysrc2cpg/parser/RegexParserTests.scala | 2 + .../parser/RescueClauseParserTests.scala | 2 + .../parser/ReturnParserTests.scala | 7 ++ .../parser/StringParserTests.scala | 6 ++ .../rubysrc2cpg/parser/YieldParserTests.scala | 16 ++++ 25 files changed, 525 insertions(+), 9 deletions(-) create mode 100644 joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BitwiseOperatorParserTests.scala create mode 100644 joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BlockParserTests.scala create mode 100644 joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MathOperatorParserTests.scala create mode 100644 joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/YieldParserTests.scala diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala index 295ff7dabefe..5011c2a7c03d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala @@ -69,7 +69,7 @@ class AstPrinter extends RubyParserBaseVisitor[String] { val condition = visit(ctx.expressionOrCommand()) val body = visit(ctx.doClause()) - s"${ctx.UNTIL.getText} $condition $body$ls${ctx.END.getText}" + s"${ctx.UNTIL.getText} $condition$ls$body$ls${ctx.END.getText}" } override def visitBeginEndExpression(ctx: RubyParser.BeginEndExpressionContext): String = { @@ -1104,6 +1104,7 @@ class AstPrinter extends RubyParserBaseVisitor[String] { } override def visitRescueClause(ctx: RubyParser.RescueClauseContext): String = { + val outputSb = new StringBuilder(ctx.RESCUE().getText) val exceptionClassList = Option(ctx.exceptionClassList).map(visit).getOrElse("") val variables = Option(ctx.exceptionVariableAssignment).map(visit).getOrElse("") val thenClause = visit(ctx.thenClause) @@ -1112,7 +1113,14 @@ class AstPrinter extends RubyParserBaseVisitor[String] { if Option(ctx.thenClause().THEN()).isDefined then s" ${ctx.thenClause().THEN().getText}" else "" - s"${ctx.RESCUE().getText} $exceptionClassList => $variables $thenKeyword $thenClause".strip() + if exceptionClassList != "" then outputSb.append(s" $exceptionClassList") + if variables != "" then outputSb.append(s" => $variables") + + outputSb.append(thenKeyword) + + if thenClause != "" then outputSb.append(s"\n${thenClause}") + + outputSb.toString() } override def visitEnsureClause(ctx: RubyParser.EnsureClauseContext): String = { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ArrayParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ArrayParserTests.scala index e7504c3421f1..5068bce911a9 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ArrayParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ArrayParserTests.scala @@ -5,6 +5,10 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class ArrayParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("[1, 2 => 3]", "[1,2=> 3]") // syntax error + } + "array structures" in { test("[]") test("%w[]") @@ -48,4 +52,8 @@ class ArrayParserTests extends RubyParserFixture with Matchers { test("%I{}") test("%I(x#{0} x1)") } + + "array params" in { + test("[1 => 2]", "[1=> 2]") + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala index 6476b49e3778..a4d96dd73ff6 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala @@ -4,14 +4,47 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class AssignmentParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("A.B ||= c 1") // Possible string issue - result is missing A.B on LHS + test("a[:b] ||= c 1, 2") // Possible string issue - result is missing a[:b] on LHS + test("A::b += 1") // Possible string issue - result is missing + in += operator + test("A::B *= c d") // Possible string issue - result is missing A::B *= on LHS + test("A::b *= c d") // Possible string issue - result is missing A::B *= on LHS + test("a.b ||= c 1") // Possible string issue - result is missing a.b ||= on LHS + test("a = b, *c, d") // Syntax error + test("*, a = b") // Syntax error + test("*, x, y, z = f") // Syntax error + } + "Single assignment" in { test("x=1", "x = 1") test("hash[:sym] = s[:sym]") test("a = 1, 2, 3, 4") + test("a = b.c 1") + test("a ||= b") + test("a &&= b") + test("a += 1") + test("a /= 1") + test("a[[1, 2]] = 3", "a[[1,2]] = 3") + test("a[] += b") + test("@a = 42") + test("a&.b = 1", "a&.b= 1") + test("c = a&.b") } "Multiple assignment" in { test("p, q = [foo(), bar()]") + test("a, b::c = d") + test("a, b.C = d") + test("::A, ::B = 1, 2") + test("[1,2,3,4][from..to] = [\"a\",\"b\",\"c\"]") + test("a, = b.c 1") + test("(a, b) = c.d") + test("a ||= b.c 2") + test("a, b, c, * = f") + test("a, b, c, *s = f") + test("*s, x, y, z = f") + test("a = b 1 rescue 2") } "Destructured Assignment" in { @@ -23,4 +56,26 @@ class AssignmentParserTests extends RubyParserFixture with Matchers { test("a, b, c = 1, 2, *list") test("a, b, c = 1, *list") } + + "Class Constant Assign" in { + test("A::b = 1") + test("a.B = 1") + } + + "Assignment with block" in { + test( + """h[k]=begin + |42 + |end + |""".stripMargin, + """h[k] = begin + |42 + |end""".stripMargin + ) + } + + "Assignment with rescue" in { + test("a = 1 rescue 2") + test("a = b(1) rescue 2") + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginStatementParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginStatementParserTests.scala index fc28e58163d2..4cbc4bb4b626 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginStatementParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BeginStatementParserTests.scala @@ -4,9 +4,15 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class BeginStatementParserTests extends RubyParserFixture with Matchers { - "BEGIN statement" in { - // TODO: Fix - valid for Ruby 2, but not 3 -// test("BEGIN { 1 }") -// test("BEGIN {}") + // TODO: Syntax Errors + "BEGIN statement" ignore { + test("BEGIN { 1 }") + test("BEGIN {}") + } + + // TODO: Syntax errors + "END statement" ignore { + test("END { 1 }") + test("END {}") } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BitwiseOperatorParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BitwiseOperatorParserTests.scala new file mode 100644 index 000000000000..c1cf500c720e --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BitwiseOperatorParserTests.scala @@ -0,0 +1,15 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class BitwiseOperatorParserTests extends RubyParserFixture with Matchers { + "Bitwise operators" in { + test("1 & 3") + test("2 & 4 & 3") + test("1 | 9") + test("1 ^ 20") + test("1 >> 2") + test("1 << 2") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BlockParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BlockParserTests.scala new file mode 100644 index 000000000000..eb81eaa3596d --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BlockParserTests.scala @@ -0,0 +1,33 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class BlockParserTests extends RubyParserFixture with Matchers { + "Blocks" in { + test("""a = 42 + |a""".stripMargin) + + test( + """a + |b # comment + |c + |""".stripMargin, + """a + |b + |c""".stripMargin + ) + + test( + """a + |b # comment + |# another comment + |c + |""".stripMargin, + """a + |b + |c""".stripMargin + ) + } + +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BooleanParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BooleanParserTests.scala index bf9bc666cd2f..74fe984dc429 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BooleanParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/BooleanParserTests.scala @@ -22,5 +22,12 @@ class BooleanParserTests extends RubyParserFixture with Matchers { test("1 && !2") test("1 || 2 || 3") test("1 && 2 && 3") + test("1 != 2") + test("1 == [:b, :c]", "1 == [:b,:c]") + test("! foo 1", "!foo 1") + } + + "Spaceship" in { + test("a <=> b") } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/CaseConditionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/CaseConditionParserTests.scala index 3214141017ea..e78a0c07658e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/CaseConditionParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/CaseConditionParserTests.scala @@ -4,6 +4,14 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class CaseConditionParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + // Splat arg missing from output + test("""case a + |when *b then + |end + |""".stripMargin) + } + "A case expression" in { test( """case something @@ -52,5 +60,6 @@ class CaseConditionParserTests extends RubyParserFixture with Matchers { |3 |end""".stripMargin ) + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ClassDefinitionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ClassDefinitionParserTests.scala index 9c9b501bd1ee..572a5d59dae6 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ClassDefinitionParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ClassDefinitionParserTests.scala @@ -29,5 +29,41 @@ class ClassDefinitionParserTests extends RubyParserFixture with Matchers { |end |end""".stripMargin ) + test( + """class Foo + | def self.show + | end + |end + |""".stripMargin, + """class Foo + |def + | + |end + |def self.show + | end + |end""".stripMargin + ) + } + + "class definitions with comments" in { + test( + """#blah 1 + |#blah 2 + |class X + |#blah 3 + |def blah + |#blah4 + |end + |end + |""".stripMargin, + """class X + |def + | + |end + |def blah + |#blah4 + |end + |end""".stripMargin + ) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ControlStructureParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ControlStructureParserTests.scala index 9b881159bed3..1e32c1a34116 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ControlStructureParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ControlStructureParserTests.scala @@ -14,6 +14,12 @@ class ControlStructureParserTests extends RubyParserFixture with Matchers { ) } + "until" in { + test("""until not var.nil? + |'foo' + |end""".stripMargin) + } + "if" in { test( """if __LINE__ > 1 then @@ -55,6 +61,27 @@ class ControlStructureParserTests extends RubyParserFixture with Matchers { |456 |end""".stripMargin ) + + test( + "if a..b then end", + """if a..b + |end""".stripMargin + ) + + test( + "if :x; end", + """if :x + |end""".stripMargin + ) + + test( + "if not var.nil? then 'foo' else 'bar'\nend", + """if not var.nil? + |'foo' + |else + |'bar' + |end""".stripMargin + ) } "for loops" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/DoBlockParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/DoBlockParserTests.scala index f5f9aaa06618..168504a40a3e 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/DoBlockParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/DoBlockParserTests.scala @@ -4,7 +4,20 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class DoBlockParserTests extends RubyParserFixture with Matchers { - "test" in {} + "fixme" ignore { + test("f { |(*a)| }") // syntax error + test("f { |a, (b, c), d| }") // syntax error + test("a { |b, c=1, *d, e, &f| }") // syntax error + test("a { |b, *c, d| }") // syntax error + test( + "break foo arg do |bar| end" + ) // syntax error - possibly false syntax error due to just having a code sample starting with break which our parser doesn't allow + test("f { |(*, a)| }") // syntax error + test("f { |(a, *b, c)| }") // syntax error + test("yield foo arg do |bar| end") // syntax error + test("f { |a, (b, *, c)| }") // syntax error + test("a.b do | ; c | end") // syntax error + } "Some block" in { test( @@ -37,5 +50,85 @@ class DoBlockParserTests extends RubyParserFixture with Matchers { """test_name 'Foo' do |end""".stripMargin ) + test( + "f{ |a, b| }", + """f { + |{|a,b|} + |}""".stripMargin + ) + test( + """f do |x, y| + |x + y + |end + |""".stripMargin, + """f do |x,y| + |x + y + |end""".stripMargin + ) + + test("""f(a) do |x,y| + |x + y + |end""".stripMargin) + + test( + """a.b do | | end""".stripMargin, + """a.b do + |end""".stripMargin + ) + } + + "Block arguments" in { + test( + "f { |a, b| }", + """f { + |{|a,b|} + |}""".stripMargin + ) + + test( + "a { |b, c=1, d, &e| }", + """a { + |{|b,c=1,d,&e|} + |}""".stripMargin + ) + + test( + "a { |b, c=1, *d| }", + """a { + |{|b,c=1,*d|} + |}""".stripMargin + ) + + test( + "f { |a, b = 42| [a, b] }", + """f { + |{|a,b=42| + |[a,b] + |} + |}""".stripMargin + ) + + test( + "a { | b=1, c=2 | }", + """a { + |{|b=1,c=2|} + |}""".stripMargin + ) + + test( + "a { |**b | }", + """a { + |{|**b|} + |}""".stripMargin + ) + + test( + "bl { |kw: :val| kw }", + """bl { + |{|kw::val| + |kw + |} + |}""".stripMargin + ) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/FieldAccessParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/FieldAccessParserTests.scala index f7e0c1b91287..b6e9ad677d0b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/FieldAccessParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/FieldAccessParserTests.scala @@ -7,5 +7,24 @@ class FieldAccessParserTests extends RubyParserFixture with Matchers { "Normal field access" in { test("x.y") test("self.x") + test( + """a + |.b + |.c""".stripMargin, + "a.b.c" + ) + test( + """a + |.b + |#.c + |.d + |""".stripMargin, + "a.b.d" + ) + test( + """a. + |b""".stripMargin, + "a.b" + ) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/HashLiteralParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/HashLiteralParserTests.scala index da0af0655af7..25e4a0e59608 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/HashLiteralParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/HashLiteralParserTests.scala @@ -10,5 +10,11 @@ class HashLiteralParserTests extends RubyParserFixture with Matchers { test("{**x, **y}", "{**x,**y}") test("{**x, y => 1, **z}", "{**x,y=> 1,**z}") test("{**group_by_type(some)}") + test( + """{ + |:s1 => 1, + |}""".stripMargin, + "{:s1=> 1}" + ) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/IndexAccessParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/IndexAccessParserTests.scala index e6f80ff77f1b..77c377aa7a6a 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/IndexAccessParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/IndexAccessParserTests.scala @@ -7,5 +7,7 @@ class IndexAccessParserTests extends RubyParserFixture with Matchers { "Index access" in { test("a[1]") test("a[1,2]") + test("a[2,]", "a[2]") + test("a[2=>3,]", "a[2=> 3]") } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesisParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesisParserTests.scala index 7815f1a565d6..b063b882a497 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesisParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithParenthesisParserTests.scala @@ -4,6 +4,13 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class InvocationWithParenthesisParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("foo.()") // syntax error + test("defined?(42)") // parentheses go missing + test("foo([:b, :c=> 1]") // syntax error + test("x(&)") // Syntax error + } + "method invocation with parenthesis" in { test("foo()") test( @@ -20,6 +27,16 @@ class InvocationWithParenthesisParserTests extends RubyParserFixture with Matche test("foo(:region)") test("foo(:region,)", "foo(:region)") test("foo(if: true)") + test("foo(1, 2=>3)", "foo(1,2=> 3)") + test("foo(1, 2=>3,)", "foo(1,2=> 3)") + test("foo(1=> 2,)", "foo(1=> 2)") + test("foo(1, kw: 2, **3)", "foo(1,kw: 2,**3)") + test("foo(b, **1)", "foo(b,**1)") + test("""foo(b: if :c + |1 + |else + |2 + |end)""".stripMargin) test("foo&.bar()") test("foo&.bar(1, 2)", "foo&.bar(1,2)") test( @@ -34,5 +51,18 @@ class InvocationWithParenthesisParserTests extends RubyParserFixture with Matche |""".stripMargin, "foo.bar" ) + + test("f(1, kw:2, **3)", "f(1,kw: 2,**3)") + } + + "Method with comments" in { + test( + """# blah 1 + |# blah 2 + |def blah + |end""".stripMargin, + """def blah + |end""".stripMargin + ) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithoutParenthesesParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithoutParenthesesParserTests.scala index d2e9164abe53..8773fd6963be 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithoutParenthesesParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/InvocationWithoutParenthesesParserTests.scala @@ -8,6 +8,7 @@ class InvocationWithoutParenthesesParserTests extends RubyParserFixture with Mat test("task.nil?") test("foo?") test("foo!") + test("foo a.b 1") } "command with do block" in { @@ -22,7 +23,6 @@ class InvocationWithoutParenthesesParserTests extends RubyParserFixture with Mat ) test("foo&.bar") - test("foo&.bar 1, 2") } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MathOperatorParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MathOperatorParserTests.scala new file mode 100644 index 000000000000..e04eaaf9bcb9 --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MathOperatorParserTests.scala @@ -0,0 +1,15 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class MathOperatorParserTests extends RubyParserFixture with Matchers { + "Math operators" in { + test("1 + 2") + test("1 - 2") + test("1 * 2") + test("1 / 2") + test("1 ** 2") + test("1 + 2 - 3 * 4 / 5 ** 2") + } +} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionParserTests.scala index 00af163900ea..aa58bc64b644 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/MethodDefinitionParserTests.scala @@ -4,6 +4,14 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class MethodDefinitionParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("def f(a=1, *b, c) end") // syntax error + test("def f(*,a) end") // AstPrinter issue possibly + test("""def a(...) + |b(...) + |end""".stripMargin) // Syntax error + } + "single line method definition" in { test( "def foo; end", @@ -11,12 +19,31 @@ class MethodDefinitionParserTests extends RubyParserFixture with Matchers { |end""".stripMargin ) + test( + """def foo arg = false + |end""".stripMargin, + """def foo(arg=false) + |end""".stripMargin + ) + test( "def foo(x); end", """def foo(x) |end""".stripMargin ) + test( + "def f(a=nil, b) end", + """def f(a=nil,b) + |end""".stripMargin + ) + + test( + "def f(a, b = :c, d) end", + """def f(a,b=:c,d) + |end""".stripMargin + ) + test( "def foo(x=1); end", """def foo(x=1) @@ -70,6 +97,7 @@ class MethodDefinitionParserTests extends RubyParserFixture with Matchers { """def foo(name:,surname:) |end""".stripMargin ) + } "multi-line method definition" in { @@ -84,6 +112,45 @@ class MethodDefinitionParserTests extends RubyParserFixture with Matchers { |rescue ZeroDivisionError => e |end""".stripMargin ) + + test("""def x(y) + |p(y) + |y *= 2 + |return y + |end""".stripMargin) + + test("""def test(**testing) + |test_splat(**testing) + |end""".stripMargin) + + test( + """def fun(kw: :val) + |kw + |end""".stripMargin, + """def fun(kw::val) + |kw + |end""".stripMargin + ) + + test( + """def x a:, b: + |end""".stripMargin, + """def x(a:,b:) + |end""".stripMargin + ) + + test( + """def exec(cmd) + |system(cmd) + |rescue + |nil + |end""".stripMargin, + """def exec(cmd) + |system(cmd) + |rescue + |nil + |end""".stripMargin + ) } "endless method definition" in { @@ -91,6 +158,7 @@ class MethodDefinitionParserTests extends RubyParserFixture with Matchers { test("def foo =\n x", "def foo = x") test("def foo = \"something\"") test("def id(x) = x") + test("def foo = bar 42") } "method def with proc params" in { @@ -103,7 +171,6 @@ class MethodDefinitionParserTests extends RubyParserFixture with Matchers { |yield |end""".stripMargin ) - } "method def for mandatory parameters" in { @@ -169,4 +236,8 @@ class MethodDefinitionParserTests extends RubyParserFixture with Matchers { |end""".stripMargin ) } + + "alias method" in { + test("alias :start :on") + } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ProcDefinitionParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ProcDefinitionParserTests.scala index 5b0aa55349aa..03ce644257b3 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ProcDefinitionParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ProcDefinitionParserTests.scala @@ -4,6 +4,12 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class ProcDefinitionParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("->a{}") // Syntax error + test("->(b, c=1, *d, e, &f){}") // Syntax error + test("-> (a;b) {}") // Syntax error + } + "one-line proc definition" in { test("-> {}") @@ -43,5 +49,24 @@ class ProcDefinitionParserTests extends RubyParserFixture with Matchers { |puts y |}""".stripMargin ) + + test( + """a -> do 1 end do 2 end""", + """a -> do + |1 + |end do + |2 + |end""".stripMargin + ) + + test( + """a ->() { g do end }""", + """a -> { + |g do + |end + |}""".stripMargin + ) + + test("""-> (k:) { }""", """->(k:) {}""") } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RangeParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RangeParserTests.scala index ff7faa351589..ff0f1e5f660c 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RangeParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RangeParserTests.scala @@ -4,7 +4,25 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class RangeParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("""0... + |; a... + |; c + |""".stripMargin) // Syntax error + } + "Range Operator" in { test("1..2") + test( + """0.. + |4 + |a.. + |b + |c + |""".stripMargin, + """0..4 + |a..b + |c""".stripMargin + ) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexParserTests.scala index 411a867ec941..0bfe4eacc3ac 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RegexParserTests.scala @@ -37,5 +37,7 @@ class RegexParserTests extends RubyParserFixture with Matchers { test("""unless /\A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\z/i.match?(value) |end""".stripMargin) + + test("1 !~ 2") } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RescueClauseParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RescueClauseParserTests.scala index fd0de615f993..d6c0d5e5f272 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RescueClauseParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RescueClauseParserTests.scala @@ -40,6 +40,8 @@ class RescueClauseParserTests extends RubyParserFixture with Matchers { |rescue ZeroDivisionError => e |end""".stripMargin ) + + test("a (b rescue c)", "a b rescue c") } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ReturnParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ReturnParserTests.scala index 9ad1d78dd525..0e18a78437d6 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ReturnParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/ReturnParserTests.scala @@ -4,9 +4,16 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class ReturnParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("return y :z=> 1") + test("return 1, :z => 1") + } + "Standalone return statement" in { test("return") test("return ::X.y()", "return self::X.y()") test("return(0)", "return 0") + test("return y(z:1)", "return y(z: 1)") + test("return y(z=> 1)") } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringParserTests.scala index 4aa5b61bfa64..6f7009c64f76 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/StringParserTests.scala @@ -4,6 +4,10 @@ import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture import org.scalatest.matchers.should.Matchers class StringParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("%{ { #{ \"#{1}\" } } }") // syntax error + } + "single quoted literal" in { test("''") test("'x' 'y'", "'x''y'") @@ -43,6 +47,7 @@ class StringParserTests extends RubyParserFixture with Matchers { test("%Q()") test("%Q{text=#{1}}") test("%Q[#{1}#{2}]") + test("%Q[before [#{nest}] after]") } "expanded `%(` string literal" in { @@ -67,6 +72,7 @@ class StringParserTests extends RubyParserFixture with Matchers { test("\"#{1}#{2}\"") test(""""#{10} \ | is a number."""".stripMargin) + test(""""{a.b? ? ""+a+"" : ""}"""") } "Expanded `%x` external command literal" in { diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/YieldParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/YieldParserTests.scala new file mode 100644 index 000000000000..bc55483ef2bf --- /dev/null +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/YieldParserTests.scala @@ -0,0 +1,16 @@ +package io.joern.rubysrc2cpg.parser + +import io.joern.rubysrc2cpg.testfixtures.RubyParserFixture +import org.scalatest.matchers.should.Matchers + +class YieldParserTests extends RubyParserFixture with Matchers { + "fixme" ignore { + test("yield y z:1") + test("yield y :z=>1") + test("yield 1, :z => 1") + } + + "Yield tests" in { + test("yield y(z: 1)") + } +}