From 194e0f4389fe809666bffddf871db4391f7d029a Mon Sep 17 00:00:00 2001 From: Xavier Pinho Date: Mon, 7 Aug 2023 21:07:43 +0100 Subject: [PATCH] RubyLexer: Generalized non-expanded delimited array (#3447) --- .../io/joern/rubysrc2cpg/parser/RubyLexer.g4 | 100 ++++-------------- .../io/joern/rubysrc2cpg/parser/RubyParser.g4 | 12 +-- ...ala => NonExpandedDelimiterHandling.scala} | 8 +- .../rubysrc2cpg/parser/RubyLexerBase.scala | 4 +- ...QuotedNonExpandedStringArrayHandling.scala | 43 -------- ...QuotedNonExpandedSymbolArrayHandling.scala | 43 -------- .../rubysrc2cpg/parser/RubyLexerTests.scala | 36 +++---- 7 files changed, 49 insertions(+), 197 deletions(-) rename joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/{NonExpandedDelimitedStringHandling.scala => NonExpandedDelimiterHandling.scala} (77%) delete mode 100644 joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerQuotedNonExpandedStringArrayHandling.scala delete mode 100644 joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerQuotedNonExpandedSymbolArrayHandling.scala diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4 index 494385887d28..1d61a0e2da66 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4 +++ b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4 @@ -289,7 +289,7 @@ QUOTED_NON_EXPANDED_STRING_LITERAL_START : '%q' {!Character.isAlphabetic(_input.LA(1))}? { pushNonExpandedDelimiter(_input.LA(1)); - setNonExpandedDelimitedStringEndToken(QUOTED_NON_EXPANDED_STRING_LITERAL_END); + setNonExpandedDelimiterEndToken(QUOTED_NON_EXPANDED_STRING_LITERAL_END); _input.consume(); pushMode(NON_EXPANDED_DELIMITED_STRING_MODE); } @@ -299,7 +299,7 @@ QUOTED_NON_EXPANDED_REGULAR_EXPRESSION_START : '%r' {!Character.isAlphabetic(_input.LA(1))}? { pushNonExpandedDelimiter(_input.LA(1)); - setNonExpandedDelimitedStringEndToken(QUOTED_NON_EXPANDED_REGULAR_EXPRESSION_END); + setNonExpandedDelimiterEndToken(QUOTED_NON_EXPANDED_REGULAR_EXPRESSION_END); _input.consume(); pushMode(NON_EXPANDED_DELIMITED_STRING_MODE); }; @@ -311,9 +311,10 @@ QUOTED_NON_EXPANDED_REGULAR_EXPRESSION_START QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START : '%w' {!Character.isAlphabetic(_input.LA(1))}? { - pushQuotedNonExpandedStringArrayDelimiter(_input.LA(1)); + pushNonExpandedDelimiter(_input.LA(1)); + setNonExpandedDelimiterEndToken(QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END); _input.consume(); - pushMode(QUOTED_NON_EXPANDED_STRING_ARRAY_MODE); + pushMode(NON_EXPANDED_DELIMITED_ARRAY_MODE); } ; @@ -324,9 +325,10 @@ QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START : '%i' {!Character.isAlphabetic(_input.LA(1))}? { - pushQuotedNonExpandedSymbolArrayDelimiter(_input.LA(1)); + pushNonExpandedDelimiter(_input.LA(1)); + setNonExpandedDelimiterEndToken(QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END); _input.consume(); - pushMode(QUOTED_NON_EXPANDED_SYMBOL_ARRAY_MODE); + pushMode(NON_EXPANDED_DELIMITED_ARRAY_MODE); } ; @@ -684,69 +686,12 @@ NON_EXPANDED_LITERAL_CHARACTER ; // -------------------------------------------------------- -// %w string (word) array mode -// -------------------------------------------------------- - -mode QUOTED_NON_EXPANDED_STRING_ARRAY_MODE; - -fragment QUOTED_NON_EXPANDED_ESCAPED_STRING_ARRAY_CHARACTER - : '\\' QUOTED_NON_EXPANDED_NON_ESCAPED_STRING_ARRAY_CHARACTER - ; - -fragment QUOTED_NON_EXPANDED_NON_ESCAPED_STRING_ARRAY_CHARACTER - : ~[\r\n] - | '\n' {_input.LA(1) != '\r'}? - ; - -fragment QUOTED_NON_EXPANDED_STRING_ARRAY_DELIMITER - : [\u0009] - | [\u000b] - | [\u000c] - | [\u000d] - | [\u0020] - | '\\' ('\r'? '\n') - ; - -QUOTED_NON_EXPANDED_STRING_ARRAY_SEPARATOR - : QUOTED_NON_EXPANDED_STRING_ARRAY_DELIMITER+ - ; - -QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER - : QUOTED_NON_EXPANDED_ESCAPED_STRING_ARRAY_CHARACTER - | QUOTED_NON_EXPANDED_NON_ESCAPED_STRING_ARRAY_CHARACTER - { - int readChar = _input.LA(-1); - - if (isQuotedNonExpandedStringArrayClosingDelimiter(readChar)) { - popQuotedNonExpandedStringArrayDelimiter(); - - if (isQuotedNonExpandedStringArrayDelimitersEmpty()) { - setType(QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END); - popMode(); - } - } - else if (isQuotedNonExpandedStringArrayOpeningDelimiter(readChar)) { - pushQuotedNonExpandedStringArrayDelimiter(readChar); - } - } - ; - -// -------------------------------------------------------- -// %i symbol array mode +// Non-expanded delimited array mode // -------------------------------------------------------- -mode QUOTED_NON_EXPANDED_SYMBOL_ARRAY_MODE; - -fragment QUOTED_NON_EXPANDED_ESCAPED_SYMBOL_ARRAY_CHARACTER - : '\\' QUOTED_NON_EXPANDED_NON_ESCAPED_SYMBOL_ARRAY_CHARACTER - ; - -fragment QUOTED_NON_EXPANDED_NON_ESCAPED_SYMBOL_ARRAY_CHARACTER - : ~[\r\n] - | '\n' {_input.LA(1) != '\r'}? - ; +mode NON_EXPANDED_DELIMITED_ARRAY_MODE; -fragment QUOTED_NON_EXPANDED_SYMBOL_ARRAY_DELIMITER +fragment NON_EXPANDED_ARRAY_ITEM_DELIMITER : [\u0009] | [\u000a] | [\u000b] @@ -756,31 +701,30 @@ fragment QUOTED_NON_EXPANDED_SYMBOL_ARRAY_DELIMITER | '\\' ('\r'? '\n') ; -QUOTED_NON_EXPANDED_SYMBOL_ARRAY_SEPARATOR - : QUOTED_NON_EXPANDED_SYMBOL_ARRAY_DELIMITER+ +NON_EXPANDED_ARRAY_ITEM_SEPARATOR + : NON_EXPANDED_ARRAY_ITEM_DELIMITER+ ; -QUOTED_NON_EXPANDED_SYMBOL_ARRAY_CHARACTER - : QUOTED_NON_EXPANDED_ESCAPED_SYMBOL_ARRAY_CHARACTER - | QUOTED_NON_EXPANDED_NON_ESCAPED_SYMBOL_ARRAY_CHARACTER +NON_EXPANDED_ARRAY_ITEM_CHARACTER + : NON_EXPANDED_LITERAL_ESCAPE_SEQUENCE + | NON_ESCAPED_LITERAL_CHARACTER { int readChar = _input.LA(-1); - if (isQuotedNonExpandedSymbolArrayClosingDelimiter(readChar)) { - popQuotedNonExpandedSymbolArrayDelimiter(); + if (isNonExpandedClosingDelimiter(readChar)) { + popNonExpandedDelimiter(); - if (isQuotedNonExpandedSymbolArrayDelimitersEmpty()) { - setType(QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END); + if (isNonExpandedDelimitersStackEmpty()) { + setType(getNonExpandedDelimitedStringEndToken()); popMode(); } } - else if (isQuotedNonExpandedSymbolArrayOpeningDelimiter(readChar)) { - pushQuotedNonExpandedSymbolArrayDelimiter(readChar); + else if (isNonExpandedOpeningDelimiter(readChar)) { + pushNonExpandedDelimiter(readChar); } } ; - // -------------------------------------------------------- // Regex literal mode // -------------------------------------------------------- diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 index e2de7a530a80..5f3640602df2 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 +++ b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 @@ -261,26 +261,26 @@ arrayConstructor nonExpandedWordArrayElements? QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END # nonExpandedWordArrayConstructor | QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_SEPARATOR* + NON_EXPANDED_ARRAY_ITEM_SEPARATOR* nonExpandedSymbolArrayElements? - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_SEPARATOR* + NON_EXPANDED_ARRAY_ITEM_SEPARATOR* QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END # nonExpandedSymbolArrayConstructor ; nonExpandedSymbolArrayElements - : nonExpandedSymbolArrayElement (QUOTED_NON_EXPANDED_SYMBOL_ARRAY_SEPARATOR+ nonExpandedSymbolArrayElement)* + : nonExpandedSymbolArrayElement (NON_EXPANDED_ARRAY_ITEM_SEPARATOR+ nonExpandedSymbolArrayElement)* ; nonExpandedSymbolArrayElement - : QUOTED_NON_EXPANDED_SYMBOL_ARRAY_CHARACTER+ + : NON_EXPANDED_ARRAY_ITEM_CHARACTER+ ; nonExpandedWordArrayElements - : nonExpandedWordArrayElement (QUOTED_NON_EXPANDED_STRING_ARRAY_SEPARATOR nonExpandedWordArrayElement)* + : nonExpandedWordArrayElement (NON_EXPANDED_ARRAY_ITEM_SEPARATOR+ nonExpandedWordArrayElement)* ; nonExpandedWordArrayElement - : QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER+ + : NON_EXPANDED_ARRAY_ITEM_CHARACTER+ ; diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/NonExpandedDelimitedStringHandling.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/NonExpandedDelimiterHandling.scala similarity index 77% rename from joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/NonExpandedDelimitedStringHandling.scala rename to joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/NonExpandedDelimiterHandling.scala index caa09a83a4f0..933cdf6bb9ef 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/NonExpandedDelimitedStringHandling.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/NonExpandedDelimiterHandling.scala @@ -2,7 +2,7 @@ package io.joern.rubysrc2cpg.parser import scala.collection.mutable -trait NonExpandedDelimitedStringHandling { this: RubyLexerBase => +trait NonExpandedDelimiterHandling { this: RubyLexerBase => private val delimiters = mutable.Stack[Int]() private var endTokenType = 0 @@ -27,15 +27,11 @@ trait NonExpandedDelimitedStringHandling { this: RubyLexerBase => char == currentClosingDelimiter() } - def isNonExpandedDelimiter(char: Int): Boolean = { - isNonExpandedOpeningDelimiter(char) || isNonExpandedClosingDelimiter(char) - } - private def currentOpeningDelimiter(): Int = { delimiters.top } - def setNonExpandedDelimitedStringEndToken(endTokenType: Int): Unit = { + def setNonExpandedDelimiterEndToken(endTokenType: Int): Unit = { this.endTokenType = endTokenType } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerBase.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerBase.scala index 28232ded9d16..ec22addbb138 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerBase.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerBase.scala @@ -9,9 +9,7 @@ abstract class RubyLexerBase(input: CharStream) extends Lexer(input) with RubyLexerRegexHandling with RubyLexerStringInterpolationHandling - with RubyLexerQuotedNonExpandedStringArrayHandling - with RubyLexerQuotedNonExpandedSymbolArrayHandling - with NonExpandedDelimitedStringHandling { + with NonExpandedDelimiterHandling { /** The previously (non-WS) emitted token (in DEFAULT_CHANNEL.) */ protected var previousNonWsToken: Option[Token] = None diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerQuotedNonExpandedStringArrayHandling.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerQuotedNonExpandedStringArrayHandling.scala deleted file mode 100644 index 424c910cf60e..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerQuotedNonExpandedStringArrayHandling.scala +++ /dev/null @@ -1,43 +0,0 @@ -package io.joern.rubysrc2cpg.parser - -import scala.collection.mutable - -// TODO: Very similar to `RubyLexerQuotedNonExpandedStringHandling`. Consider refactoring them. -trait RubyLexerQuotedNonExpandedStringArrayHandling { this: RubyLexerBase => - - private val quotedNonExpandedStringArrayLiteralOpeningDelimiters = mutable.Stack[Int]() - - def pushQuotedNonExpandedStringArrayDelimiter(char: Int): Unit = { - quotedNonExpandedStringArrayLiteralOpeningDelimiters.push(char) - } - - def popQuotedNonExpandedStringArrayDelimiter(): Unit = { - quotedNonExpandedStringArrayLiteralOpeningDelimiters.pop() - } - - def isQuotedNonExpandedStringArrayDelimitersEmpty: Boolean = { - quotedNonExpandedStringArrayLiteralOpeningDelimiters.isEmpty - } - - def isQuotedNonExpandedStringArrayOpeningDelimiter(char: Int): Boolean = { - char == currentOpeningDelimiter() - } - - def isQuotedNonExpandedStringArrayClosingDelimiter(char: Int): Boolean = { - char == currentClosingDelimiter() - } - - private def currentOpeningDelimiter(): Int = { - quotedNonExpandedStringArrayLiteralOpeningDelimiters.top - } - - private def currentClosingDelimiter(): Int = - currentOpeningDelimiter() match { - case '(' => ')' - case '[' => ']' - case '{' => '}' - case '<' => '>' - case c => c - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerQuotedNonExpandedSymbolArrayHandling.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerQuotedNonExpandedSymbolArrayHandling.scala deleted file mode 100644 index 36cf86a75630..000000000000 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyLexerQuotedNonExpandedSymbolArrayHandling.scala +++ /dev/null @@ -1,43 +0,0 @@ -package io.joern.rubysrc2cpg.parser - -import scala.collection.mutable - -// TODO: Refactor with `RubyLexerQuotedNonExpandedStringArrayHandling` in mind. -trait RubyLexerQuotedNonExpandedSymbolArrayHandling { this: RubyLexerBase => - - private val quotedNonExpandedSymbolArrayLiteralOpeningDelimiters = mutable.Stack[Int]() - - def pushQuotedNonExpandedSymbolArrayDelimiter(char: Int): Unit = { - quotedNonExpandedSymbolArrayLiteralOpeningDelimiters.push(char) - } - - def popQuotedNonExpandedSymbolArrayDelimiter(): Unit = { - quotedNonExpandedSymbolArrayLiteralOpeningDelimiters.pop() - } - - def isQuotedNonExpandedSymbolArrayDelimitersEmpty: Boolean = { - quotedNonExpandedSymbolArrayLiteralOpeningDelimiters.isEmpty - } - - def isQuotedNonExpandedSymbolArrayOpeningDelimiter(char: Int): Boolean = { - char == currentOpeningDelimiter() - } - - def isQuotedNonExpandedSymbolArrayClosingDelimiter(char: Int): Boolean = { - char == currentClosingDelimiter() - } - - private def currentOpeningDelimiter(): Int = { - quotedNonExpandedSymbolArrayLiteralOpeningDelimiters.top - } - - private def currentClosingDelimiter(): Int = - currentOpeningDelimiter() match { - case '(' => ')' - case '[' => ']' - case '{' => '}' - case '<' => '>' - case c => c - } - -} diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RubyLexerTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RubyLexerTests.scala index f9e327a7f512..69957602e4db 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RubyLexerTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/RubyLexerTests.scala @@ -593,7 +593,7 @@ class RubyLexerTests extends AnyFlatSpec with Matchers { Seq("%w(x)", "%w[y]", "%w{z}", "%w", "%w#a#", "%w!b!", "%w-_-", "%w@c@", "%w+d+", "%w*e*", "%w/#/", "%w&!&") all(eg.map(tokenize)) shouldBe Seq( QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START, - QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, EOF ) @@ -616,10 +616,10 @@ class RubyLexerTests extends AnyFlatSpec with Matchers { ) all(eg.map(tokenize)) shouldBe Seq( QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START, - QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER, - QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER, - QUOTED_NON_EXPANDED_STRING_ARRAY_SEPARATOR, - QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_SEPARATOR, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, EOF ) @@ -629,9 +629,9 @@ class RubyLexerTests extends AnyFlatSpec with Matchers { val code = """%w[x\ y]""" tokenize(code) shouldBe Seq( QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_START, - QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER, - QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER, - QUOTED_NON_EXPANDED_STRING_ARRAY_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, QUOTED_NON_EXPANDED_STRING_ARRAY_LITERAL_END, EOF ) @@ -651,7 +651,7 @@ class RubyLexerTests extends AnyFlatSpec with Matchers { Seq("%i(x)", "%i[y]", "%i{z}", "%i", "%i#a#", "%i!b!", "%i-_-", "%i@c@", "%i+d+", "%i*e*", "%i/#/", "%i&!&") all(eg.map(tokenize)) shouldBe Seq( QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END, EOF ) @@ -674,10 +674,10 @@ class RubyLexerTests extends AnyFlatSpec with Matchers { ) all(eg.map(tokenize)) shouldBe Seq( QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_CHARACTER, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_CHARACTER, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_SEPARATOR, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_SEPARATOR, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END, EOF ) @@ -691,11 +691,11 @@ class RubyLexerTests extends AnyFlatSpec with Matchers { |)""".stripMargin tokenize(code) shouldBe Seq( QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_START, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_SEPARATOR, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_CHARACTER, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_SEPARATOR, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_CHARACTER, - QUOTED_NON_EXPANDED_SYMBOL_ARRAY_SEPARATOR, + NON_EXPANDED_ARRAY_ITEM_SEPARATOR, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_SEPARATOR, + NON_EXPANDED_ARRAY_ITEM_CHARACTER, + NON_EXPANDED_ARRAY_ITEM_SEPARATOR, QUOTED_NON_EXPANDED_SYMBOL_ARRAY_LITERAL_END, EOF )