Move input processing out of RubyLex (#683)

* Add a test case for Ctrl-C handling

* Test symbol aliases with integration tests

There are a few places that also need to check symbol aliases before
`Irb#eval_input`. But since the current command test skip them, we
don't have test coverage on them.

* Move each_top_level_statement and readmultiline to Irb

This will save RubyLex from knowning information about commands and aliases.

lib/irb.rb

@@ -12,6 +12,7 @@
 require_relative "irb/extend-command"
 
 require_relative "irb/ruby-lex"
+require_relative "irb/statement"
 require_relative "irb/input-method"
 require_relative "irb/locale"
 require_relative "irb/color"
@@ -541,27 +542,9 @@ def eval_input
         @context.io.prompt
       end
 
-      @scanner.set_input do
-        signal_status(:IN_INPUT) do
-          if l = @context.io.gets
-            print l if @context.verbose?
-          else
-            if @context.ignore_eof? and @context.io.readable_after_eof?
-              l = "\n"
-              if @context.verbose?
-                printf "Use \"exit\" to leave %s\n", @context.ap_name
-              end
-            else
-              print "\n" if @context.prompting?
-            end
-          end
-          l
-        end
-      end
-
       configure_io
 
-      @scanner.each_top_level_statement do |statement, line_no|
+      each_top_level_statement do |statement, line_no|
         signal_status(:IN_EVAL) do
           begin
             # If the integration with debugger is activated, we need to handle certain input differently
@@ -591,6 +574,86 @@ def eval_input
       end
     end
 
+    def read_input
+      signal_status(:IN_INPUT) do
+        if l = @context.io.gets
+          print l if @context.verbose?
+        else
+          if @context.ignore_eof? and @context.io.readable_after_eof?
+            l = "\n"
+            if @context.verbose?
+              printf "Use \"exit\" to leave %s\n", @context.ap_name
+            end
+          else
+            print "\n" if @context.prompting?
+          end
+        end
+        l
+      end
+    end
+
+    def readmultiline
+      @scanner.save_prompt_to_context_io([], false, 0)
+
+      # multiline
+      return read_input if @context.io.respond_to?(:check_termination)
+
+      # nomultiline
+      code = ''
+      line_offset = 0
+      loop do
+        line = read_input
+        unless line
+          return code.empty? ? nil : code
+        end
+
+        code << line
+
+        # Accept any single-line input for symbol aliases or commands that transform args
+        return code if single_line_command?(code)
+
+        tokens, opens, terminated = @scanner.check_code_state(code)
+        return code if terminated
+
+        line_offset += 1
+        continue = @scanner.should_continue?(tokens)
+        @scanner.save_prompt_to_context_io(opens, continue, line_offset)
+      end
+    end
+
+    def each_top_level_statement
+      loop do
+        code = readmultiline
+        break unless code
+
+        if code != "\n"
+          yield build_statement(code), @scanner.line_no
+        end
+        @scanner.increase_line_no(code.count("\n"))
+      rescue RubyLex::TerminateLineInput
+      end
+    end
+
+    def build_statement(code)
+      code.force_encoding(@context.io.encoding)
+      command_or_alias, arg = code.split(/\s/, 2)
+      # Transform a non-identifier alias (@, $) or keywords (next, break)
+      command_name = @context.command_aliases[command_or_alias.to_sym]
+      command = command_name || command_or_alias
+      command_class = ExtendCommandBundle.load_command(command)
+
+      if command_class
+        Statement::Command.new(code, command, arg, command_class)
+      else
+        Statement::Expression.new(code, @scanner.assignment_expression?(code))
+      end
+    end
+
+    def single_line_command?(code)
+      command = code.split(/\s/, 2).first
+      @context.symbol_alias?(command) || @context.transform_args?(command)
+    end
+
     def configure_io
       if @context.io.respond_to?(:check_termination)
         @context.io.check_termination do |code|
@@ -607,7 +670,7 @@ def configure_io
             end
           else
             # Accept any single-line input for symbol aliases or commands that transform args
-            next true if @scanner.single_line_command?(code)
+            next true if single_line_command?(code)
 
             _tokens, _opens, terminated = @scanner.check_code_state(code)
             terminated

lib/irb/ruby-lex.rb

@@ -7,7 +7,6 @@
 require "ripper"
 require "jruby" if RUBY_ENGINE == "jruby"
 require_relative "nesting_parser"
-require_relative "statement"
 
 # :stopdoc:
 class RubyLex
@@ -42,6 +41,8 @@ def initialize
     end
   end
 
+  attr_reader :line_no
+
   def initialize(context)
     @context = context
     @line_no = 1
@@ -65,16 +66,6 @@ def self.compile_with_errors_suppressed(code, line_no: 1)
     result
   end
 
-  def single_line_command?(code)
-    command = code.split(/\s/, 2).first
-    @context.symbol_alias?(command) || @context.transform_args?(command)
-  end
-
-  # io functions
-  def set_input(&block)
-    @input = block
-  end
-
   def set_prompt(&block)
     @prompt = block
   end
@@ -188,62 +179,6 @@ def increase_line_no(addition)
     @line_no += addition
   end
 
-  def readmultiline
-    save_prompt_to_context_io([], false, 0)
-
-    # multiline
-    return @input.call if @context.io.respond_to?(:check_termination)
-
-    # nomultiline
-    code = ''
-    line_offset = 0
-    loop do
-      line = @input.call
-      unless line
-        return code.empty? ? nil : code
-      end
-
-      code << line
-      # Accept any single-line input for symbol aliases or commands that transform args
-      return code if single_line_command?(code)
-
-      tokens, opens, terminated = check_code_state(code)
-      return code if terminated
-
-      line_offset += 1
-      continue = should_continue?(tokens)
-      save_prompt_to_context_io(opens, continue, line_offset)
-    end
-  end
-
-  def each_top_level_statement
-    loop do
-      code = readmultiline
-      break unless code
-
-      if code != "\n"
-        yield build_statement(code), @line_no
-      end
-      increase_line_no(code.count("\n"))
-    rescue TerminateLineInput
-    end
-  end
-
-  def build_statement(code)
-    code.force_encoding(@context.io.encoding)
-    command_or_alias, arg = code.split(/\s/, 2)
-    # Transform a non-identifier alias (@, $) or keywords (next, break)
-    command_name = @context.command_aliases[command_or_alias.to_sym]
-    command = command_name || command_or_alias
-    command_class = IRB::ExtendCommandBundle.load_command(command)
-
-    if command_class
-      IRB::Statement::Command.new(code, command, arg, command_class)
-    else
-      IRB::Statement::Expression.new(code, assignment_expression?(code))
-    end
-  end
-
   def assignment_expression?(code)
     # Try to parse the code and check if the last of possibly multiple
     # expressions is an assignment type.

test/irb/test_cmd.rb

@@ -62,23 +62,6 @@ def test_calling_command_on_a_frozen_main
     end
   end
 
-  class CommnadAliasTest < CommandTestCase
-    def test_vars_with_aliases
-      @foo = "foo"
-      $bar = "bar"
-      out, err = execute_lines(
-        "@foo\n",
-        "$bar\n",
-      )
-      assert_empty err
-      assert_match(/"foo"/, out)
-      assert_match(/"bar"/, out)
-    ensure
-      remove_instance_variable(:@foo)
-      $bar = nil
-    end
-  end
-
   class InfoTest < CommandTestCase
     def setup
       super

test/irb/test_irb.rb

@@ -4,6 +4,67 @@
 require_relative "helper"
 
 module TestIRB
+  class InputTest < IntegrationTestCase
+    def test_symbol_aliases_are_handled_correctly
+      write_ruby <<~'RUBY'
+        class Foo
+        end
+        binding.irb
+      RUBY
+
+      output = run_ruby_file do
+        type "$ Foo"
+        type "exit!"
+      end
+
+      assert_include output, "From: #{@ruby_file.path}:1"
+    end
+
+    def test_symbol_aliases_are_handled_correctly_with_singleline_mode
+      @irbrc = Tempfile.new('irbrc')
+      @irbrc.write <<~RUBY
+        IRB.conf[:USE_SINGLELINE] = true
+      RUBY
+      @irbrc.close
+      @envs['IRBRC'] = @irbrc.path
+
+      write_ruby <<~'RUBY'
+        class Foo
+        end
+        binding.irb
+      RUBY
+
+      output = run_ruby_file do
+        type "irb_info"
+        type "$ Foo"
+        type "exit!"
+      end
+
+      # Make sure it's tested in singleline mode
+      assert_include output, "InputMethod: ReadlineInputMethod"
+      assert_include output, "From: #{@ruby_file.path}:1"
+    ensure
+      @irbrc.unlink if @irbrc
+    end
+
+    def test_symbol_aliases_dont_affect_ruby_syntax
+      write_ruby <<~'RUBY'
+        $foo = "It's a foo"
+        @bar = "It's a bar"
+        binding.irb
+      RUBY
+
+      output = run_ruby_file do
+        type "$foo"
+        type "@bar"
+        type "exit!"
+      end
+
+      assert_include output, "=> \"It's a foo\""
+      assert_include output, "=> \"It's a bar\""
+    end
+  end
+
   class IrbIOConfigurationTest < TestCase
     Row = Struct.new(:content, :current_line_spaces, :new_line_spaces, :indent_level)
 

test/irb/yamatanooroti/test_rendering.rb

@@ -249,6 +249,22 @@ def test_assignment_expression_truncate
     EOC
   end
 
+  def test_ctrl_c_is_handled
+    write_irbrc <<~'LINES'
+      puts 'start IRB'
+    LINES
+    start_terminal(40, 80, %W{ruby -I#{@pwd}/lib #{@pwd}/exe/irb}, startup_message: 'start IRB')
+    # Assignment expression code that turns into non-assignment expression after evaluation
+    write("\C-c")
+    close
+    assert_screen(<<~EOC)
+      start IRB
+      irb(main):001>
+      ^C
+      irb(main):001>
+    EOC
+  end
+
   def test_show_cmds_with_pager_can_quit_with_ctrl_c
     write_irbrc <<~'LINES'
       puts 'start IRB'