Provide code navigation features to erb files (#2235)

* Provide features to erb files

This commit allows Ruby LSP to start handling requests for ERB files,
such as definition, completion, hover...etc., which will give users the
same level of code navigation features in ERB and Ruby.

However, certain requests are not supported:
- formatting
- on_type_formatting
- diagnostics
- code_actions

Co-authored-by: Vinicius Stock <vinicius.stock@shopify.com>

* Treat top-level method calls in erb files as receiver unknown

Currently their receivers will be assumed as `Object`, which means
most helper methods like `link_to` won't have completion even though
we do have it in the index and are sure that `Object` doesn't define it.

By avoiding setting the receiver as `Object`, we can greatly increase the
number methods that can have definition support.

* Handle windows newline characters correctly

* Avoid passing document to listeners

* Update lib/ruby_lsp/store.rb

Co-authored-by: Vinicius Stock <vinistock@users.noreply.github.com>

---------

Co-authored-by: Vinicius Stock <vinicius.stock@shopify.com>
Co-authored-by: Vinicius Stock <vinistock@users.noreply.github.com>

lib/ruby_lsp/document.rb

@@ -3,6 +3,13 @@
 
 module RubyLsp
   class Document
+    class LanguageId < T::Enum
+      enums do
+        Ruby = new("ruby")
+        ERB = new("erb")
+      end
+    end
+
     extend T::Sig
     extend T::Helpers
 
@@ -36,9 +43,12 @@ def initialize(source:, version:, uri:, encoding: Encoding::UTF_8)
 
     sig { params(other: Document).returns(T::Boolean) }
     def ==(other)
-      @source == other.source
+      self.class == other.class && uri == other.uri && @source == other.source
     end
 
+    sig { abstract.returns(LanguageId) }
+    def language_id; end
+
     # TODO: remove this method once all nonpositional requests have been migrated to the listener pattern
     sig do
       type_parameters(:T)

lib/ruby_lsp/erb_document.rb

@@ -0,0 +1,125 @@
+# typed: strict
+# frozen_string_literal: true
+
+module RubyLsp
+  class ERBDocument < Document
+    extend T::Sig
+
+    sig { override.returns(Prism::ParseResult) }
+    def parse
+      return @parse_result unless @needs_parsing
+
+      @needs_parsing = false
+      scanner = ERBScanner.new(@source)
+      scanner.scan
+      @parse_result = Prism.parse(scanner.ruby)
+    end
+
+    sig { override.returns(T::Boolean) }
+    def syntax_error?
+      @parse_result.failure?
+    end
+
+    sig { override.returns(LanguageId) }
+    def language_id
+      LanguageId::ERB
+    end
+
+    class ERBScanner
+      extend T::Sig
+
+      sig { returns(String) }
+      attr_reader :ruby, :html
+
+      sig { params(source: String).void }
+      def initialize(source)
+        @source = source
+        @html = T.let(+"", String)
+        @ruby = T.let(+"", String)
+        @current_pos = T.let(0, Integer)
+        @inside_ruby = T.let(false, T::Boolean)
+      end
+
+      sig { void }
+      def scan
+        while @current_pos < @source.length
+          scan_char
+          @current_pos += 1
+        end
+      end
+
+      private
+
+      sig { void }
+      def scan_char
+        char = @source[@current_pos]
+
+        case char
+        when "<"
+          if next_char == "%"
+            @inside_ruby = true
+            @current_pos += 1
+            push_char("  ")
+
+            if next_char == "=" && @source[@current_pos + 2] == "="
+              @current_pos += 2
+              push_char("  ")
+            elsif next_char == "=" || next_char == "-"
+              @current_pos += 1
+              push_char(" ")
+            end
+          else
+            push_char(T.must(char))
+          end
+        when "-"
+          if @inside_ruby && next_char == "%" &&
+              @source[@current_pos + 2] == ">"
+            @current_pos += 2
+            push_char("   ")
+            @inside_ruby = false
+          else
+            push_char(T.must(char))
+          end
+        when "%"
+          if @inside_ruby && next_char == ">"
+            @inside_ruby = false
+            @current_pos += 1
+            push_char("  ")
+          else
+            push_char(T.must(char))
+          end
+        when "\r"
+          @ruby << char
+          @html << char
+
+          if next_char == "\n"
+            @ruby << next_char
+            @html << next_char
+            @current_pos += 1
+          end
+        when "\n"
+          @ruby << char
+          @html << char
+        else
+          push_char(T.must(char))
+        end
+      end
+
+      sig { params(char: String).void }
+      def push_char(char)
+        if @inside_ruby
+          @ruby << char
+          @html << " " * char.length
+        else
+          @ruby << " " * char.length
+          @html << char
+        end
+      end
+
+      sig { returns(String) }
+      def next_char
+        @source[@current_pos + 1] || ""
+      end
+    end
+  end
+end

lib/ruby_lsp/internal.rb

@@ -15,6 +15,7 @@
 require "uri"
 require "cgi"
 require "set"
+require "strscan"
 require "prism"
 require "prism/visitor"
 require "language_server-protocol"
@@ -34,6 +35,7 @@
 require "ruby_lsp/node_context"
 require "ruby_lsp/document"
 require "ruby_lsp/ruby_document"
+require "ruby_lsp/erb_document"
 require "ruby_lsp/store"
 require "ruby_lsp/addon"
 require "ruby_lsp/requests/support/rubocop_runner"

lib/ruby_lsp/listeners/definition.rb

@@ -16,17 +16,19 @@ class Definition
             Interface::LocationLink,
           )],
           global_state: GlobalState,
+          language_id: Document::LanguageId,
           uri: URI::Generic,
           node_context: NodeContext,
           dispatcher: Prism::Dispatcher,
           typechecker_enabled: T::Boolean,
         ).void
       end
-      def initialize(response_builder, global_state, uri, node_context, dispatcher, typechecker_enabled) # rubocop:disable Metrics/ParameterLists
+      def initialize(response_builder, global_state, language_id, uri, node_context, dispatcher, typechecker_enabled) # rubocop:disable Metrics/ParameterLists
         @response_builder = response_builder
         @global_state = global_state
         @index = T.let(global_state.index, RubyIndexer::Index)
         @type_inferrer = T.let(global_state.type_inferrer, TypeInferrer)
+        @language_id = language_id
         @uri = uri
         @node_context = node_context
         @typechecker_enabled = typechecker_enabled
@@ -52,7 +54,15 @@ def on_call_node_enter(node)
         message = node.message
         return unless message
 
-        handle_method_definition(message, @type_inferrer.infer_receiver_type(@node_context))
+        inferrer_receiver_type = @type_inferrer.infer_receiver_type(@node_context)
+
+        # Until we can properly infer the receiver type in erb files (maybe with ruby-lsp-rails),
+        # treating method calls' type as `nil` will allow users to get some completion support first
+        if @language_id == Document::LanguageId::ERB && inferrer_receiver_type == "Object"
+          inferrer_receiver_type = nil
+        end
+
+        handle_method_definition(message, inferrer_receiver_type)
       end
 
       sig { params(node: Prism::StringNode).void }

lib/ruby_lsp/requests/definition.rb

@@ -90,6 +90,7 @@ def initialize(document, global_state, position, dispatcher, typechecker_enabled
           Listeners::Definition.new(
             @response_builder,
             global_state,
+            document.language_id,
             document.uri,
             node_context,
             dispatcher,

lib/ruby_lsp/requests/formatting.rb

@@ -40,6 +40,8 @@ def perform
         return unless @active_formatter
         return if @document.syntax_error?
 
+        # We don't format erb documents yet
+
         formatted_text = @active_formatter.run_formatting(@uri, @document)
         return unless formatted_text
 

lib/ruby_lsp/ruby_document.rb

@@ -15,5 +15,10 @@ def parse
     def syntax_error?
       @parse_result.failure?
     end
+
+    sig { override.returns(LanguageId) }
+    def language_id
+      LanguageId::Ruby
+    end
   end
 end

lib/ruby_lsp/server.rb

@@ -288,11 +288,18 @@ def run_initialized
     def text_document_did_open(message)
       @mutex.synchronize do
         text_document = message.dig(:params, :textDocument)
+        language_id = case text_document[:languageId]
+        when "erb"
+          Document::LanguageId::ERB
+        else
+          Document::LanguageId::Ruby
+        end
         @store.set(
           uri: text_document[:uri],
           source: text_document[:text],
           version: text_document[:version],
           encoding: @global_state.encoding,
+          language_id: language_id,
         )
       end
     end
@@ -421,7 +428,11 @@ def text_document_formatting(message)
         return
       end
 
-      response = Requests::Formatting.new(@global_state, @store.get(uri)).perform
+      document = @store.get(uri)
+
+      return send_empty_response(message[:id]) if document.is_a?(ERBDocument)
+
+      response = Requests::Formatting.new(@global_state, document).perform
       send_message(Result.new(id: message[:id], response: response))
     rescue Requests::Request::InvalidFormatter => error
       send_message(Notification.window_show_error("Configuration error: #{error.message}"))
@@ -444,12 +455,15 @@ def text_document_document_highlight(message)
     sig { params(message: T::Hash[Symbol, T.untyped]).void }
     def text_document_on_type_formatting(message)
       params = message[:params]
+      document = @store.get(params.dig(:textDocument, :uri))
+
+      return send_empty_response(message[:id]) if document.is_a?(ERBDocument)
 
       send_message(
         Result.new(
           id: message[:id],
           response: Requests::OnTypeFormatting.new(
-            @store.get(params.dig(:textDocument, :uri)),
+            document,
             params[:position],
             params[:ch],
             @store.client_name,
@@ -499,6 +513,8 @@ def text_document_code_action(message)
       params = message[:params]
       document = @store.get(params.dig(:textDocument, :uri))
 
+      return send_empty_response(message[:id]) if document.is_a?(ERBDocument)
+
       send_message(
         Result.new(
           id: message[:id],
@@ -552,7 +568,11 @@ def text_document_diagnostic(message)
         return
       end
 
-      response = @store.cache_fetch(uri, "textDocument/diagnostic") do |document|
+      document = @store.get(uri)
+
+      return send_empty_response(message[:id]) if document.is_a?(ERBDocument)
+
+      response = document.cache_fetch("textDocument/diagnostic") do |document|
         Requests::Diagnostics.new(@global_state, document).perform
       end
 

lib/ruby_lsp/store.rb

@@ -37,13 +37,32 @@ def get(uri)
       return document unless document.nil?
 
       path = T.must(uri.to_standardized_path)
-      set(uri: uri, source: File.binread(path), version: 0)
+      ext = File.extname(path)
+      language_id = if ext == ".erb" || ext == ".rhtml"
+        Document::LanguageId::ERB
+      else
+        Document::LanguageId::Ruby
+      end
+      set(uri: uri, source: File.binread(path), version: 0, language_id: language_id)
       T.must(@state[uri.to_s])
     end
 
-    sig { params(uri: URI::Generic, source: String, version: Integer, encoding: Encoding).void }
-    def set(uri:, source:, version:, encoding: Encoding::UTF_8)
-      document = RubyDocument.new(source: source, version: version, uri: uri, encoding: encoding)
+    sig do
+      params(
+        uri: URI::Generic,
+        source: String,
+        version: Integer,
+        language_id: Document::LanguageId,
+        encoding: Encoding,
+      ).void
+    end
+    def set(uri:, source:, version:, language_id:, encoding: Encoding::UTF_8)
+      document = case language_id
+      when Document::LanguageId::ERB
+        ERBDocument.new(source: source, version: version, uri: uri, encoding: encoding)
+      else
+        RubyDocument.new(source: source, version: version, uri: uri, encoding: encoding)
+      end
       @state[uri.to_s] = document
     end
 

lib/ruby_lsp/test_helper.rb

@@ -22,6 +22,7 @@ def with_server(source = nil, uri = Kernel.URI("file:///fake.rb"), stub_no_typec
       server = RubyLsp::Server.new(test_mode: true)
       server.global_state.stubs(:has_type_checker).returns(false) if stub_no_typechecker
       server.global_state.apply_options({})
+      language_id = uri.to_s.end_with?(".erb") ? "erb" : "ruby"
 
       if source
         server.process_message({
@@ -31,6 +32,7 @@ def with_server(source = nil, uri = Kernel.URI("file:///fake.rb"), stub_no_typec
               uri: uri,
               text: source,
               version: 1,
+              languageId: language_id,
             },
           },
         })