Introduce indexables (#950)

Create Indexable as the unit of items to be indexed

lib/ruby_indexer/lib/ruby_indexer/configuration.rb

@@ -24,11 +24,11 @@ def initialize
 
       @excluded_gems = T.let(excluded_gem_names, T::Array[String])
       @included_gems = T.let([], T::Array[String])
-      @excluded_patterns = T.let(["**/*_test.rb"], T::Array[String])
+      @excluded_patterns = T.let([File.join("**", "*_test.rb")], T::Array[String])
       path = Bundler.settings["path"]
-      @excluded_patterns << "#{File.expand_path(path, Dir.pwd)}/**/*.rb" if path
+      @excluded_patterns << File.join(File.expand_path(path, Dir.pwd), "**", "*.rb") if path
 
-      @included_patterns = T.let(["#{Dir.pwd}/**/*.rb"], T::Array[String])
+      @included_patterns = T.let([File.join(Dir.pwd, "**", "*.rb")], T::Array[String])
       @excluded_magic_comments = T.let(
         [
           "frozen_string_literal:",
@@ -61,28 +61,31 @@ def load_config
       raise e, "Syntax error while loading .index.yml configuration: #{e.message}"
     end
 
-    sig { returns(T::Array[String]) }
-    def files_to_index
+    sig { returns(T::Array[IndexablePath]) }
+    def indexables
       excluded_gems = @excluded_gems - @included_gems
       locked_gems = Bundler.locked_gems&.specs
 
       # NOTE: indexing the patterns (both included and excluded) needs to happen before indexing gems, otherwise we risk
       # having duplicates if BUNDLE_PATH is set to a folder inside the project structure
 
       # Add user specified patterns
-      files_to_index = @included_patterns.flat_map do |pattern|
-        Dir.glob(pattern, File::FNM_PATHNAME | File::FNM_EXTGLOB)
+      indexables = @included_patterns.flat_map do |pattern|
+        Dir.glob(pattern, File::FNM_PATHNAME | File::FNM_EXTGLOB).map! do |path|
+          load_path_entry = $LOAD_PATH.find { |load_path| path.start_with?(load_path) }
+          IndexablePath.new(load_path_entry, path)
+        end
       end
 
       # Remove user specified patterns
-      files_to_index.reject! do |path|
+      indexables.reject! do |indexable|
         @excluded_patterns.any? do |pattern|
-          File.fnmatch?(pattern, path, File::FNM_PATHNAME | File::FNM_EXTGLOB)
+          File.fnmatch?(pattern, indexable.full_path, File::FNM_PATHNAME | File::FNM_EXTGLOB)
         end
       end
 
       # Add default gems to the list of files to be indexed
-      Dir.glob("#{RbConfig::CONFIG["rubylibdir"]}/*").each do |default_path|
+      Dir.glob(File.join(RbConfig::CONFIG["rubylibdir"], "*")).each do |default_path|
         # The default_path might be a Ruby file or a folder with the gem's name. For example:
         #   bundler/
         #   bundler.rb
@@ -103,10 +106,14 @@ def files_to_index
 
         if pathname.directory?
           # If the default_path is a directory, we index all the Ruby files in it
-          files_to_index.concat(Dir.glob("#{default_path}/**/*.rb", File::FNM_PATHNAME | File::FNM_EXTGLOB))
+          indexables.concat(
+            Dir.glob(File.join(default_path, "**", "*.rb"), File::FNM_PATHNAME | File::FNM_EXTGLOB).map! do |path|
+              IndexablePath.new(RbConfig::CONFIG["rubylibdir"], path)
+            end,
+          )
         else
           # If the default_path is a Ruby file, we index it
-          files_to_index << default_path
+          indexables << IndexablePath.new(RbConfig::CONFIG["rubylibdir"], default_path)
         end
       end
 
@@ -121,15 +128,20 @@ def files_to_index
         # duplicates or accidentally ignoring exclude patterns
         next if spec.full_gem_path == Dir.pwd
 
-        files_to_index.concat(Dir.glob("#{spec.full_gem_path}/{#{spec.require_paths.join(",")}}/**/*.rb"))
+        indexables.concat(
+          spec.require_paths.flat_map do |require_path|
+            load_path_entry = File.join(spec.full_gem_path, require_path)
+            Dir.glob(File.join(load_path_entry, "**", "*.rb")).map! { |path| IndexablePath.new(load_path_entry, path) }
+          end,
+        )
       rescue Gem::MissingSpecError
         # If a gem is scoped only to some specific platform, then its dependencies may not be installed either, but they
         # are still listed in locked_gems. We can't index them because they are not installed for the platform, so we
         # just ignore if they're missing
       end
 
-      files_to_index.uniq!
-      files_to_index
+      indexables.uniq!
+      indexables
     end
 
     sig { returns(Regexp) }

lib/ruby_indexer/lib/ruby_indexer/index.rb

@@ -25,11 +25,11 @@ def initialize
       @files_to_entries = T.let({}, T::Hash[String, T::Array[Entry]])
     end
 
-    sig { params(path: String).void }
-    def delete(path)
+    sig { params(indexable: IndexablePath).void }
+    def delete(indexable)
       # For each constant discovered in `path`, delete the associated entry from the index. If there are no entries
       # left, delete the constant from the index.
-      @files_to_entries[path]&.each do |entry|
+      @files_to_entries[indexable.full_path]&.each do |entry|
         entries = @entries[entry.name]
         next unless entries
 
@@ -39,7 +39,7 @@ def delete(path)
         @entries.delete(entry.name) if entries.empty?
       end
 
-      @files_to_entries.delete(path)
+      @files_to_entries.delete(indexable.full_path)
     end
 
     sig { params(entry: Entry).void }
@@ -85,15 +85,15 @@ def resolve(name, nesting)
       nil
     end
 
-    sig { params(paths: T::Array[String]).void }
-    def index_all(paths: RubyIndexer.configuration.files_to_index)
-      paths.each { |path| index_single(path) }
+    sig { params(indexable_paths: T::Array[IndexablePath]).void }
+    def index_all(indexable_paths: RubyIndexer.configuration.indexables)
+      indexable_paths.each { |path| index_single(path) }
     end
 
-    sig { params(path: String, source: T.nilable(String)).void }
-    def index_single(path, source = nil)
-      content = source || File.read(path)
-      visitor = IndexVisitor.new(self, YARP.parse(content), path)
+    sig { params(indexable_path: IndexablePath, source: T.nilable(String)).void }
+    def index_single(indexable_path, source = nil)
+      content = source || File.read(indexable_path.full_path)
+      visitor = IndexVisitor.new(self, YARP.parse(content), indexable_path.full_path)
       visitor.run
     rescue Errno::EISDIR
       # If `path` is a directory, just ignore it and continue indexing

lib/ruby_indexer/lib/ruby_indexer/indexable_path.rb

@@ -0,0 +1,29 @@
+# typed: strict
+# frozen_string_literal: true
+
+module RubyIndexer
+  class IndexablePath
+    extend T::Sig
+
+    sig { returns(T.nilable(String)) }
+    attr_reader :require_path
+
+    sig { returns(String) }
+    attr_reader :full_path
+
+    # An IndexablePath is instantiated with a load_path_entry and a full_path. The load_path_entry is where the file can
+    # be found in the $LOAD_PATH, which we use to determine the require_path. The load_path_entry may be `nil` if the
+    # indexer is configured to go through files that do not belong in the $LOAD_PATH. For example,
+    # `sorbet/tapioca/require.rb` ends up being a part of the paths to be indexed because it's a Ruby file inside the
+    # project, but the `sorbet` folder is not a part of the $LOAD_PATH. That means that both its load_path_entry and
+    # require_path will be `nil`, since it cannot be required by the project
+    sig { params(load_path_entry: T.nilable(String), full_path: String).void }
+    def initialize(load_path_entry, full_path)
+      @full_path = full_path
+      @require_path = T.let(
+        load_path_entry ? Pathname.new(full_path).relative_path_from(load_path_entry).to_s.delete_suffix(".rb") : nil,
+        T.nilable(String),
+      )
+    end
+  end
+end

lib/ruby_indexer/ruby_indexer.rb

@@ -4,6 +4,7 @@
 require "yaml"
 require "did_you_mean"
 
+require "ruby_indexer/lib/ruby_indexer/indexable_path"
 require "ruby_indexer/lib/ruby_indexer/visitor"
 require "ruby_indexer/lib/ruby_indexer/index"
 require "ruby_indexer/lib/ruby_indexer/configuration"

lib/ruby_indexer/test/classes_and_modules_test.rb

@@ -139,7 +139,7 @@ class Foo
 
       assert_entry("Foo", Index::Entry::Class, "/fake/path/foo.rb:0-0:1-2")
 
-      @index.delete("/fake/path/foo.rb")
+      @index.delete(IndexablePath.new(nil, "/fake/path/foo.rb"))
       refute_entry("Foo")
       assert_empty(@index.instance_variable_get(:@files_to_entries))
     end

lib/ruby_indexer/test/configuration_test.rb

@@ -11,55 +11,57 @@ def setup
 
     def test_load_configuration_executes_configure_block
       @config.load_config
-      files_to_index = @config.files_to_index
+      indexables = @config.indexables
 
-      assert(files_to_index.none? { |path| path.include?("test/fixtures") })
-      assert(files_to_index.none? { |path| path.include?("minitest-reporters") })
-      assert(files_to_index.none? { |path| path == __FILE__ })
+      assert(indexables.none? { |indexable| indexable.full_path.include?("test/fixtures") })
+      assert(indexables.none? { |indexable| indexable.full_path.include?("minitest-reporters") })
+      assert(indexables.none? { |indexable| indexable.full_path == __FILE__ })
     end
 
-    def test_files_to_index_only_includes_gem_require_paths
+    def test_indexables_only_includes_gem_require_paths
       @config.load_config
-      files_to_index = @config.files_to_index
+      indexables = @config.indexables
 
       Bundler.locked_gems.specs.each do |lazy_spec|
         next if lazy_spec.name == "ruby-lsp"
 
         spec = Gem::Specification.find_by_name(lazy_spec.name)
-        assert(files_to_index.none? { |path| path.start_with?("#{spec.full_gem_path}/test/") })
+        assert(indexables.none? { |indexable| indexable.full_path.start_with?("#{spec.full_gem_path}/test/") })
       rescue Gem::MissingSpecError
         # Transitive dependencies might be missing when running tests on Windows
       end
     end
 
-    def test_files_to_index_does_not_include_default_gem_path_when_in_bundle
+    def test_indexables_does_not_include_default_gem_path_when_in_bundle
       @config.load_config
-      files_to_index = @config.files_to_index
+      indexables = @config.indexables
 
-      assert(files_to_index.none? { |path| path.start_with?("#{RbConfig::CONFIG["rubylibdir"]}/psych") })
+      assert(
+        indexables.none? { |indexable| indexable.full_path.start_with?("#{RbConfig::CONFIG["rubylibdir"]}/psych") },
+      )
     end
 
-    def test_files_to_index_includes_default_gems
+    def test_indexables_includes_default_gems
       @config.load_config
-      files_to_index = @config.files_to_index
+      indexables = @config.indexables.map(&:full_path)
 
-      assert_includes(files_to_index, "#{RbConfig::CONFIG["rubylibdir"]}/pathname.rb")
-      assert_includes(files_to_index, "#{RbConfig::CONFIG["rubylibdir"]}/ipaddr.rb")
-      assert_includes(files_to_index, "#{RbConfig::CONFIG["rubylibdir"]}/abbrev.rb")
+      assert_includes(indexables, "#{RbConfig::CONFIG["rubylibdir"]}/pathname.rb")
+      assert_includes(indexables, "#{RbConfig::CONFIG["rubylibdir"]}/ipaddr.rb")
+      assert_includes(indexables, "#{RbConfig::CONFIG["rubylibdir"]}/abbrev.rb")
     end
 
-    def test_files_to_index_includes_project_files
+    def test_indexables_includes_project_files
       @config.load_config
-      files_to_index = @config.files_to_index
+      indexables = @config.indexables.map(&:full_path)
 
       Dir.glob("#{Dir.pwd}/lib/**/*.rb").each do |path|
         next if path.end_with?("_test.rb")
 
-        assert_includes(files_to_index, path)
+        assert_includes(indexables, path)
       end
     end
 
-    def test_files_to_index_avoids_duplicates_if_bundle_path_is_inside_project
+    def test_indexables_avoids_duplicates_if_bundle_path_is_inside_project
       Bundler.settings.set_global("path", "vendor/bundle")
       config = Configuration.new
       config.load_config
@@ -69,18 +71,22 @@ def test_files_to_index_avoids_duplicates_if_bundle_path_is_inside_project
       Bundler.settings.set_global("path", nil)
     end
 
-    def test_files_to_index_does_not_include_gems_own_installed_files
+    def test_indexables_does_not_include_gems_own_installed_files
       @config.load_config
-      files_to_index = @config.files_to_index
+      indexables = @config.indexables
 
-      assert(files_to_index.none? { |path| path.start_with?(Bundler.bundle_path.join("gems", "ruby-lsp").to_s) })
+      assert(
+        indexables.none? do |indexable|
+          indexable.full_path.start_with?(Bundler.bundle_path.join("gems", "ruby-lsp").to_s)
+        end,
+      )
     end
 
     def test_paths_are_unique
       @config.load_config
-      files_to_index = @config.files_to_index
+      indexables = @config.indexables
 
-      assert_equal(files_to_index.uniq.length, files_to_index.length)
+      assert_equal(indexables.uniq.length, indexables.length)
     end
 
     def test_configuration_raises_for_unknown_keys

lib/ruby_indexer/test/index_test.rb

@@ -6,39 +6,39 @@
 module RubyIndexer
   class IndexTest < TestCase
     def test_deleting_one_entry_for_a_class
-      @index.index_single("/fake/path/foo.rb", <<~RUBY)
+      @index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
         class Foo
         end
       RUBY
-      @index.index_single("/fake/path/other_foo.rb", <<~RUBY)
+      @index.index_single(IndexablePath.new(nil, "/fake/path/other_foo.rb"), <<~RUBY)
         class Foo
         end
       RUBY
 
       entries = @index["Foo"]
       assert_equal(2, entries.length)
 
-      @index.delete("/fake/path/other_foo.rb")
+      @index.delete(IndexablePath.new(nil, "/fake/path/other_foo.rb"))
       entries = @index["Foo"]
       assert_equal(1, entries.length)
     end
 
     def test_deleting_all_entries_for_a_class
-      @index.index_single("/fake/path/foo.rb", <<~RUBY)
+      @index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
         class Foo
         end
       RUBY
 
       entries = @index["Foo"]
       assert_equal(1, entries.length)
 
-      @index.delete("/fake/path/foo.rb")
+      @index.delete(IndexablePath.new(nil, "/fake/path/foo.rb"))
       entries = @index["Foo"]
       assert_nil(entries)
     end
 
     def test_index_resolve
-      @index.index_single("/fake/path/foo.rb", <<~RUBY)
+      @index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
         class Bar; end
 
         module Foo
@@ -72,7 +72,7 @@ class Something
     end
 
     def test_accessing_with_colon_colon_prefix
-      @index.index_single("/fake/path/foo.rb", <<~RUBY)
+      @index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
         class Bar; end
 
         module Foo
@@ -92,7 +92,7 @@ class Something
     end
 
     def test_fuzzy_search
-      @index.index_single("/fake/path/foo.rb", <<~RUBY)
+      @index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
         class Bar; end
 
         module Foo
@@ -121,7 +121,7 @@ class Something
 
     def test_index_single_ignores_directories
       FileUtils.mkdir("lib/this_is_a_dir.rb")
-      @index.index_single("lib/this_is_a_dir.rb")
+      @index.index_single(IndexablePath.new(nil, "lib/this_is_a_dir.rb"))
     ensure
       FileUtils.rm_r("lib/this_is_a_dir.rb")
     end

lib/ruby_indexer/test/test_case.rb

@@ -12,7 +12,7 @@ def setup
     private
 
     def index(source)
-      @index.index_single("/fake/path/foo.rb", source)
+      @index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), source)
     end
 
     def assert_entry(expected_name, type, expected_location)