Skip to content

Commit

Permalink
Introduce indexables (#950)
Browse files Browse the repository at this point in the history
Create Indexable as the unit of items to be indexed
  • Loading branch information
vinistock authored Sep 1, 2023
1 parent c706a5b commit 3c39ada
Show file tree
Hide file tree
Showing 13 changed files with 150 additions and 80 deletions.
42 changes: 27 additions & 15 deletions lib/ruby_indexer/lib/ruby_indexer/configuration.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ def initialize

@excluded_gems = T.let(excluded_gem_names, T::Array[String])
@included_gems = T.let([], T::Array[String])
@excluded_patterns = T.let(["**/*_test.rb"], T::Array[String])
@excluded_patterns = T.let([File.join("**", "*_test.rb")], T::Array[String])
path = Bundler.settings["path"]
@excluded_patterns << "#{File.expand_path(path, Dir.pwd)}/**/*.rb" if path
@excluded_patterns << File.join(File.expand_path(path, Dir.pwd), "**", "*.rb") if path

@included_patterns = T.let(["#{Dir.pwd}/**/*.rb"], T::Array[String])
@included_patterns = T.let([File.join(Dir.pwd, "**", "*.rb")], T::Array[String])
@excluded_magic_comments = T.let(
[
"frozen_string_literal:",
Expand Down Expand Up @@ -61,28 +61,31 @@ def load_config
raise e, "Syntax error while loading .index.yml configuration: #{e.message}"
end

sig { returns(T::Array[String]) }
def files_to_index
sig { returns(T::Array[IndexablePath]) }
def indexables
excluded_gems = @excluded_gems - @included_gems
locked_gems = Bundler.locked_gems&.specs

# NOTE: indexing the patterns (both included and excluded) needs to happen before indexing gems, otherwise we risk
# having duplicates if BUNDLE_PATH is set to a folder inside the project structure

# Add user specified patterns
files_to_index = @included_patterns.flat_map do |pattern|
Dir.glob(pattern, File::FNM_PATHNAME | File::FNM_EXTGLOB)
indexables = @included_patterns.flat_map do |pattern|
Dir.glob(pattern, File::FNM_PATHNAME | File::FNM_EXTGLOB).map! do |path|
load_path_entry = $LOAD_PATH.find { |load_path| path.start_with?(load_path) }
IndexablePath.new(load_path_entry, path)
end
end

# Remove user specified patterns
files_to_index.reject! do |path|
indexables.reject! do |indexable|
@excluded_patterns.any? do |pattern|
File.fnmatch?(pattern, path, File::FNM_PATHNAME | File::FNM_EXTGLOB)
File.fnmatch?(pattern, indexable.full_path, File::FNM_PATHNAME | File::FNM_EXTGLOB)
end
end

# Add default gems to the list of files to be indexed
Dir.glob("#{RbConfig::CONFIG["rubylibdir"]}/*").each do |default_path|
Dir.glob(File.join(RbConfig::CONFIG["rubylibdir"], "*")).each do |default_path|
# The default_path might be a Ruby file or a folder with the gem's name. For example:
# bundler/
# bundler.rb
Expand All @@ -103,10 +106,14 @@ def files_to_index

if pathname.directory?
# If the default_path is a directory, we index all the Ruby files in it
files_to_index.concat(Dir.glob("#{default_path}/**/*.rb", File::FNM_PATHNAME | File::FNM_EXTGLOB))
indexables.concat(
Dir.glob(File.join(default_path, "**", "*.rb"), File::FNM_PATHNAME | File::FNM_EXTGLOB).map! do |path|
IndexablePath.new(RbConfig::CONFIG["rubylibdir"], path)
end,
)
else
# If the default_path is a Ruby file, we index it
files_to_index << default_path
indexables << IndexablePath.new(RbConfig::CONFIG["rubylibdir"], default_path)
end
end

Expand All @@ -121,15 +128,20 @@ def files_to_index
# duplicates or accidentally ignoring exclude patterns
next if spec.full_gem_path == Dir.pwd

files_to_index.concat(Dir.glob("#{spec.full_gem_path}/{#{spec.require_paths.join(",")}}/**/*.rb"))
indexables.concat(
spec.require_paths.flat_map do |require_path|
load_path_entry = File.join(spec.full_gem_path, require_path)
Dir.glob(File.join(load_path_entry, "**", "*.rb")).map! { |path| IndexablePath.new(load_path_entry, path) }
end,
)
rescue Gem::MissingSpecError
# If a gem is scoped only to some specific platform, then its dependencies may not be installed either, but they
# are still listed in locked_gems. We can't index them because they are not installed for the platform, so we
# just ignore if they're missing
end

files_to_index.uniq!
files_to_index
indexables.uniq!
indexables
end

sig { returns(Regexp) }
Expand Down
22 changes: 11 additions & 11 deletions lib/ruby_indexer/lib/ruby_indexer/index.rb
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ def initialize
@files_to_entries = T.let({}, T::Hash[String, T::Array[Entry]])
end

sig { params(path: String).void }
def delete(path)
sig { params(indexable: IndexablePath).void }
def delete(indexable)
# For each constant discovered in `path`, delete the associated entry from the index. If there are no entries
# left, delete the constant from the index.
@files_to_entries[path]&.each do |entry|
@files_to_entries[indexable.full_path]&.each do |entry|
entries = @entries[entry.name]
next unless entries

Expand All @@ -39,7 +39,7 @@ def delete(path)
@entries.delete(entry.name) if entries.empty?
end

@files_to_entries.delete(path)
@files_to_entries.delete(indexable.full_path)
end

sig { params(entry: Entry).void }
Expand Down Expand Up @@ -85,15 +85,15 @@ def resolve(name, nesting)
nil
end

sig { params(paths: T::Array[String]).void }
def index_all(paths: RubyIndexer.configuration.files_to_index)
paths.each { |path| index_single(path) }
sig { params(indexable_paths: T::Array[IndexablePath]).void }
def index_all(indexable_paths: RubyIndexer.configuration.indexables)
indexable_paths.each { |path| index_single(path) }
end

sig { params(path: String, source: T.nilable(String)).void }
def index_single(path, source = nil)
content = source || File.read(path)
visitor = IndexVisitor.new(self, YARP.parse(content), path)
sig { params(indexable_path: IndexablePath, source: T.nilable(String)).void }
def index_single(indexable_path, source = nil)
content = source || File.read(indexable_path.full_path)
visitor = IndexVisitor.new(self, YARP.parse(content), indexable_path.full_path)
visitor.run
rescue Errno::EISDIR
# If `path` is a directory, just ignore it and continue indexing
Expand Down
29 changes: 29 additions & 0 deletions lib/ruby_indexer/lib/ruby_indexer/indexable_path.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# typed: strict
# frozen_string_literal: true

module RubyIndexer
class IndexablePath
extend T::Sig

sig { returns(T.nilable(String)) }
attr_reader :require_path

sig { returns(String) }
attr_reader :full_path

# An IndexablePath is instantiated with a load_path_entry and a full_path. The load_path_entry is where the file can
# be found in the $LOAD_PATH, which we use to determine the require_path. The load_path_entry may be `nil` if the
# indexer is configured to go through files that do not belong in the $LOAD_PATH. For example,
# `sorbet/tapioca/require.rb` ends up being a part of the paths to be indexed because it's a Ruby file inside the
# project, but the `sorbet` folder is not a part of the $LOAD_PATH. That means that both its load_path_entry and
# require_path will be `nil`, since it cannot be required by the project
sig { params(load_path_entry: T.nilable(String), full_path: String).void }
def initialize(load_path_entry, full_path)
@full_path = full_path
@require_path = T.let(
load_path_entry ? Pathname.new(full_path).relative_path_from(load_path_entry).to_s.delete_suffix(".rb") : nil,
T.nilable(String),
)
end
end
end
1 change: 1 addition & 0 deletions lib/ruby_indexer/ruby_indexer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
require "yaml"
require "did_you_mean"

require "ruby_indexer/lib/ruby_indexer/indexable_path"
require "ruby_indexer/lib/ruby_indexer/visitor"
require "ruby_indexer/lib/ruby_indexer/index"
require "ruby_indexer/lib/ruby_indexer/configuration"
Expand Down
2 changes: 1 addition & 1 deletion lib/ruby_indexer/test/classes_and_modules_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ class Foo

assert_entry("Foo", Index::Entry::Class, "/fake/path/foo.rb:0-0:1-2")

@index.delete("/fake/path/foo.rb")
@index.delete(IndexablePath.new(nil, "/fake/path/foo.rb"))
refute_entry("Foo")
assert_empty(@index.instance_variable_get(:@files_to_entries))
end
Expand Down
54 changes: 30 additions & 24 deletions lib/ruby_indexer/test/configuration_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,55 +11,57 @@ def setup

def test_load_configuration_executes_configure_block
@config.load_config
files_to_index = @config.files_to_index
indexables = @config.indexables

assert(files_to_index.none? { |path| path.include?("test/fixtures") })
assert(files_to_index.none? { |path| path.include?("minitest-reporters") })
assert(files_to_index.none? { |path| path == __FILE__ })
assert(indexables.none? { |indexable| indexable.full_path.include?("test/fixtures") })
assert(indexables.none? { |indexable| indexable.full_path.include?("minitest-reporters") })
assert(indexables.none? { |indexable| indexable.full_path == __FILE__ })
end

def test_files_to_index_only_includes_gem_require_paths
def test_indexables_only_includes_gem_require_paths
@config.load_config
files_to_index = @config.files_to_index
indexables = @config.indexables

Bundler.locked_gems.specs.each do |lazy_spec|
next if lazy_spec.name == "ruby-lsp"

spec = Gem::Specification.find_by_name(lazy_spec.name)
assert(files_to_index.none? { |path| path.start_with?("#{spec.full_gem_path}/test/") })
assert(indexables.none? { |indexable| indexable.full_path.start_with?("#{spec.full_gem_path}/test/") })
rescue Gem::MissingSpecError
# Transitive dependencies might be missing when running tests on Windows
end
end

def test_files_to_index_does_not_include_default_gem_path_when_in_bundle
def test_indexables_does_not_include_default_gem_path_when_in_bundle
@config.load_config
files_to_index = @config.files_to_index
indexables = @config.indexables

assert(files_to_index.none? { |path| path.start_with?("#{RbConfig::CONFIG["rubylibdir"]}/psych") })
assert(
indexables.none? { |indexable| indexable.full_path.start_with?("#{RbConfig::CONFIG["rubylibdir"]}/psych") },
)
end

def test_files_to_index_includes_default_gems
def test_indexables_includes_default_gems
@config.load_config
files_to_index = @config.files_to_index
indexables = @config.indexables.map(&:full_path)

assert_includes(files_to_index, "#{RbConfig::CONFIG["rubylibdir"]}/pathname.rb")
assert_includes(files_to_index, "#{RbConfig::CONFIG["rubylibdir"]}/ipaddr.rb")
assert_includes(files_to_index, "#{RbConfig::CONFIG["rubylibdir"]}/abbrev.rb")
assert_includes(indexables, "#{RbConfig::CONFIG["rubylibdir"]}/pathname.rb")
assert_includes(indexables, "#{RbConfig::CONFIG["rubylibdir"]}/ipaddr.rb")
assert_includes(indexables, "#{RbConfig::CONFIG["rubylibdir"]}/abbrev.rb")
end

def test_files_to_index_includes_project_files
def test_indexables_includes_project_files
@config.load_config
files_to_index = @config.files_to_index
indexables = @config.indexables.map(&:full_path)

Dir.glob("#{Dir.pwd}/lib/**/*.rb").each do |path|
next if path.end_with?("_test.rb")

assert_includes(files_to_index, path)
assert_includes(indexables, path)
end
end

def test_files_to_index_avoids_duplicates_if_bundle_path_is_inside_project
def test_indexables_avoids_duplicates_if_bundle_path_is_inside_project
Bundler.settings.set_global("path", "vendor/bundle")
config = Configuration.new
config.load_config
Expand All @@ -69,18 +71,22 @@ def test_files_to_index_avoids_duplicates_if_bundle_path_is_inside_project
Bundler.settings.set_global("path", nil)
end

def test_files_to_index_does_not_include_gems_own_installed_files
def test_indexables_does_not_include_gems_own_installed_files
@config.load_config
files_to_index = @config.files_to_index
indexables = @config.indexables

assert(files_to_index.none? { |path| path.start_with?(Bundler.bundle_path.join("gems", "ruby-lsp").to_s) })
assert(
indexables.none? do |indexable|
indexable.full_path.start_with?(Bundler.bundle_path.join("gems", "ruby-lsp").to_s)
end,
)
end

def test_paths_are_unique
@config.load_config
files_to_index = @config.files_to_index
indexables = @config.indexables

assert_equal(files_to_index.uniq.length, files_to_index.length)
assert_equal(indexables.uniq.length, indexables.length)
end

def test_configuration_raises_for_unknown_keys
Expand Down
18 changes: 9 additions & 9 deletions lib/ruby_indexer/test/index_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,39 +6,39 @@
module RubyIndexer
class IndexTest < TestCase
def test_deleting_one_entry_for_a_class
@index.index_single("/fake/path/foo.rb", <<~RUBY)
@index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
class Foo
end
RUBY
@index.index_single("/fake/path/other_foo.rb", <<~RUBY)
@index.index_single(IndexablePath.new(nil, "/fake/path/other_foo.rb"), <<~RUBY)
class Foo
end
RUBY

entries = @index["Foo"]
assert_equal(2, entries.length)

@index.delete("/fake/path/other_foo.rb")
@index.delete(IndexablePath.new(nil, "/fake/path/other_foo.rb"))
entries = @index["Foo"]
assert_equal(1, entries.length)
end

def test_deleting_all_entries_for_a_class
@index.index_single("/fake/path/foo.rb", <<~RUBY)
@index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
class Foo
end
RUBY

entries = @index["Foo"]
assert_equal(1, entries.length)

@index.delete("/fake/path/foo.rb")
@index.delete(IndexablePath.new(nil, "/fake/path/foo.rb"))
entries = @index["Foo"]
assert_nil(entries)
end

def test_index_resolve
@index.index_single("/fake/path/foo.rb", <<~RUBY)
@index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
class Bar; end
module Foo
Expand Down Expand Up @@ -72,7 +72,7 @@ class Something
end

def test_accessing_with_colon_colon_prefix
@index.index_single("/fake/path/foo.rb", <<~RUBY)
@index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
class Bar; end
module Foo
Expand All @@ -92,7 +92,7 @@ class Something
end

def test_fuzzy_search
@index.index_single("/fake/path/foo.rb", <<~RUBY)
@index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), <<~RUBY)
class Bar; end
module Foo
Expand Down Expand Up @@ -121,7 +121,7 @@ class Something

def test_index_single_ignores_directories
FileUtils.mkdir("lib/this_is_a_dir.rb")
@index.index_single("lib/this_is_a_dir.rb")
@index.index_single(IndexablePath.new(nil, "lib/this_is_a_dir.rb"))
ensure
FileUtils.rm_r("lib/this_is_a_dir.rb")
end
Expand Down
2 changes: 1 addition & 1 deletion lib/ruby_indexer/test/test_case.rb
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def setup
private

def index(source)
@index.index_single("/fake/path/foo.rb", source)
@index.index_single(IndexablePath.new(nil, "/fake/path/foo.rb"), source)
end

def assert_entry(expected_name, type, expected_location)
Expand Down
Loading

0 comments on commit 3c39ada

Please sign in to comment.