diff --git a/lib/ruby_lsp/setup_bundler.rb b/lib/ruby_lsp/setup_bundler.rb
index af354f9a20..903f43a3d7 100644
--- a/lib/ruby_lsp/setup_bundler.rb
+++ b/lib/ruby_lsp/setup_bundler.rb
@@ -285,13 +285,11 @@ def correct_relative_remote_paths
     # Detects if the project is a Rails app by looking if the superclass of the main class is `Rails::Application`
     sig { returns(T::Boolean) }
     def rails_app?
-      /class .* < Rails::Application/.match?(rails_application_content)
-    end
-
-    sig { returns(T.nilable(String)) }
-    def rails_application_content
       config = Pathname.new("config/application.rb").expand_path
-      config.read if config.exist?
+      application_contents = config.read if config.exist?
+      return false unless application_contents
+
+      /class .* < Rails::Application/.match?(application_contents)
     end
   end
 end
diff --git a/test/fixtures/rails_application.rb b/test/fixtures/rails_application.rb
new file mode 100644
index 0000000000..27eff561a4
--- /dev/null
+++ b/test/fixtures/rails_application.rb
@@ -0,0 +1,4 @@
+module MyApp
+  class Application < Rails::Application
+  end
+end
diff --git a/test/setup_bundler_test.rb b/test/setup_bundler_test.rb
index a65d7b3cad..e511c4bf35 100644
--- a/test/setup_bundler_test.rb
+++ b/test/setup_bundler_test.rb
@@ -82,12 +82,14 @@ def test_creates_custom_bundle_for_a_rails_app
 
     # There is some unknown state leak with Bundler which prevents us from creating the
     # folder structure ourselves lest we encounter flaky tests
-    RubyLsp::SetupBundler.any_instance.stubs(:rails_application_content).returns(<<~RUBY)
-      module MyApp
-        class Application < Rails::Application
-        end
-      end
-    RUBY
+    # RubyLsp::SetupBundler.any_instance.stubs(:rails_application_content).returns(<<~RUBY)
+    #   module MyApp
+    #     class Application < Rails::Application
+    #     end
+    #   end
+    # RUBY
+    FileUtils.mkdir("config")
+    FileUtils.cp("test/fixtures/rails_application.rb", "config/application.rb")
 
     Bundler::LockfileParser.any_instance.expects(:dependencies).returns({ "rails" => true }).at_least_once
 
@@ -102,6 +104,7 @@ class Application < Rails::Application
     assert_match("ruby-lsp-rails", File.read(".ruby-lsp/Gemfile"))
   ensure
     FileUtils.rm_r(".ruby-lsp") if Dir.exist?(".ruby-lsp")
+    FileUtils.rm_rf("config") if Dir.exist?("config")
   end
 
   def test_changing_lockfile_causes_custom_bundle_to_be_rebuilt