diff --git a/lib/irb.rb b/lib/irb.rb
index 5a064cfce..b3435c257 100644
--- a/lib/irb.rb
+++ b/lib/irb.rb
@@ -931,8 +931,11 @@ class Irb
     # The lexer used by this irb session
     attr_accessor :scanner
 
+    attr_reader :from_binding
+
     # Creates a new irb session
-    def initialize(workspace = nil, input_method = nil)
+    def initialize(workspace = nil, input_method = nil, from_binding: false)
+      @from_binding = from_binding
       @context = Context.new(self, workspace, input_method)
       @context.workspace.load_helper_methods_to_main
       @signal_status = :IN_IRB
@@ -1596,7 +1599,7 @@ def irb(show_code: true)
     else
       # If we're not in a debugger session, create a new IRB instance with the current
       # workspace
-      binding_irb = IRB::Irb.new(workspace)
+      binding_irb = IRB::Irb.new(workspace, from_binding: true)
       binding_irb.context.irb_path = irb_path
       binding_irb.run(IRB.conf)
       binding_irb.debug_break
diff --git a/lib/irb/command/debug.rb b/lib/irb/command/debug.rb
index f9aca0a67..8a091a49e 100644
--- a/lib/irb/command/debug.rb
+++ b/lib/irb/command/debug.rb
@@ -8,11 +8,6 @@ class Debug < Base
       category "Debugging"
       description "Start the debugger of debug.gem."
 
-      BINDING_IRB_FRAME_REGEXPS = [
-        '<internal:prelude>',
-        binding.method(:irb).source_location.first,
-      ].map { |file| /\A#{Regexp.escape(file)}:\d+:in (`|'Binding#)irb'\z/ }
-
       def execute(_arg)
         execute_debug_command
       end
@@ -36,7 +31,7 @@ def execute_debug_command(pre_cmds: nil, do_cmds: nil)
           # 3. Insert a debug breakpoint at `Irb#debug_break` with the intended command.
           # 4. Exit the current Irb#run call via `throw :IRB_EXIT`.
           # 5. `Irb#debug_break` will be called and trigger the breakpoint, which will run the intended command.
-          unless binding_irb?
+          unless irb_context.from_binding?
             puts "Debugging commands are only available when IRB is started with binding.irb"
             return
           end
@@ -60,16 +55,6 @@ def execute_debug_command(pre_cmds: nil, do_cmds: nil)
           throw :IRB_EXIT
         end
       end
-
-      private
-
-      def binding_irb?
-        caller.any? do |frame|
-          BINDING_IRB_FRAME_REGEXPS.any? do |regexp|
-            frame.match?(regexp)
-          end
-        end
-      end
     end
 
     class DebugCommand < Debug
diff --git a/lib/irb/context.rb b/lib/irb/context.rb
index 173f3dae5..5d2ff9732 100644
--- a/lib/irb/context.rb
+++ b/lib/irb/context.rb
@@ -607,6 +607,10 @@ def evaluate(statement, line_no) # :nodoc:
       nil
     end
 
+    def from_binding?
+      @irb.from_binding
+    end
+
     def evaluate_expression(code, line_no) # :nodoc:
       result = nil
       if IRB.conf[:MEASURE] && IRB.conf[:MEASURE_CALLBACKS].empty?
diff --git a/test/irb/test_debugger_integration.rb b/test/irb/test_debugger_integration.rb
index eca40c570..8b1bddea1 100644
--- a/test/irb/test_debugger_integration.rb
+++ b/test/irb/test_debugger_integration.rb
@@ -67,6 +67,22 @@ def test_debug_command_only_runs_once
       assert_match(/IRB is already running with a debug session/, output)
     end
 
+    def test_debug_command_can_only_be_called_from_binding_irb
+      write_ruby <<~'ruby'
+        require "irb"
+        # trick test framework
+        puts "binding.irb"
+        IRB.start
+      ruby
+
+      output = run_ruby_file do
+        type "debug"
+        type "exit"
+      end
+
+      assert_include(output, "Debugging commands are only available when IRB is started with binding.irb")
+    end
+
     def test_next
       write_ruby <<~'ruby'
         binding.irb