diff --git a/parinfer-rust-helper.el b/parinfer-rust-helper.el
index e2f0e34..b9abc0d 100644
--- a/parinfer-rust-helper.el
+++ b/parinfer-rust-helper.el
@@ -238,8 +238,10 @@ mode to better emulate users."
           (t num))))
 (defun parinfer-rust--defer-loading (&rest _)
   "Defer loading of `parinfer-rust-mode' until the buffer is in focus."
-  (when (eq (current-buffer)
-            (window-buffer (selected-window)))
+  ;; This is a parinfer enabled buffer that started in the background and has now been moved to the foreground
+  (when (and parinfer-rust-enabled
+             (eq (current-buffer)
+                 (window-buffer (selected-window))))
     (remove-hook 'window-selection-change-functions #'parinfer-rust--defer-loading t)
     (parinfer-rust-mode-enable)))
 ;; Disable fill column warning only for this buffer to enable long strings of text without
diff --git a/parinfer-rust-mode.el b/parinfer-rust-mode.el
index eacea6f..7e83089 100644
--- a/parinfer-rust-mode.el
+++ b/parinfer-rust-mode.el
@@ -606,8 +606,12 @@ not available."
    (parinfer-rust-enabled
     (parinfer-rust-mode-disable))
    ;; Don't do anything if the buffer is not selected
+   ;; TODO: Come up with a better way to defer and disable loading
+   ;; Defer waits for window selection change and disabled waits for a change event
+   ;; there is also the idea of deferring the running of parinfer vs deferring the loading
    ((not (eq (current-buffer)
              (window-buffer (selected-window))))
+    (setq-local parinfer-rust-enabled t)
     (add-hook 'window-selection-change-functions #'parinfer-rust--defer-loading nil t))
    (t
     (parinfer-rust-mode-enable))))