diff --git a/great_expectations/data_context/data_context/abstract_data_context.py b/great_expectations/data_context/data_context/abstract_data_context.py
index 2e4fa5d7cd66..42bac0d8aeef 100644
--- a/great_expectations/data_context/data_context/abstract_data_context.py
+++ b/great_expectations/data_context/data_context/abstract_data_context.py
@@ -3063,17 +3063,25 @@ def _instantiate_datasource_from_config_and_update_project_config(
         Raises:
             DatasourceInitializationError
         """
-        substituted_config = self._perform_substitutions_on_datasource_config(config)
+        # Note that the call to `DatasourcStore.set` may alter the config object's state
+        # As such, we invoke it at the top of our function so any changes are reflected downstream
+        if save_changes:
+            config = self._datasource_store.set(key=None, value=config)  # type: ignore[attr-defined]
 
         datasource: Optional[Datasource] = None
         if initialize:
-            datasource = self._instantiate_datasource_from_config(
-                raw_config=config, substituted_config=substituted_config
-            )
-            self._cached_datasources[config.name] = datasource
-
-        if save_changes:
-            config = self._datasource_store.set(key=None, value=config)  # type: ignore[attr-defined]
+            try:
+                substituted_config = self._perform_substitutions_on_datasource_config(
+                    config
+                )
+                datasource = self._instantiate_datasource_from_config(
+                    raw_config=config, substituted_config=substituted_config
+                )
+                self._cached_datasources[config.name] = datasource
+            except ge_exceptions.DatasourceInitializationError as e:
+                if save_changes:
+                    self._datasource_store.delete(config)  # type: ignore[attr-defined]
+                raise e
 
         self.config.datasources[config.name] = config  # type: ignore[index,assignment]