diff --git a/crates/bevy_text/src/pipeline.rs b/crates/bevy_text/src/pipeline.rs
index 0180cab7ce9829..b05cd87e07b908 100644
--- a/crates/bevy_text/src/pipeline.rs
+++ b/crates/bevy_text/src/pipeline.rs
@@ -72,10 +72,8 @@ impl Default for SwashCache {
 pub struct TextPipeline {
     /// Identifies a font [`ID`](cosmic_text::fontdb::ID) by its [`Font`] [`Asset`](bevy_asset::Asset) [`HandleId`].
     map_handle_to_font_id: HashMap<AssetId<Font>, cosmic_text::fontdb::ID>,
-    /// Identifies a [`FontAtlasSet`] handle by its font [`ID`](cosmic_text::fontdb::ID).
-    ///
-    /// Note that this is a strong handle, so that textures are not dropped.
-    map_font_id_to_handle: HashMap<cosmic_text::fontdb::ID, Handle<FontAtlasSet>>,
+    /// Identifies a [`FontAtlasSet`] by its font [`ID`](cosmic_text::fontdb::ID).
+    map_font_id_to_atlas: HashMap<cosmic_text::fontdb::ID, FontAtlasSet>,
     /// The font system is used to retrieve fonts and their information, including glyph outlines.
     ///
     /// See [`cosmic_text::FontSystem`] for more information.
@@ -283,8 +281,6 @@ impl TextPipeline {
             .map(|(layout_glyph, line_w, line_y)| {
                 let section_index = layout_glyph.metadata;
 
-                // TODO(totalkrill): this is probably very wrong, investigate the 
-                // cause, instead of "what makes it compile"
                 let font_atlas_set: &mut FontAtlasSet = match sections[section_index].style.font {
                     FontRef::Asset(ref font_handle) => {
                         let handle: Handle<Font> = font_handle.clone_weak();
@@ -303,12 +299,11 @@ impl TextPipeline {
                             weight: query.weight,
                             metadata: 0,
                         })[0];
-                        let handle = self
-                            .map_font_id_to_handle
+                        let atlas = self
+                            .map_font_id_to_atlas
                             .entry(font_id)
                             .or_default();
-
-                        font_atlas_sets.get_mut(handle.clone().untyped().id()).unwrap()
+                        atlas
                     }
                 };