corealloc: operate ring-at-a-time on remote queues

microsoft · Dec 13, 2023 · 2b14a78 · 2b14a78
1 parent aeecad8
commit 2b14a78
Showing 1 changed file with 58 additions and 13 deletions.
diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h
@@ -557,26 +557,36 @@ namespace snmalloc
 
       if (SNMALLOC_LIKELY(entry.get_remote() == public_state()))
       {
-        auto key_tweak = entry.get_slab_metadata()->as_key_tweak();
-        freelist::HeadPtr tail = RemoteMessage::to_free_ring(msg);
-        freelist::HeadPtr curr = tail;
-        do
+        auto unreturned =
+          dealloc_local_objects_fast(entry, msg, entropy, domesticate);
+        if (SNMALLOC_UNLIKELY(unreturned.needs_work()))
         {
-          auto next =
-            curr->read_next(freelist::Object::key_root, key_tweak, domesticate);
-          if (!SNMALLOC_LIKELY(
-                dealloc_local_object_fast(entry, curr.as_void(), entropy)))
-            dealloc_local_object_slow(curr.as_void(), entry);
-          curr = next;
-        } while (curr != tail);
+          dealloc_local_object_slow(msg.as_void(), entry);
+          if (unreturned.batch_size() > 0)
+          {
+            auto meta = entry.get_slab_metadata();
+            do
+            {
+              unreturned = meta->return_objects(unreturned.batch_size());
+              if (unreturned.needs_work())
+                dealloc_local_object_slower(entry, meta);
+            } while (unreturned.batch_size() > 0);
+          }
+        }
       }
       else
       {
-        // XXX message size
+        uint16_t nelem = freelist::Object::cyclic_size(
+          RemoteMessage::to_free_ring(msg),
+          freelist::Object::key_root,
+          entry.get_slab_metadata()->as_key_tweak(),
+          domesticate);
         if (
           !need_post &&
-          !attached_cache->remote_dealloc_cache.reserve_space(entry))
+          !attached_cache->remote_dealloc_cache.reserve_space(entry, nelem))
+        {
           need_post = true;
+        }
         attached_cache->remote_dealloc_cache
           .template forward<sizeof(CoreAllocator)>(
             entry.get_remote()->trunc_id(), msg);
@@ -754,6 +764,41 @@ namespace snmalloc
       return SNMALLOC_LIKELY(!meta->return_object());
     }
 
+    template<typename Domesticator>
+    SNMALLOC_FAST_PATH static auto dealloc_local_objects_fast(
+      const PagemapEntry& entry,
+      capptr::Alloc<RemoteMessage> msg,
+      LocalEntropy& entropy,
+      Domesticator domesticate)
+    {
+      auto meta = entry.get_slab_metadata();
+
+      SNMALLOC_ASSERT(!meta->is_unused());
+
+      snmalloc_check_client(
+        mitigations(sanity_checks),
+        is_start_of_object(entry.get_sizeclass(), address_cast(msg)),
+        "Not deallocating start of an object");
+
+      auto cp = RemoteMessage::to_free_ring(msg);
+
+      SNMALLOC_CHECK(
+        !meta->is_large() ||
+        (freelist::Object::cyclic_size(
+           cp, freelist::Object::key_root, meta->as_key_tweak(), domesticate) ==
+         1));
+
+      // Update the head and the next pointer in the free list.
+      auto size = meta->free_queue.append_cyclic(
+        cp,
+        freelist::Object::key_root,
+        meta->as_key_tweak(),
+        entropy,
+        domesticate);
+
+      return meta->return_objects(size);
+    }
+
     template<ZeroMem zero_mem>
     SNMALLOC_SLOW_PATH capptr::Alloc<void>
     small_alloc(smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list)