diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 92ebd1609..f0ea7e370 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -557,26 +557,36 @@ namespace snmalloc if (SNMALLOC_LIKELY(entry.get_remote() == public_state())) { - auto key_tweak = entry.get_slab_metadata()->as_key_tweak(); - freelist::HeadPtr tail = RemoteMessage::to_free_ring(msg); - freelist::HeadPtr curr = tail; - do + auto unreturned = + dealloc_local_objects_fast(entry, msg, entropy, domesticate); + if (SNMALLOC_UNLIKELY(unreturned.needs_work())) { - auto next = - curr->read_next(freelist::Object::key_root, key_tweak, domesticate); - if (!SNMALLOC_LIKELY( - dealloc_local_object_fast(entry, curr.as_void(), entropy))) - dealloc_local_object_slow(curr.as_void(), entry); - curr = next; - } while (curr != tail); + dealloc_local_object_slow(msg.as_void(), entry); + if (unreturned.batch_size() > 0) + { + auto meta = entry.get_slab_metadata(); + do + { + unreturned = meta->return_objects(unreturned.batch_size()); + if (unreturned.needs_work()) + dealloc_local_object_slower(entry, meta); + } while (unreturned.batch_size() > 0); + } + } } else { - // XXX message size + uint16_t nelem = freelist::Object::cyclic_size( + RemoteMessage::to_free_ring(msg), + freelist::Object::key_root, + entry.get_slab_metadata()->as_key_tweak(), + domesticate); if ( !need_post && - !attached_cache->remote_dealloc_cache.reserve_space(entry)) + !attached_cache->remote_dealloc_cache.reserve_space(entry, nelem)) + { need_post = true; + } attached_cache->remote_dealloc_cache .template forward( entry.get_remote()->trunc_id(), msg); @@ -754,6 +764,41 @@ namespace snmalloc return SNMALLOC_LIKELY(!meta->return_object()); } + template + SNMALLOC_FAST_PATH static auto dealloc_local_objects_fast( + const PagemapEntry& entry, + capptr::Alloc msg, + LocalEntropy& entropy, + Domesticator domesticate) + { + auto meta = entry.get_slab_metadata(); + + SNMALLOC_ASSERT(!meta->is_unused()); + + snmalloc_check_client( + mitigations(sanity_checks), + is_start_of_object(entry.get_sizeclass(), address_cast(msg)), + "Not deallocating start of an object"); + + auto cp = RemoteMessage::to_free_ring(msg); + + SNMALLOC_CHECK( + !meta->is_large() || + (freelist::Object::cyclic_size( + cp, freelist::Object::key_root, meta->as_key_tweak(), domesticate) == + 1)); + + // Update the head and the next pointer in the free list. + auto size = meta->free_queue.append_cyclic( + cp, + freelist::Object::key_root, + meta->as_key_tweak(), + entropy, + domesticate); + + return meta->return_objects(size); + } + template SNMALLOC_SLOW_PATH capptr::Alloc small_alloc(smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list)