From 0cdbe594ca93aebcb194a554e0ad8307eb1401f8 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 4 Jan 2024 20:24:37 +0000 Subject: [PATCH] sizeclass: distinguish min alloc and step size Add support for a minimum allocation size that isn't the minimum step of the sizeclass table. --- src/snmalloc/ds/allocconfig.h | 23 +++++++++++++++++++---- src/snmalloc/mem/sizeclasstable.h | 31 ++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/snmalloc/ds/allocconfig.h b/src/snmalloc/ds/allocconfig.h index 858940f05..1218a92b9 100644 --- a/src/snmalloc/ds/allocconfig.h +++ b/src/snmalloc/ds/allocconfig.h @@ -20,10 +20,21 @@ namespace snmalloc // Used to isolate values on cache lines to prevent false sharing. static constexpr size_t CACHELINE_SIZE = 64; - // Minimum allocation size is space for two pointers. - static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*)); + /// The "machine epsilon" for the small sizeclass machinery. + static constexpr size_t MIN_ALLOC_STEP_SIZE = 2 * sizeof(void*); + + /// Derived from MIN_ALLOC_STEP_SIZE + static constexpr size_t MIN_ALLOC_STEP_BITS = + bits::ctz_const(MIN_ALLOC_STEP_SIZE); + static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE)); + + /** + * Minimum allocation size is space for two pointers. If the small sizeclass + * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is + * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must + * be large or not a power of two, those smaller size classes will be unused. + */ static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*); - static constexpr size_t MIN_ALLOC_BITS = bits::ctz_const(MIN_ALLOC_SIZE); // Minimum slab size. #if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) @@ -78,11 +89,15 @@ namespace snmalloc static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1; static_assert( - INTERMEDIATE_BITS < MIN_ALLOC_BITS, + INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS, "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS"); static_assert( MIN_ALLOC_SIZE >= (sizeof(void*) * 2), "MIN_ALLOC_SIZE must be sufficient for two pointers"); + static_assert( + 1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >= + bits::next_pow2_const(MIN_ALLOC_SIZE), + "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE"); // Return remote small allocs when the local cache reaches this size. static constexpr int64_t REMOTE_CACHE = diff --git a/src/snmalloc/mem/sizeclasstable.h b/src/snmalloc/mem/sizeclasstable.h index 2d13d9c09..137a2ef64 100644 --- a/src/snmalloc/mem/sizeclasstable.h +++ b/src/snmalloc/mem/sizeclasstable.h @@ -24,7 +24,7 @@ namespace snmalloc // For example, 24 byte allocations can be // problematic for some data due to alignment issues. auto sc = static_cast( - bits::to_exp_mant_const(size)); + bits::to_exp_mant_const(size)); SNMALLOC_ASSERT(sc == static_cast(sc)); @@ -214,7 +214,8 @@ namespace snmalloc auto& meta = fast_small(sizeclass); size_t rsize = - bits::from_exp_mant(sizeclass); + bits::from_exp_mant( + sizeclass); meta.size = rsize; size_t slab_bits = bits::max( bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS); @@ -405,7 +406,7 @@ namespace snmalloc { // We subtract and shift to reduce the size of the table, i.e. we don't have // to store a value for every size. - return (s - 1) >> MIN_ALLOC_BITS; + return (s - 1) >> MIN_ALLOC_STEP_BITS; } constexpr size_t sizeclass_lookup_size = @@ -421,13 +422,29 @@ namespace snmalloc constexpr SizeClassLookup() { + constexpr sizeclass_compress_t minimum_class = + static_cast( + size_to_sizeclass_const(MIN_ALLOC_SIZE)); + + /* Some unused sizeclasses is OK, but keep it within reason! */ + static_assert(minimum_class < sizeclass_lookup_size); + size_t curr = 1; - for (sizeclass_compress_t sizeclass = 0; - sizeclass < NUM_SMALL_SIZECLASSES; - sizeclass++) + + sizeclass_compress_t sizeclass = 0; + for (; sizeclass < minimum_class; sizeclass++) + { + for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; + curr += 1 << MIN_ALLOC_STEP_BITS) + { + table[sizeclass_lookup_index(curr)] = minimum_class; + } + } + + for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; - curr += 1 << MIN_ALLOC_BITS) + curr += 1 << MIN_ALLOC_STEP_BITS) { auto i = sizeclass_lookup_index(curr); if (i == sizeclass_lookup_size)