diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a8daa429..3ae8c0f82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,9 @@ if (SNMALLOC_SANITIZER) message(STATUS "Using sanitizer=${SNMALLOC_SANITIZER}") endif() +set(SNMALLOC_MIN_ALLOC_SIZE "" CACHE STRING "Minimum allocation bytes (power of 2)") +set(SNMALLOC_MIN_ALLOC_STEP_SIZE "" CACHE STRING "Minimum allocation step (power of 2)") + if(MSVC AND SNMALLOC_STATIC_LIBRARY AND (SNMALLOC_STATIC_LIBRARY_PREFIX STREQUAL "")) message(FATAL_ERROR "Empty static library prefix not supported on MSVC") endif() @@ -226,6 +229,11 @@ endif() function(add_as_define FLAG) target_compile_definitions(snmalloc INTERFACE $<$:${FLAG}>) endfunction() +function(add_as_define_value KEY) + if (NOT ${${KEY}} STREQUAL "") + target_compile_definitions(snmalloc INTERFACE ${KEY}=${${KEY}}) + endif () +endfunction() add_as_define(SNMALLOC_QEMU_WORKAROUND) add_as_define(SNMALLOC_TRACING) @@ -238,6 +246,8 @@ endif() if (SNMALLOC_NO_REALLOCARR) add_as_define(SNMALLOC_NO_REALLOCARR) endif() +add_as_define_value(SNMALLOC_MIN_ALLOC_SIZE) +add_as_define_value(SNMALLOC_MIN_ALLOC_STEP_SIZE) target_compile_definitions(snmalloc INTERFACE $<$:MALLOC_USABLE_SIZE_QUALIFIER=const>) diff --git a/src/snmalloc/ds/allocconfig.h b/src/snmalloc/ds/allocconfig.h index 858940f05..bcb19213e 100644 --- a/src/snmalloc/ds/allocconfig.h +++ b/src/snmalloc/ds/allocconfig.h @@ -20,10 +20,31 @@ namespace snmalloc // Used to isolate values on cache lines to prevent false sharing. static constexpr size_t CACHELINE_SIZE = 64; - // Minimum allocation size is space for two pointers. - static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*)); - static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*); - static constexpr size_t MIN_ALLOC_BITS = bits::ctz_const(MIN_ALLOC_SIZE); + /// The "machine epsilon" for the small sizeclass machinery. + static constexpr size_t MIN_ALLOC_STEP_SIZE = +#if defined(SNMALLOC_MIN_ALLOC_STEP_SIZE) + SNMALLOC_MIN_ALLOC_STEP_SIZE; +#else + 2 * sizeof(void*); +#endif + + /// Derived from MIN_ALLOC_STEP_SIZE + static constexpr size_t MIN_ALLOC_STEP_BITS = + bits::ctz_const(MIN_ALLOC_STEP_SIZE); + static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE)); + + /** + * Minimum allocation size is space for two pointers. If the small sizeclass + * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is + * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must + * be large or not a power of two, those smaller size classes will be unused. + */ + static constexpr size_t MIN_ALLOC_SIZE = +#if defined(SNMALLOC_MIN_ALLOC_SIZE) + SNMALLOC_MIN_ALLOC_SIZE; +#else + 2 * sizeof(void*); +#endif // Minimum slab size. #if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64) @@ -78,11 +99,18 @@ namespace snmalloc static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1; static_assert( - INTERMEDIATE_BITS < MIN_ALLOC_BITS, + INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS, "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS"); static_assert( MIN_ALLOC_SIZE >= (sizeof(void*) * 2), "MIN_ALLOC_SIZE must be sufficient for two pointers"); + static_assert( + 1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >= + bits::next_pow2_const(MIN_ALLOC_SIZE), + "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE"); + static_assert( + MIN_ALLOC_SIZE >= MIN_ALLOC_STEP_SIZE, + "Minimum alloc sizes below minimum step size; raise MIN_ALLOC_SIZE"); // Return remote small allocs when the local cache reaches this size. static constexpr int64_t REMOTE_CACHE = diff --git a/src/snmalloc/ds_core/bits.h b/src/snmalloc/ds_core/bits.h index ac1c7e19b..74f7b3a2b 100644 --- a/src/snmalloc/ds_core/bits.h +++ b/src/snmalloc/ds_core/bits.h @@ -322,22 +322,6 @@ namespace snmalloc * * Does not work for value=0. ***********************************************/ - template - static size_t to_exp_mant(size_t value) - { - constexpr size_t LEADING_BIT = one_at_bit(MANTISSA_BITS + LOW_BITS) >> 1; - constexpr size_t MANTISSA_MASK = one_at_bit(MANTISSA_BITS) - 1; - - value = value - 1; - - size_t e = - bits::BITS - MANTISSA_BITS - LOW_BITS - clz(value | LEADING_BIT); - size_t b = (e == 0) ? 0 : 1; - size_t m = (value >> (LOW_BITS + e - b)) & MANTISSA_MASK; - - return (e << MANTISSA_BITS) + m; - } - template constexpr size_t to_exp_mant_const(size_t value) { diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index e164f5921..84b34ab97 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -597,23 +597,6 @@ namespace snmalloc init_message_queue(); message_queue().invariant(); } - - if constexpr (DEBUG) - { - for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) - { - size_t size = sizeclass_to_size(i); - smallsizeclass_t sc1 = size_to_sizeclass(size); - smallsizeclass_t sc2 = size_to_sizeclass_const(size); - size_t size1 = sizeclass_to_size(sc1); - size_t size2 = sizeclass_to_size(sc2); - - SNMALLOC_CHECK(sc1 == i); - SNMALLOC_CHECK(sc1 == sc2); - SNMALLOC_CHECK(size1 == size); - SNMALLOC_CHECK(size1 == size2); - } - } } public: diff --git a/src/snmalloc/mem/sizeclasstable.h b/src/snmalloc/mem/sizeclasstable.h index 2d13d9c09..900e8332b 100644 --- a/src/snmalloc/mem/sizeclasstable.h +++ b/src/snmalloc/mem/sizeclasstable.h @@ -24,7 +24,7 @@ namespace snmalloc // For example, 24 byte allocations can be // problematic for some data due to alignment issues. auto sc = static_cast( - bits::to_exp_mant_const(size)); + bits::to_exp_mant_const(size)); SNMALLOC_ASSERT(sc == static_cast(sc)); @@ -214,7 +214,8 @@ namespace snmalloc auto& meta = fast_small(sizeclass); size_t rsize = - bits::from_exp_mant(sizeclass); + bits::from_exp_mant( + sizeclass); meta.size = rsize; size_t slab_bits = bits::max( bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS); @@ -405,7 +406,7 @@ namespace snmalloc { // We subtract and shift to reduce the size of the table, i.e. we don't have // to store a value for every size. - return (s - 1) >> MIN_ALLOC_BITS; + return (s - 1) >> MIN_ALLOC_STEP_BITS; } constexpr size_t sizeclass_lookup_size = @@ -421,13 +422,29 @@ namespace snmalloc constexpr SizeClassLookup() { + constexpr sizeclass_compress_t minimum_class = + static_cast( + size_to_sizeclass_const(MIN_ALLOC_SIZE)); + + /* Some unused sizeclasses is OK, but keep it within reason! */ + static_assert(minimum_class < sizeclass_lookup_size); + size_t curr = 1; - for (sizeclass_compress_t sizeclass = 0; - sizeclass < NUM_SMALL_SIZECLASSES; - sizeclass++) + + sizeclass_compress_t sizeclass = 0; + for (; sizeclass < minimum_class; sizeclass++) + { + for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; + curr += MIN_ALLOC_STEP_SIZE) + { + table[sizeclass_lookup_index(curr)] = minimum_class; + } + } + + for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; - curr += 1 << MIN_ALLOC_BITS) + curr += MIN_ALLOC_STEP_SIZE) { auto i = sizeclass_lookup_index(curr); if (i == sizeclass_lookup_size) diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index ff1856fac..f435b4572 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -57,6 +57,9 @@ extern "C" void abort() { longjmp(jmp, 1); } +# if __has_builtin(__builtin_trap) + __builtin_trap(); +# endif exit(-1); } @@ -152,7 +155,11 @@ int main() // Some sizes to check for out-of-bounds access. As we are only able to // catch overflows past the end of the sizeclass-padded allocation, make // sure we don't try to test on smaller allocations. - std::initializer_list sizes = {MIN_ALLOC_SIZE, 1024, 2 * 1024 * 1024}; + + static constexpr size_t min_class_size = + sizeclass_to_size(size_to_sizeclass(MIN_ALLOC_SIZE)); + + std::initializer_list sizes = {min_class_size, 1024, 2 * 1024 * 1024}; static_assert( MIN_ALLOC_SIZE < 1024, "Can't detect overflow except at sizeclass boundaries"); diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 2a2ada2ee..8b3606012 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -237,7 +237,9 @@ void test_external_pointer() // Malloc does not have an external pointer querying mechanism. auto& alloc = ThreadAlloc::get(); - for (uint8_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); + sc < NUM_SMALL_SIZECLASSES; + sc++) { size_t size = sizeclass_to_size(sc); void* p1 = alloc.alloc(size); @@ -470,7 +472,9 @@ void test_static_sized_allocs() void test_remaining_bytes() { auto& alloc = ThreadAlloc::get(); - for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); + sc < NUM_SMALL_SIZECLASSES; + sc++) { auto size = sizeclass_to_size(sc); char* p = (char*)alloc.alloc(size); diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index d42794e44..836c62111 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -8,6 +8,9 @@ snmalloc::smallsizeclass_t size_to_sizeclass(size_t size) return snmalloc::size_to_sizeclass(size); } +static constexpr snmalloc::smallsizeclass_t minimum_sizeclass = + snmalloc::size_to_sizeclass_const(snmalloc::MIN_ALLOC_SIZE); + void test_align_size() { bool failed = false; @@ -72,6 +75,10 @@ int main(int, char**) bool failed = false; size_t size_low = 0; + std::cout << "Configured with minimum allocation size " + << snmalloc::MIN_ALLOC_SIZE << " and step size " + << snmalloc::MIN_ALLOC_STEP_SIZE << std::endl; + std::cout << "0 has sizeclass: " << (size_t)snmalloc::size_to_sizeclass(0) << std::endl; @@ -86,12 +93,14 @@ int main(int, char**) slab_size != snmalloc::sizeclass_to_slab_size(sz)) { slab_size = snmalloc::sizeclass_to_slab_size(sz); - std::cout << std::endl; + std::cout << std::endl << "slab size: " << slab_size << std::endl; } size_t size = snmalloc::sizeclass_to_size(sz); std::cout << (size_t)sz << " |-> " - << "[" << size_low + 1 << ", " << size << "]" << std::endl; + << "[" << size_low + 1 << ", " << size << "]" + << (sz == minimum_sizeclass ? " is minimum class" : "") + << std::endl; if (size < size_low) { @@ -102,7 +111,30 @@ int main(int, char**) for (size_t i = size_low + 1; i <= size; i++) { - if (size_to_sizeclass(i) != sz) + /* All sizes should, via bit-math, come back to their class value */ + if (snmalloc::size_to_sizeclass_const(i) != sz) + { + std::cout << "Size " << i << " has _const sizeclass " + << (size_t)snmalloc::size_to_sizeclass_const(i) + << " but expected sizeclass " << (size_t)sz << std::endl; + failed = true; + } + + if (size < snmalloc::MIN_ALLOC_SIZE) + { + /* + * It is expected that these sizes have the "wrong" class from tabular + * lookup: they will have been clipped up to the minimum class. + */ + if (size_to_sizeclass(i) != minimum_sizeclass) + { + std::cout << "Size " << i << " below minimum size; sizeclass " + << (size_t)size_to_sizeclass(i) << " not expected minimum " + << (size_t)minimum_sizeclass << std::endl; + failed = true; + } + } + else if (size_to_sizeclass(i) != sz) { std::cout << "Size " << i << " has sizeclass " << (size_t)size_to_sizeclass(i) << " but expected sizeclass "