diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bcb6b974f..a2a8f918d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -224,15 +224,14 @@ jobs: matrix: # Build just release variant as Debug is too slow. build-type: [ Release ] + os: ["ubuntu-latest", "ubuntu-20.04"] include: - os: "ubuntu-latest" - continue-on-error: # Don't class as an error if this fails, until we have a more reliablity. variant: "libc++ (TSan + UBSan)" dependencies: "sudo apt install ninja-build" extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread" # Also test specifically with clang-10 (on ubuntu-20.04) - os: "ubuntu-20.04" - continue-on-error: # Don't class as an error if this fails, until we have a more reliablity. variant: "clang-10 libc++ (TSan + UBSan)" dependencies: "sudo apt install ninja-build" extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread" @@ -452,7 +451,7 @@ jobs: git diff --exit-code - name: Run clang-tidy run: | - clang-tidy-15 src/snmalloc/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 + clang-tidy-15 src/snmalloc/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 -Isrc if [ -f tidy.fail ] ; then cat tidy.fail exit 1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5697790cc..745ae198f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -331,6 +331,9 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) if(SNMALLOC_SANITIZER) target_compile_options(${TESTNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer) target_link_libraries(${TESTNAME} -fsanitize=${SNMALLOC_SANITIZER}) + if (${SNMALLOC_SANITIZER} MATCHES "thread") + target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_THREAD_SANITIZER_ENABLED) + endif() endif() add_warning_flags(${TESTNAME}) diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index ce5e757ed..883c471c6 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -23,9 +23,6 @@ namespace snmalloc using Pal = PAL; using SlabMetadata = typename PagemapEntry::SlabMetadata; - static constexpr size_t SizeofMetadata = - bits::next_pow2_const(sizeof(SlabMetadata)); - public: /** * Provide a block of meta-data with size and align. @@ -90,13 +87,26 @@ namespace snmalloc * (remote, sizeclass, slab_metadata) * where slab_metadata, is the second element of the pair return. */ - static std::pair, SlabMetadata*> - alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras) + static std::pair, SlabMetadata*> alloc_chunk( + LocalState& local_state, + size_t size, + uintptr_t ras, + sizeclass_t sizeclass) { SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); - auto meta_cap = local_state.get_meta_range().alloc_range(SizeofMetadata); + // Calculate the extra bytes required to store the client meta-data. + size_t extra_bytes = SlabMetadata::get_extra_bytes(sizeclass); + + auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes); + +#ifdef SNMALLOC_TRACING + message<1024>( + "Allocating metadata of size: {} ({})", meta_size, extra_bytes); +#endif + + auto meta_cap = local_state.get_meta_range().alloc_range(meta_size); auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); @@ -113,7 +123,7 @@ namespace snmalloc #endif if (p == nullptr) { - local_state.get_meta_range().dealloc_range(meta_cap, SizeofMetadata); + local_state.get_meta_range().dealloc_range(meta_cap, meta_size); errno = ENOMEM; #ifdef SNMALLOC_TRACING message<1024>("Out of memory"); @@ -140,7 +150,8 @@ namespace snmalloc LocalState& local_state, SlabMetadata& slab_metadata, capptr::Alloc alloc, - size_t size) + size_t size, + sizeclass_t sizeclass) { /* * The backend takes possession of these chunks now, by disassociating @@ -167,8 +178,12 @@ namespace snmalloc */ capptr::Arena arena = Authmap::amplify(alloc); + // Calculate the extra bytes required to store the client meta-data. + size_t extra_bytes = SlabMetadata::get_extra_bytes(sizeclass); + + auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes); local_state.get_meta_range().dealloc_range( - capptr::Arena::unsafe_from(&slab_metadata), SizeofMetadata); + capptr::Arena::unsafe_from(&slab_metadata), meta_size); local_state.get_object_range()->dealloc_range(arena, size); } diff --git a/src/snmalloc/backend/fixedglobalconfig.h b/src/snmalloc/backend/fixedglobalconfig.h index 37a78c287..83e111747 100644 --- a/src/snmalloc/backend/fixedglobalconfig.h +++ b/src/snmalloc/backend/fixedglobalconfig.h @@ -8,11 +8,14 @@ namespace snmalloc /** * A single fixed address range allocator configuration */ - template + template< + SNMALLOC_CONCEPT(IsPAL) PAL, + typename ClientMetaDataProvider = NoClientMetaDataProvider> class FixedRangeConfig final : public CommonConfig { public: - using PagemapEntry = DefaultPagemapEntry; + using PagemapEntry = DefaultPagemapEntry; + using ClientMeta = ClientMetaDataProvider; private: using ConcretePagemap = diff --git a/src/snmalloc/backend/globalconfig.h b/src/snmalloc/backend/globalconfig.h index 525c77275..72a6c3d42 100644 --- a/src/snmalloc/backend/globalconfig.h +++ b/src/snmalloc/backend/globalconfig.h @@ -1,13 +1,9 @@ #pragma once -// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own -// definition of `snmalloc::Alloc` before including any files that include -// `snmalloc.h` or consume the global allocation APIs. -#ifndef SNMALLOC_PROVIDE_OWN_CONFIG -# include "../backend_helpers/backend_helpers.h" -# include "backend.h" -# include "meta_protected_range.h" -# include "standard_range.h" +#include "../backend_helpers/backend_helpers.h" +#include "backend.h" +#include "meta_protected_range.h" +#include "standard_range.h" namespace snmalloc { @@ -28,13 +24,16 @@ namespace snmalloc * The Configuration sets up a Pagemap for the backend to use, and the state * required to build new allocators (GlobalPoolState). */ - class StandardConfig final : public CommonConfig + template + class StandardConfigClientMeta final : public CommonConfig { - using GlobalPoolState = PoolState>; + using GlobalPoolState = PoolState< + CoreAllocator>>; public: using Pal = DefaultPal; - using PagemapEntry = DefaultPagemapEntry; + using PagemapEntry = DefaultPagemapEntry; + using ClientMeta = ClientMetaDataProvider; private: using ConcretePagemap = @@ -98,9 +97,9 @@ namespace snmalloc SNMALLOC_SLOW_PATH static void ensure_init_slow() { FlagLock lock{initialisation_lock}; -# ifdef SNMALLOC_TRACING +#ifdef SNMALLOC_TRACING message<1024>("Run init_impl"); -# endif +#endif if (initialised) return; @@ -162,10 +161,4 @@ namespace snmalloc snmalloc::register_clean_up(); } }; - - /** - * Create allocator type for this configuration. - */ - using Alloc = snmalloc::LocalAllocator; } // namespace snmalloc -#endif diff --git a/src/snmalloc/backend_helpers/commonconfig.h b/src/snmalloc/backend_helpers/commonconfig.h index a69b6a389..5ef023ce0 100644 --- a/src/snmalloc/backend_helpers/commonconfig.h +++ b/src/snmalloc/backend_helpers/commonconfig.h @@ -95,6 +95,39 @@ namespace snmalloc bool HasDomesticate = false; }; + struct NoClientMetaDataProvider + { + using StorageType = Empty; + using DataRef = Empty&; + + static size_t required_count(size_t) + { + return 1; + } + + static DataRef get(StorageType* base, size_t) + { + return *base; + } + }; + + template + struct ArrayClientMetaDataProvider + { + using StorageType = T; + using DataRef = T&; + + static size_t required_count(size_t max_count) + { + return max_count; + } + + static DataRef get(StorageType* base, size_t index) + { + return base[index]; + } + }; + /** * Class containing definitions that are likely to be used by all except for * the most unusual back-end implementations. This can be subclassed as a diff --git a/src/snmalloc/backend_helpers/defaultpagemapentry.h b/src/snmalloc/backend_helpers/defaultpagemapentry.h index 2083db30e..5e1f703d2 100644 --- a/src/snmalloc/backend_helpers/defaultpagemapentry.h +++ b/src/snmalloc/backend_helpers/defaultpagemapentry.h @@ -64,9 +64,14 @@ namespace snmalloc SNMALLOC_FAST_PATH DefaultPagemapEntryT() = default; }; - class DefaultSlabMetadata : public FrontendSlabMetadata + template + class DefaultSlabMetadata : public FrontendSlabMetadata< + DefaultSlabMetadata, + ClientMetaDataProvider> {}; - using DefaultPagemapEntry = DefaultPagemapEntryT; + template + using DefaultPagemapEntry = + DefaultPagemapEntryT>; } // namespace snmalloc diff --git a/src/snmalloc/ds/mpmcstack.h b/src/snmalloc/ds/mpmcstack.h index cd005e9bf..e6a3b1d9f 100644 --- a/src/snmalloc/ds/mpmcstack.h +++ b/src/snmalloc/ds/mpmcstack.h @@ -4,12 +4,6 @@ #include "aba.h" #include "allocconfig.h" -#if defined(__has_feature) -# if __has_feature(thread_sanitizer) -# define SNMALLOC_THREAD_SANITIZER_ENABLED -# endif -#endif - namespace snmalloc { template diff --git a/src/snmalloc/ds/pagemap.h b/src/snmalloc/ds/pagemap.h index 267fe9a0b..abfc539d0 100644 --- a/src/snmalloc/ds/pagemap.h +++ b/src/snmalloc/ds/pagemap.h @@ -1,5 +1,7 @@ #pragma once +#include "../ds_core/ds_core.h" + namespace snmalloc { /** @@ -179,7 +181,13 @@ namespace snmalloc // Allocate a power of two extra to allow the placement of the // pagemap be difficult to guess if randomize_position set. size_t additional_size = +#ifdef SNMALLOC_THREAD_SANITIZER_ENABLED + // When running with TSAN we failed to allocate the very large range + // randomly + randomize_position ? bits::next_pow2(REQUIRED_SIZE) : 0; +#else randomize_position ? bits::next_pow2(REQUIRED_SIZE) * 4 : 0; +#endif size_t request_size = REQUIRED_SIZE + additional_size; auto new_body_untyped = PAL::reserve(request_size); diff --git a/src/snmalloc/global/global.h b/src/snmalloc/global/global.h index a2f1159a1..514d69b7c 100644 --- a/src/snmalloc/global/global.h +++ b/src/snmalloc/global/global.h @@ -1,4 +1,5 @@ #include "bounds_checks.h" +#include "libc.h" #include "memcpy.h" #include "scopedalloc.h" #include "threadalloc.h" diff --git a/src/snmalloc/override/libc.h b/src/snmalloc/global/libc.h similarity index 91% rename from src/snmalloc/override/libc.h rename to src/snmalloc/global/libc.h index 587d94ac2..700fd3901 100644 --- a/src/snmalloc/override/libc.h +++ b/src/snmalloc/global/libc.h @@ -1,6 +1,6 @@ #pragma once -#include "../global/global.h" +#include "threadalloc.h" #include #include @@ -176,4 +176,17 @@ namespace snmalloc::libc *memptr = p; return 0; } + + inline typename snmalloc::Alloc::Config::ClientMeta::DataRef + get_client_meta_data(void* p) + { + return ThreadAlloc::get().get_client_meta_data(p); + } + + inline std::add_const_t + get_client_meta_data_const(void* p) + { + return ThreadAlloc::get().get_client_meta_data_const(p); + } + } // namespace snmalloc::libc diff --git a/src/snmalloc/global/memcpy.h b/src/snmalloc/global/memcpy.h index 565719a90..3054484d9 100644 --- a/src/snmalloc/global/memcpy.h +++ b/src/snmalloc/global/memcpy.h @@ -1,5 +1,4 @@ #pragma once -#include "../backend/globalconfig.h" #include "bounds_checks.h" namespace snmalloc diff --git a/src/snmalloc/global/scopedalloc.h b/src/snmalloc/global/scopedalloc.h index cb9f0fc8b..345635a70 100644 --- a/src/snmalloc/global/scopedalloc.h +++ b/src/snmalloc/global/scopedalloc.h @@ -1,5 +1,4 @@ #pragma once -#include "../backend/globalconfig.h" /** * This header requires that Alloc has been defined. diff --git a/src/snmalloc/global/threadalloc.h b/src/snmalloc/global/threadalloc.h index d900fb272..7ba8ddd79 100644 --- a/src/snmalloc/global/threadalloc.h +++ b/src/snmalloc/global/threadalloc.h @@ -1,7 +1,5 @@ #pragma once -#include "../backend/globalconfig.h" - #if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC) # define SNMALLOC_THREAD_TEARDOWN_DEFINED #endif diff --git a/src/snmalloc/mem/backend_concept.h b/src/snmalloc/mem/backend_concept.h index 1680391f7..f1b428607 100644 --- a/src/snmalloc/mem/backend_concept.h +++ b/src/snmalloc/mem/backend_concept.h @@ -2,6 +2,7 @@ #ifdef __cpp_concepts # include "../ds/ds.h" +# include "sizeclasstable.h" # include namespace snmalloc @@ -97,9 +98,13 @@ namespace snmalloc template concept IsBackend = - requires(LocalState& local_state, size_t size, uintptr_t ras) { + requires( + LocalState& local_state, + size_t size, + uintptr_t ras, + sizeclass_t sizeclass) { { - Backend::alloc_chunk(local_state, size, ras) + Backend::alloc_chunk(local_state, size, ras, sizeclass) } -> ConceptSame< std::pair, typename Backend::SlabMetadata*>>; } && @@ -112,9 +117,11 @@ namespace snmalloc LocalState& local_state, typename Backend::SlabMetadata& slab_metadata, capptr::Alloc alloc, - size_t size) { + size_t size, + sizeclass_t sizeclass) { { - Backend::dealloc_chunk(local_state, slab_metadata, alloc, size) + Backend::dealloc_chunk( + local_state, slab_metadata, alloc, size, sizeclass) } -> ConceptSame; } && requires(address_t p) { diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 84b34ab97..2577eab3b 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -368,7 +368,8 @@ namespace snmalloc get_backend_local_state(), *meta, start, - sizeclass_to_slab_size(sizeclass)); + sizeclass_to_slab_size(sizeclass), + sizeclass_t::from_small_class(sizeclass)); }); } @@ -401,7 +402,7 @@ namespace snmalloc meta->node.remove(); Config::Backend::dealloc_chunk( - get_backend_local_state(), *meta, p, size); + get_backend_local_state(), *meta, p, size, entry.get_sizeclass()); return; } @@ -796,7 +797,8 @@ namespace snmalloc get_backend_local_state(), slab_size, PagemapEntry::encode( - public_state(), sizeclass_t::from_small_class(sizeclass))); + public_state(), sizeclass_t::from_small_class(sizeclass)), + sizeclass_t::from_small_class(sizeclass)); if (slab == nullptr) { diff --git a/src/snmalloc/mem/freelist.h b/src/snmalloc/mem/freelist.h index 49348d1d8..fb401d4b2 100644 --- a/src/snmalloc/mem/freelist.h +++ b/src/snmalloc/mem/freelist.h @@ -187,7 +187,7 @@ namespace snmalloc signed_prev(address_cast(this), address_cast(n_tame), key)); } } - Aal::prefetch(&(n_tame->next_object)); + Aal::prefetch(n_tame.unsafe_ptr()); return n_tame; } diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index e26243a9b..e0096d80d 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -197,7 +197,8 @@ namespace snmalloc core_alloc->get_backend_local_state(), large_size_to_chunk_size(size), PagemapEntry::encode( - core_alloc->public_state(), size_to_sizeclass_full(size))); + core_alloc->public_state(), size_to_sizeclass_full(size)), + size_to_sizeclass_full(size)); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -816,6 +817,57 @@ namespace snmalloc } } + /** + * @brief Get the client meta data for the snmalloc allocation covering this + * pointer. + */ + typename Config::ClientMeta::DataRef get_client_meta_data(void* p) + { + const PagemapEntry& entry = + Config::Backend::template get_metaentry(address_cast(p)); + + size_t index = slab_index(entry.get_sizeclass(), address_cast(p)); + + auto* meta_slab = entry.get_slab_metadata(); + + if (SNMALLOC_UNLIKELY(entry.is_backend_owned())) + { + error("Cannot access meta-data for write for freed memory!"); + } + + if (SNMALLOC_UNLIKELY(meta_slab == nullptr)) + { + error( + "Cannot access meta-data for non-snmalloc object in writable form!"); + } + + return meta_slab->get_meta_for_object(index); + } + + /** + * @brief Get the client meta data for the snmalloc allocation covering this + * pointer. + */ + std::add_const_t + get_client_meta_data_const(void* p) + { + const PagemapEntry& entry = + Config::Backend::template get_metaentry(address_cast(p)); + + size_t index = slab_index(entry.get_sizeclass(), address_cast(p)); + + auto* meta_slab = entry.get_slab_metadata(); + + if (SNMALLOC_UNLIKELY( + (meta_slab == nullptr) || (entry.is_backend_owned()))) + { + static typename Config::ClientMeta::StorageType null_meta_store{}; + return Config::ClientMeta::get(&null_meta_store, 0); + } + + return meta_slab->get_meta_for_object(index); + } + /** * Returns the number of remaining bytes in an object. * diff --git a/src/snmalloc/mem/metadata.h b/src/snmalloc/mem/metadata.h index e7937e9d8..10fa93e37 100644 --- a/src/snmalloc/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -368,21 +368,26 @@ namespace snmalloc class FrontendSlabMetadata_Trait { private: - template + template friend class FrontendSlabMetadata; // Can only be constructed by FrontendSlabMetadata - FrontendSlabMetadata_Trait() = default; + constexpr FrontendSlabMetadata_Trait() = default; }; /** * The FrontendSlabMetadata represent the metadata associated with a single * slab. */ - template + template class FrontendSlabMetadata : public FrontendSlabMetadata_Trait { public: + /** + * Type that encapsulates logic for accessing client meta-data. + */ + using ClientMeta = ClientMeta_; + /** * Used to link slab metadata together in various other data-structures. * This is used with `SeqSet` and so may actually hold a subclass of this @@ -424,6 +429,13 @@ namespace snmalloc */ bool large_ = false; + /** + * Stores client meta-data for this slab. This must be last element in the + * slab. The meta data will actually allocate multiple elements after this + * type, so that client_meta_[1] will work for the required meta-data size. + */ + SNMALLOC_NO_UNIQUE_ADDRESS typename ClientMeta::StorageType client_meta_{}; + uint16_t& needed() { return needed_; @@ -452,6 +464,9 @@ namespace snmalloc set_sleeping(sizeclass, 0); large_ = false; + + new (&client_meta_) + typename ClientMeta::StorageType[get_client_storage_count(sizeclass)]; } /** @@ -469,6 +484,8 @@ namespace snmalloc // Jump to slow path on first deallocation. needed() = 1; + + new (&client_meta_) typename ClientMeta::StorageType(); } /** @@ -583,6 +600,33 @@ namespace snmalloc { return address_cast(free_queue.read_head(0, key)); } + + typename ClientMeta::DataRef get_meta_for_object(size_t index) + { + return ClientMeta::get(&client_meta_, index); + } + + static size_t get_client_storage_count(smallsizeclass_t sizeclass) + { + auto count = sizeclass_to_slab_object_count(sizeclass); + auto result = ClientMeta::required_count(count); + if (result == 0) + return 1; + return result; + } + + static size_t get_extra_bytes(sizeclass_t sizeclass) + { + if (sizeclass.is_small()) + // We remove one from the extra-bytes as there is one in the metadata to + // start with. + return (get_client_storage_count(sizeclass.as_small()) - 1) * + sizeof(typename ClientMeta::StorageType); + + // For large classes there is only a single entry, so this is covered by + // the existing entry in the metaslab, and further bytes are not required. + return 0; + } }; /** @@ -646,7 +690,7 @@ namespace snmalloc */ [[nodiscard]] SNMALLOC_FAST_PATH SlabMetadata* get_slab_metadata() const { - SNMALLOC_ASSERT(get_remote() != nullptr); + SNMALLOC_ASSERT(!is_backend_owned()); return unsafe_from_uintptr(meta & ~META_BOUNDARY_BIT); } }; diff --git a/src/snmalloc/mem/sizeclasstable.h b/src/snmalloc/mem/sizeclasstable.h index 900e8332b..0a033319d 100644 --- a/src/snmalloc/mem/sizeclasstable.h +++ b/src/snmalloc/mem/sizeclasstable.h @@ -333,14 +333,11 @@ namespace snmalloc .capacity; } - constexpr address_t start_of_object(sizeclass_t sc, address_t addr) + SNMALLOC_FAST_PATH constexpr size_t slab_index(sizeclass_t sc, address_t addr) { auto meta = sizeclass_metadata.fast(sc); - address_t slab_start = addr & ~meta.slab_mask; size_t offset = addr & meta.slab_mask; - size_t size = meta.size; - - if constexpr (sizeof(addr) >= 8) + if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. @@ -351,17 +348,27 @@ namespace snmalloc // the slab_mask by making the `div_mult` zero. The link uses 128 bit // multiplication, we have shrunk the range of the calculation to remove // this dependency. - size_t offset_start = ((offset * meta.div_mult) >> DIV_MULT_SHIFT) * size; - return slab_start + offset_start; + size_t index = ((offset * meta.div_mult) >> DIV_MULT_SHIFT); + return index; } else { + size_t size = meta.size; if (size == 0) return 0; - return slab_start + (offset / size) * size; + return offset / size; } } + SNMALLOC_FAST_PATH constexpr address_t + start_of_object(sizeclass_t sc, address_t addr) + { + auto meta = sizeclass_metadata.fast(sc); + address_t slab_start = addr & ~meta.slab_mask; + size_t index = slab_index(sc, addr); + return slab_start + (index * meta.size); + } + constexpr size_t index_in_object(sizeclass_t sc, address_t addr) { return addr - start_of_object(sc, addr); diff --git a/src/snmalloc/override/malloc.cc b/src/snmalloc/override/malloc.cc index 22d78244b..9c0571f74 100644 --- a/src/snmalloc/override/malloc.cc +++ b/src/snmalloc/override/malloc.cc @@ -1,4 +1,3 @@ -#include "libc.h" #include "override.h" using namespace snmalloc; diff --git a/src/snmalloc/override/new.cc b/src/snmalloc/override/new.cc index ff83b281e..19aa9f58c 100644 --- a/src/snmalloc/override/new.cc +++ b/src/snmalloc/override/new.cc @@ -1,4 +1,4 @@ -#include "libc.h" +#include "snmalloc/snmalloc.h" #ifdef _WIN32 # ifdef __clang__ diff --git a/src/snmalloc/override/override.h b/src/snmalloc/override/override.h index 0ca70bc11..5dda309c0 100644 --- a/src/snmalloc/override/override.h +++ b/src/snmalloc/override/override.h @@ -1,6 +1,6 @@ #pragma once -#include "../global/global.h" +#include "snmalloc/snmalloc.h" #ifndef SNMALLOC_EXPORT # define SNMALLOC_EXPORT diff --git a/src/snmalloc/override/rust.cc b/src/snmalloc/override/rust.cc index f7825cda1..4d7e79a7c 100644 --- a/src/snmalloc/override/rust.cc +++ b/src/snmalloc/override/rust.cc @@ -1,5 +1,5 @@ #define SNMALLOC_NAME_MANGLE(a) sn_##a -#include "malloc.cc" +#include "snmalloc/snmalloc.h" #include @@ -48,6 +48,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)( extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_statistics)( size_t* current_memory_usage, size_t* peak_memory_usage) { - *current_memory_usage = StandardConfig::Backend::get_current_usage(); - *peak_memory_usage = StandardConfig::Backend::get_peak_usage(); + *current_memory_usage = Alloc::Config::Backend::get_current_usage(); + *peak_memory_usage = Alloc::Config::Backend::get_peak_usage(); } \ No newline at end of file diff --git a/src/snmalloc/pal/pal_posix.h b/src/snmalloc/pal/pal_posix.h index 6c9ae05e8..097d18f61 100644 --- a/src/snmalloc/pal/pal_posix.h +++ b/src/snmalloc/pal/pal_posix.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/src/snmalloc/snmalloc.h b/src/snmalloc/snmalloc.h index 47bd6e78a..b05b1a330 100644 --- a/src/snmalloc/snmalloc.h +++ b/src/snmalloc/snmalloc.h @@ -3,8 +3,22 @@ // Core implementation of snmalloc independent of the configuration mode #include "snmalloc_core.h" -// If the user has defined SNMALLOC_PROVIDE_OWN_CONFIG, this include does -// nothing. Otherwise, it provide a default configuration of snmalloc::Alloc. +// Provides the global configuration for the snmalloc implementation. #include "backend/globalconfig.h" + +// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own +// definition of `snmalloc::Alloc` before including any files that include +// `snmalloc.h` or consume the global allocation APIs. +#ifndef SNMALLOC_PROVIDE_OWN_CONFIG +namespace snmalloc +{ + /** + * Create allocator type for this configuration. + */ + using Alloc = snmalloc::LocalAllocator< + snmalloc::StandardConfigClientMeta>; +} // namespace snmalloc +#endif + // User facing API surface, needs to know what `Alloc` is. #include "snmalloc_front.h" diff --git a/src/snmalloc/snmalloc_front.h b/src/snmalloc/snmalloc_front.h index 7a16b6c75..4c5aa60ca 100644 --- a/src/snmalloc/snmalloc_front.h +++ b/src/snmalloc/snmalloc_front.h @@ -1,2 +1 @@ #include "global/global.h" -#include "override/libc.h" \ No newline at end of file diff --git a/src/test/func/client_meta/client_meta.cc b/src/test/func/client_meta/client_meta.cc new file mode 100644 index 000000000..88f3cc319 --- /dev/null +++ b/src/test/func/client_meta/client_meta.cc @@ -0,0 +1,68 @@ +/** + * This test performs a very simple use of the client_meta data feature in + * snmalloc. + */ + +#include "test/setup.h" + +#include +#include +#include +#include + +namespace snmalloc +{ + // Create an allocator that stores an std::atomic> per allocation. + using Alloc = snmalloc::LocalAllocator>>>; +} +#define SNMALLOC_PROVIDE_OWN_CONFIG +#include + +int main() +{ +#ifdef SNMALLOC_PASS_THROUGH + // This test does not make sense in pass-through + return 0; +#else + // Allocate a bunch of objects, and store the index into the meta-data. + std::vector ptrs; + for (size_t i = 0; i < 10000; i++) + { + auto p = snmalloc::libc::malloc(1024); + auto& meta = snmalloc::libc::get_client_meta_data(p); + meta = i; + ptrs.push_back(p); + memset(p, (uint8_t)i, 1024); + } + + // Check meta-data contains expected value, and that the memory contains + // the expected pattern. + for (size_t i = 0; i < 10000; i++) + { + auto p = ptrs[i]; + auto& meta = snmalloc::libc::get_client_meta_data(p); + if (meta != i) + { + std::cout << "Failed at index " << i << std::endl; + abort(); + } + for (size_t j = 0; j < 1024; j++) + { + if (reinterpret_cast(p)[j] != (uint8_t)i) + { + std::cout << "Failed at index " << i << " byte " << j << std::endl; + abort(); + } + } + snmalloc::libc::free(p); + } + + // Access in a read-only way meta-data associated with the stack. + // This would fail if it was accessed for write. + auto& meta = snmalloc::libc::get_client_meta_data_const(&ptrs); + std::cout << "meta for stack" << meta << std::endl; + + return 0; +#endif +} diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index 8ff4fa977..aa84ecdf2 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -23,7 +23,8 @@ namespace snmalloc { public: using Pal = DefaultPal; - using PagemapEntry = DefaultPagemapEntry; + using PagemapEntry = DefaultPagemapEntry; + using ClientMeta = NoClientMetaDataProvider; private: using ConcretePagemap = diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 1d4c31da9..6549e5834 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -375,6 +375,6 @@ int main(int argc, char** argv) our_malloc_usable_size(nullptr) == 0, "malloc_usable_size(nullptr) should be zero"); - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); return 0; } diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 8b3606012..7d176f43d 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -184,7 +184,7 @@ void test_calloc() alloc.dealloc(p, size); } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } void test_double_alloc() @@ -229,7 +229,7 @@ void test_double_alloc() } } } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } void test_external_pointer() @@ -275,7 +275,7 @@ void test_external_pointer() alloc.dealloc(p1, size); } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); }; void check_offset(void* base, void* interior) diff --git a/src/test/func/miracle_ptr/miracle_ptr.cc b/src/test/func/miracle_ptr/miracle_ptr.cc new file mode 100644 index 000000000..6d4ea26af --- /dev/null +++ b/src/test/func/miracle_ptr/miracle_ptr.cc @@ -0,0 +1,203 @@ +/** + * This file demonstrates how the snmalloc library could be implemented to + * provide a miracle pointer like feature. This is not a hardened + * implementation and is purely for illustrative purposes. + * + * Do not use as is. + */ + +#ifdef SNMALLOC_THREAD_SANITIZER_ENABLED +int main() +{ + return 0; +} +#else + +# include "test/setup.h" + +# include +# include +# include +# include + +namespace snmalloc +{ + // Instantiate the allocator with a client meta data provider that uses an + // atomic size_t to store the reference count. + using Alloc = snmalloc::LocalAllocator>>>; +} +# define SNMALLOC_PROVIDE_OWN_CONFIG +# include + +SNMALLOC_SLOW_PATH void error(std::string msg) +{ + std::cout << msg << std::endl; + abort(); +} + +SNMALLOC_FAST_PATH_INLINE void check(bool b, std::string msg) +{ + if (SNMALLOC_UNLIKELY(!b)) + error(msg); +} + +namespace snmalloc::miracle +{ + // snmalloc meta-data representation + // * 2n + 1: Represents an object that has not been deallocated with n + // additional references to it + // * 2n : Represents a deallocated object that + // has n additional references to it + + inline void* malloc(size_t size) + { + auto p = snmalloc::libc::malloc(size); + if (SNMALLOC_UNLIKELY(p == nullptr)) + return nullptr; + + snmalloc::libc::get_client_meta_data(p) = 1; + return p; + } + + inline void free(void* ptr) + { + if (ptr == nullptr) + return; + + // TODO could build a check into this that it is the start of the object? + auto previous = + snmalloc::libc::get_client_meta_data(ptr).fetch_add((size_t)-1); + + if (SNMALLOC_LIKELY(previous == 1)) + { + std::cout << "Freeing " << ptr << std::endl; + snmalloc::libc::free(ptr); + return; + } + + check((previous & 1) == 1, "Double free detected"); + + // We have additional references to this object. + // We should not free it. + // TOOD this assumes this is not an internal pointer. + memset(ptr, 0, snmalloc::libc::malloc_usable_size(ptr)); + } + + inline void acquire(void* p) + { + auto previous = + snmalloc::libc::get_client_meta_data(p).fetch_add((size_t)2); + + // Can we take new pointers to a deallocated object? + check((previous & 1) == 1, "Acquiring a deallocated object"); + } + + inline void release(void* p) + { + auto previous = + snmalloc::libc::get_client_meta_data(p).fetch_add((size_t)-2); + + if (previous > 2) + return; + + check(previous == 2, "Releasing an object with insufficient references"); + + std::cout << "Freeing from release " << p << std::endl; + snmalloc::libc::free(p); + } + + /** + * This class can be used to replace a raw pointer. It will automatically use + * the underlying backup reference counting design from the miracle pointer + * docs. + */ + template + class raw_ptr + { + T* p; + + public: + raw_ptr() : p(nullptr) {} + + raw_ptr(T* p) : p(p) + { + snmalloc::miracle::acquire(p); + } + + T& operator*() + { + return *p; + } + + ~raw_ptr() + { + if (p == nullptr) + return; + snmalloc::miracle::release(p); + } + + raw_ptr(const raw_ptr& rp) : p(rp.p) + { + snmalloc::miracle::acquire(p); + } + + raw_ptr& operator=(const raw_ptr& other) + { + p = other.p; + snmalloc::miracle::acquire(other.p); + return *this; + } + + raw_ptr(raw_ptr&& other) : p(other.p) + { + other.p = nullptr; + } + + raw_ptr& operator=(raw_ptr&& other) + { + p = other.p; + other.p = nullptr; + return *this; + } + }; +} // namespace snmalloc::miracle + +/** + * Overload new and delete to use the "miracle pointer" implementation. + */ +void* operator new(size_t size) +{ + return snmalloc::miracle::malloc(size); +} + +void operator delete(void* p) +{ + snmalloc::miracle::free(p); +} + +void operator delete(void* p, size_t) +{ + snmalloc::miracle::free(p); +} + +int main() +{ +# ifndef SNMALLOC_PASS_THROUGH + snmalloc::miracle::raw_ptr p; + { + auto up1 = std::make_unique(41); + auto up = std::make_unique(42); + auto up2 = std::make_unique(40); + auto up3 = std::make_unique(39); + p = up.get(); + check(*p == 42, "Failed to set p"); + } + // Still safe to access here. The unique_ptr has been destroyed, but the + // raw_ptr has kept the memory live. + // Current implementation zeros the memory when the unique_ptr is destroyed. + check(*p == 0, "Failed to keep memory live"); +# endif + return 0; +} +#endif \ No newline at end of file diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index c8db1cad7..214a0bcf3 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -17,7 +17,7 @@ void debug_check_empty_1() auto r = a.alloc(size); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != false) { std::cout << "debug_check_empty failed to detect leaked memory:" << size @@ -27,7 +27,7 @@ void debug_check_empty_1() a.dealloc(r); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != true) { std::cout << "debug_check_empty failed to say empty:" << size << std::endl; @@ -36,7 +36,7 @@ void debug_check_empty_1() r = a.alloc(size); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != false) { std::cout << "debug_check_empty failed to detect leaked memory:" << size @@ -46,7 +46,7 @@ void debug_check_empty_1() a.dealloc(r); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != true) { std::cout << "debug_check_empty failed to say empty:" << size << std::endl; @@ -72,7 +72,7 @@ void debug_check_empty_2() } auto r = a.alloc(size); allocs.push_back(r); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != false) { std::cout << "False empty after " << i << " allocations of " << size @@ -88,7 +88,7 @@ void debug_check_empty_2() { std::cout << "." << std::flush; } - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != false) { std::cout << "False empty after " << i << " deallocations of " << size @@ -98,7 +98,7 @@ void debug_check_empty_2() a.dealloc(allocs[i]); } std::cout << std::endl; - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } int main() diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index 2b10ed8cb..686c08dc4 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -12,7 +12,8 @@ namespace snmalloc { - using Alloc = snmalloc::LocalAllocator; + using Alloc = snmalloc::LocalAllocator< + snmalloc::StandardConfigClientMeta>; } using namespace snmalloc; diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index e266f0491..9e12b660a 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -154,7 +154,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) } #ifndef NDEBUG - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); #endif }; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index be3306cba..96d465820 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -47,7 +47,7 @@ namespace test alloc.dealloc(objects[i]); } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } void test_external_pointer(xoroshiro::p128r64& r) diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc index e3bee7d2c..763dcd72e 100644 --- a/src/test/perf/memcpy/memcpy.cc +++ b/src/test/perf/memcpy/memcpy.cc @@ -1,5 +1,4 @@ -#include "snmalloc/global/memcpy.h" - +#include #include #include #include diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index b93dcd428..b8a995fb8 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -60,7 +60,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) } } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } int main(int, char**)