Configurable client meta-data

.github/workflows/main.yml

@@ -224,15 +224,14 @@ jobs:
       matrix:
         # Build just release variant as Debug is too slow.
         build-type: [ Release ]
+        os: ["ubuntu-latest", "ubuntu-20.04"]
         include:
           - os: "ubuntu-latest"
-            continue-on-error: # Don't class as an error if this fails, until we have a more reliablity.
             variant: "libc++ (TSan + UBSan)"
             dependencies: "sudo apt install ninja-build"
             extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread"
           # Also test specifically with clang-10 (on ubuntu-20.04)
           - os: "ubuntu-20.04"
-            continue-on-error: # Don't class as an error if this fails, until we have a more reliablity.
             variant: "clang-10 libc++ (TSan + UBSan)"
             dependencies: "sudo apt install ninja-build"
             extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread"
@@ -452,7 +451,7 @@ jobs:
         git diff --exit-code
     - name: Run clang-tidy
       run: |
-        clang-tidy-15  src/snmalloc/override/malloc.cc  -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0
+        clang-tidy-15  src/snmalloc/override/malloc.cc  -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 -Isrc
         if [ -f tidy.fail ] ; then
           cat tidy.fail
           exit 1

CMakeLists.txt

@@ -331,6 +331,9 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY)
         if(SNMALLOC_SANITIZER)
           target_compile_options(${TESTNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer)
           target_link_libraries(${TESTNAME} -fsanitize=${SNMALLOC_SANITIZER})
+          if (${SNMALLOC_SANITIZER} MATCHES "thread")
+            target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_THREAD_SANITIZER_ENABLED)
+          endif()
         endif()
 
         add_warning_flags(${TESTNAME})

src/snmalloc/backend/backend.h

@@ -23,9 +23,6 @@ namespace snmalloc
     using Pal = PAL;
     using SlabMetadata = typename PagemapEntry::SlabMetadata;
 
-    static constexpr size_t SizeofMetadata =
-      bits::next_pow2_const(sizeof(SlabMetadata));
-
   public:
     /**
      * Provide a block of meta-data with size and align.
@@ -90,13 +87,26 @@ namespace snmalloc
      *   (remote, sizeclass, slab_metadata)
      * where slab_metadata, is the second element of the pair return.
      */
-    static std::pair<capptr::Chunk<void>, SlabMetadata*>
-    alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras)
+    static std::pair<capptr::Chunk<void>, SlabMetadata*> alloc_chunk(
+      LocalState& local_state,
+      size_t size,
+      uintptr_t ras,
+      sizeclass_t sizeclass)
     {
       SNMALLOC_ASSERT(bits::is_pow2(size));
       SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
 
-      auto meta_cap = local_state.get_meta_range().alloc_range(SizeofMetadata);
+      // Calculate the extra bytes required to store the client meta-data.
+      size_t extra_bytes = SlabMetadata::get_extra_bytes(sizeclass);
+
+      auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes);
+
+#ifdef SNMALLOC_TRACING
+      message<1024>(
+        "Allocating metadata of size: {} ({})", meta_size, extra_bytes);
+#endif
+
+      auto meta_cap = local_state.get_meta_range().alloc_range(meta_size);
 
       auto meta = meta_cap.template as_reinterpret<SlabMetadata>().unsafe_ptr();
 
@@ -113,7 +123,7 @@ namespace snmalloc
 #endif
       if (p == nullptr)
       {
-        local_state.get_meta_range().dealloc_range(meta_cap, SizeofMetadata);
+        local_state.get_meta_range().dealloc_range(meta_cap, meta_size);
         errno = ENOMEM;
 #ifdef SNMALLOC_TRACING
         message<1024>("Out of memory");
@@ -140,7 +150,8 @@ namespace snmalloc
       LocalState& local_state,
       SlabMetadata& slab_metadata,
       capptr::Alloc<void> alloc,
-      size_t size)
+      size_t size,
+      sizeclass_t sizeclass)
     {
       /*
        * The backend takes possession of these chunks now, by disassociating
@@ -167,8 +178,12 @@ namespace snmalloc
        */
       capptr::Arena<void> arena = Authmap::amplify(alloc);
 
+      // Calculate the extra bytes required to store the client meta-data.
+      size_t extra_bytes = SlabMetadata::get_extra_bytes(sizeclass);
+
+      auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes);
       local_state.get_meta_range().dealloc_range(
-        capptr::Arena<void>::unsafe_from(&slab_metadata), SizeofMetadata);
+        capptr::Arena<void>::unsafe_from(&slab_metadata), meta_size);
 
       local_state.get_object_range()->dealloc_range(arena, size);
     }

src/snmalloc/backend/fixedglobalconfig.h

@@ -8,11 +8,14 @@ namespace snmalloc
   /**
    * A single fixed address range allocator configuration
    */
-  template<SNMALLOC_CONCEPT(IsPAL) PAL>
+  template<
+    SNMALLOC_CONCEPT(IsPAL) PAL,
+    typename ClientMetaDataProvider = NoClientMetaDataProvider>
   class FixedRangeConfig final : public CommonConfig
   {
   public:
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<ClientMetaDataProvider>;
+    using ClientMeta = ClientMetaDataProvider;
 
   private:
     using ConcretePagemap =

src/snmalloc/backend/globalconfig.h

@@ -1,13 +1,9 @@
 #pragma once
-// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own
-// definition of `snmalloc::Alloc` before including any files that include
-// `snmalloc.h` or consume the global allocation APIs.
-#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
 
-#  include "../backend_helpers/backend_helpers.h"
-#  include "backend.h"
-#  include "meta_protected_range.h"
-#  include "standard_range.h"
+#include "../backend_helpers/backend_helpers.h"
+#include "backend.h"
+#include "meta_protected_range.h"
+#include "standard_range.h"
 
 namespace snmalloc
 {
@@ -28,13 +24,16 @@ namespace snmalloc
    * The Configuration sets up a Pagemap for the backend to use, and the state
    * required to build new allocators (GlobalPoolState).
    */
-  class StandardConfig final : public CommonConfig
+  template<typename ClientMetaDataProvider = NoClientMetaDataProvider>
+  class StandardConfigClientMeta final : public CommonConfig
   {
-    using GlobalPoolState = PoolState<CoreAllocator<StandardConfig>>;
+    using GlobalPoolState = PoolState<
+      CoreAllocator<StandardConfigClientMeta<ClientMetaDataProvider>>>;
 
   public:
     using Pal = DefaultPal;
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<ClientMetaDataProvider>;
+    using ClientMeta = ClientMetaDataProvider;
 
   private:
     using ConcretePagemap =
@@ -98,9 +97,9 @@ namespace snmalloc
     SNMALLOC_SLOW_PATH static void ensure_init_slow()
     {
       FlagLock lock{initialisation_lock};
-#  ifdef SNMALLOC_TRACING
+#ifdef SNMALLOC_TRACING
       message<1024>("Run init_impl");
-#  endif
+#endif
 
       if (initialised)
         return;
@@ -162,10 +161,4 @@ namespace snmalloc
       snmalloc::register_clean_up();
     }
   };
-
-  /**
-   * Create allocator type for this configuration.
-   */
-  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfig>;
 } // namespace snmalloc
-#endif

src/snmalloc/backend_helpers/commonconfig.h

@@ -95,6 +95,39 @@ namespace snmalloc
     bool HasDomesticate = false;
   };
 
+  struct NoClientMetaDataProvider
+  {
+    using StorageType = Empty;
+    using DataRef = Empty&;
+
+    static size_t required_count(size_t)
+    {
+      return 1;
+    }
+
+    static DataRef get(StorageType* base, size_t)
+    {
+      return *base;
+    }
+  };
+
+  template<typename T>
+  struct ArrayClientMetaDataProvider
+  {
+    using StorageType = T;
+    using DataRef = T&;
+
+    static size_t required_count(size_t max_count)
+    {
+      return max_count;
+    }
+
+    static DataRef get(StorageType* base, size_t index)
+    {
+      return base[index];
+    }
+  };
+
   /**
    * Class containing definitions that are likely to be used by all except for
    * the most unusual back-end implementations.  This can be subclassed as a

src/snmalloc/backend_helpers/defaultpagemapentry.h

@@ -64,9 +64,14 @@ namespace snmalloc
     SNMALLOC_FAST_PATH DefaultPagemapEntryT() = default;
   };
 
-  class DefaultSlabMetadata : public FrontendSlabMetadata<DefaultSlabMetadata>
+  template<typename ClientMetaDataProvider>
+  class DefaultSlabMetadata : public FrontendSlabMetadata<
+                                DefaultSlabMetadata<ClientMetaDataProvider>,
+                                ClientMetaDataProvider>
   {};
 
-  using DefaultPagemapEntry = DefaultPagemapEntryT<DefaultSlabMetadata>;
+  template<typename ClientMetaDataProvider>
+  using DefaultPagemapEntry =
+    DefaultPagemapEntryT<DefaultSlabMetadata<ClientMetaDataProvider>>;
 
 } // namespace snmalloc

src/snmalloc/ds/mpmcstack.h

@@ -4,12 +4,6 @@
 #include "aba.h"
 #include "allocconfig.h"
 
-#if defined(__has_feature)
-#  if __has_feature(thread_sanitizer)
-#    define SNMALLOC_THREAD_SANITIZER_ENABLED
-#  endif
-#endif
-
 namespace snmalloc
 {
   template<class T, Construction c = RequiresInit>

src/snmalloc/ds/pagemap.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "../ds_core/ds_core.h"
+
 namespace snmalloc
 {
   /**
@@ -179,7 +181,13 @@ namespace snmalloc
       // Allocate a power of two extra to allow the placement of the
       // pagemap be difficult to guess if randomize_position set.
       size_t additional_size =
+#ifdef SNMALLOC_THREAD_SANITIZER_ENABLED
+        // When running with TSAN we failed to allocate the very large range
+        // randomly
+        randomize_position ? bits::next_pow2(REQUIRED_SIZE) : 0;
+#else
         randomize_position ? bits::next_pow2(REQUIRED_SIZE) * 4 : 0;
+#endif
       size_t request_size = REQUIRED_SIZE + additional_size;
 
       auto new_body_untyped = PAL::reserve(request_size);

src/snmalloc/global/global.h

@@ -1,4 +1,5 @@
 #include "bounds_checks.h"
+#include "libc.h"
 #include "memcpy.h"
 #include "scopedalloc.h"
 #include "threadalloc.h"

src/snmalloc/override/libc.h → src/snmalloc/global/libc.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "../global/global.h"
+#include "threadalloc.h"
 
 #include <errno.h>
 #include <string.h>
@@ -176,4 +176,17 @@ namespace snmalloc::libc
     *memptr = p;
     return 0;
   }
+
+  inline typename snmalloc::Alloc::Config::ClientMeta::DataRef
+  get_client_meta_data(void* p)
+  {
+    return ThreadAlloc::get().get_client_meta_data(p);
+  }
+
+  inline std::add_const_t<typename snmalloc::Alloc::Config::ClientMeta::DataRef>
+  get_client_meta_data_const(void* p)
+  {
+    return ThreadAlloc::get().get_client_meta_data_const(p);
+  }
+
 } // namespace snmalloc::libc

src/snmalloc/global/memcpy.h

@@ -1,5 +1,4 @@
 #pragma once
-#include "../backend/globalconfig.h"
 #include "bounds_checks.h"
 
 namespace snmalloc

src/snmalloc/global/scopedalloc.h

@@ -1,5 +1,4 @@
 #pragma once
-#include "../backend/globalconfig.h"
 
 /**
  * This header requires that Alloc has been defined.

src/snmalloc/global/threadalloc.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "../backend/globalconfig.h"
-
 #if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC)
 #  define SNMALLOC_THREAD_TEARDOWN_DEFINED
 #endif

src/snmalloc/mem/backend_concept.h

@@ -2,6 +2,7 @@
 
 #ifdef __cpp_concepts
 #  include "../ds/ds.h"
+#  include "sizeclasstable.h"
 
 #  include <cstddef>
 namespace snmalloc
@@ -97,9 +98,13 @@ namespace snmalloc
 
   template<typename LocalState, typename PagemapEntry, typename Backend>
   concept IsBackend =
-    requires(LocalState& local_state, size_t size, uintptr_t ras) {
+    requires(
+      LocalState& local_state,
+      size_t size,
+      uintptr_t ras,
+      sizeclass_t sizeclass) {
       {
-        Backend::alloc_chunk(local_state, size, ras)
+        Backend::alloc_chunk(local_state, size, ras, sizeclass)
         } -> ConceptSame<
           std::pair<capptr::Chunk<void>, typename Backend::SlabMetadata*>>;
     } &&
@@ -112,9 +117,11 @@ namespace snmalloc
       LocalState& local_state,
       typename Backend::SlabMetadata& slab_metadata,
       capptr::Alloc<void> alloc,
-      size_t size) {
+      size_t size,
+      sizeclass_t sizeclass) {
       {
-        Backend::dealloc_chunk(local_state, slab_metadata, alloc, size)
+        Backend::dealloc_chunk(
+          local_state, slab_metadata, alloc, size, sizeclass)
         } -> ConceptSame<void>;
     } &&
     requires(address_t p) {

src/snmalloc/mem/corealloc.h

@@ -368,7 +368,8 @@ namespace snmalloc
           get_backend_local_state(),
           *meta,
           start,
-          sizeclass_to_slab_size(sizeclass));
+          sizeclass_to_slab_size(sizeclass),
+          sizeclass_t::from_small_class(sizeclass));
       });
     }
 
@@ -401,7 +402,7 @@ namespace snmalloc
         meta->node.remove();
 
         Config::Backend::dealloc_chunk(
-          get_backend_local_state(), *meta, p, size);
+          get_backend_local_state(), *meta, p, size, entry.get_sizeclass());
 
         return;
       }
@@ -796,7 +797,8 @@ namespace snmalloc
         get_backend_local_state(),
         slab_size,
         PagemapEntry::encode(
-          public_state(), sizeclass_t::from_small_class(sizeclass)));
+          public_state(), sizeclass_t::from_small_class(sizeclass)),
+        sizeclass_t::from_small_class(sizeclass));
 
       if (slab == nullptr)
       {