Configurable client meta-data

.github/workflows/main.yml

@@ -224,15 +224,14 @@ jobs:
       matrix:
         # Build just release variant as Debug is too slow.
         build-type: [ Release ]
+        os: ["ubuntu-latest", "ubuntu-20.04"]
         include:
           - os: "ubuntu-latest"
-            continue-on-error: # Don't class as an error if this fails, until we have a more reliablity.
             variant: "libc++ (TSan + UBSan)"
             dependencies: "sudo apt install ninja-build"
             extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread"
           # Also test specifically with clang-10 (on ubuntu-20.04)
           - os: "ubuntu-20.04"
-            continue-on-error: # Don't class as an error if this fails, until we have a more reliablity.
             variant: "clang-10 libc++ (TSan + UBSan)"
             dependencies: "sudo apt install ninja-build"
             extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread"
@@ -452,7 +451,7 @@ jobs:
         git diff --exit-code
     - name: Run clang-tidy
       run: |
-        clang-tidy-15  src/snmalloc/override/malloc.cc  -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0
+        clang-tidy-15  src/snmalloc/override/malloc.cc  -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 -Isrc
         if [ -f tidy.fail ] ; then
           cat tidy.fail
           exit 1

src/snmalloc/backend/backend.h

@@ -23,9 +23,6 @@ namespace snmalloc
     using Pal = PAL;
     using SlabMetadata = typename PagemapEntry::SlabMetadata;
 
-    static constexpr size_t SizeofMetadata =
-      bits::next_pow2_const(sizeof(SlabMetadata));
-
   public:
     /**
      * Provide a block of meta-data with size and align.
@@ -90,13 +87,23 @@ namespace snmalloc
      *   (remote, sizeclass, slab_metadata)
      * where slab_metadata, is the second element of the pair return.
      */
-    static std::pair<capptr::Chunk<void>, SlabMetadata*>
-    alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras)
+    static std::pair<capptr::Chunk<void>, SlabMetadata*> alloc_chunk(
+      LocalState& local_state,
+      size_t size,
+      uintptr_t ras,
+      size_t extra_bytes = 0)
     {
       SNMALLOC_ASSERT(bits::is_pow2(size));
       SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
 
-      auto meta_cap = local_state.get_meta_range().alloc_range(SizeofMetadata);
+      auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes);
+
+#ifdef SNMALLOC_TRACING
+      message<1024>(
+        "Allocating metadata of size: {} ({})", meta_size, extra_bytes);
+#endif
+
+      auto meta_cap = local_state.get_meta_range().alloc_range(meta_size);
 
       auto meta = meta_cap.template as_reinterpret<SlabMetadata>().unsafe_ptr();
 
@@ -113,7 +120,7 @@ namespace snmalloc
 #endif
       if (p == nullptr)
       {
-        local_state.get_meta_range().dealloc_range(meta_cap, SizeofMetadata);
+        local_state.get_meta_range().dealloc_range(meta_cap, meta_size);
         errno = ENOMEM;
 #ifdef SNMALLOC_TRACING
         message<1024>("Out of memory");
@@ -140,7 +147,8 @@ namespace snmalloc
       LocalState& local_state,
       SlabMetadata& slab_metadata,
       capptr::Alloc<void> alloc,
-      size_t size)
+      size_t size,
+      size_t extra_bytes = 0)
     {
       /*
        * The backend takes possession of these chunks now, by disassociating
@@ -167,8 +175,9 @@ namespace snmalloc
        */
       capptr::Arena<void> arena = Authmap::amplify(alloc);
 
+      auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes);
       local_state.get_meta_range().dealloc_range(
-        capptr::Arena<void>::unsafe_from(&slab_metadata), SizeofMetadata);
+        capptr::Arena<void>::unsafe_from(&slab_metadata), meta_size);
 
       local_state.get_object_range()->dealloc_range(arena, size);
     }

src/snmalloc/backend/fixedglobalconfig.h

@@ -8,11 +8,14 @@ namespace snmalloc
   /**
    * A single fixed address range allocator configuration
    */
-  template<SNMALLOC_CONCEPT(IsPAL) PAL>
+  template<
+    SNMALLOC_CONCEPT(IsPAL) PAL,
+    typename ClientMetaDataProvider = NoClientMetaDataProvider>
   class FixedRangeConfig final : public CommonConfig
   {
   public:
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<ClientMetaDataProvider>;
+    using ClientMeta = ClientMetaDataProvider;
 
   private:
     using ConcretePagemap =

src/snmalloc/backend/globalconfig.h

@@ -1,13 +1,9 @@
 #pragma once
-// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own
-// definition of `snmalloc::Alloc` before including any files that include
-// `snmalloc.h` or consume the global allocation APIs.
-#ifndef SNMALLOC_PROVIDE_OWN_CONFIG
 
-#  include "../backend_helpers/backend_helpers.h"
-#  include "backend.h"
-#  include "meta_protected_range.h"
-#  include "standard_range.h"
+#include "../backend_helpers/backend_helpers.h"
+#include "backend.h"
+#include "meta_protected_range.h"
+#include "standard_range.h"
 
 namespace snmalloc
 {
@@ -28,13 +24,16 @@ namespace snmalloc
    * The Configuration sets up a Pagemap for the backend to use, and the state
    * required to build new allocators (GlobalPoolState).
    */
-  class StandardConfig final : public CommonConfig
+  template<typename ClientMetaDataProvider = NoClientMetaDataProvider>
+  class StandardConfigClientMeta final : public CommonConfig
   {
-    using GlobalPoolState = PoolState<CoreAllocator<StandardConfig>>;
+    using GlobalPoolState = PoolState<
+      CoreAllocator<StandardConfigClientMeta<ClientMetaDataProvider>>>;
 
   public:
     using Pal = DefaultPal;
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<ClientMetaDataProvider>;
+    using ClientMeta = ClientMetaDataProvider;
 
   private:
     using ConcretePagemap =
@@ -98,9 +97,9 @@ namespace snmalloc
     SNMALLOC_SLOW_PATH static void ensure_init_slow()
     {
       FlagLock lock{initialisation_lock};
-#  ifdef SNMALLOC_TRACING
+#ifdef SNMALLOC_TRACING
       message<1024>("Run init_impl");
-#  endif
+#endif
 
       if (initialised)
         return;
@@ -162,10 +161,4 @@ namespace snmalloc
       snmalloc::register_clean_up();
     }
   };
-
-  /**
-   * Create allocator type for this configuration.
-   */
-  using Alloc = snmalloc::LocalAllocator<snmalloc::StandardConfig>;
 } // namespace snmalloc
-#endif

src/snmalloc/backend_helpers/commonconfig.h

@@ -95,6 +95,41 @@ namespace snmalloc
     bool HasDomesticate = false;
   };
 
+  struct NoClientMetaDataProvider
+  {
+    using StorageType = Empty;
+    using DataRef = Empty&;
+    using ConstDataRef = const Empty&;
+
+    static size_t required_count(size_t)
+    {
+      return 1;
+    }
+
+    static DataRef get(StorageType* base, size_t)
+    {
+      return *base;
+    }
+  };
+
+  template<typename T>
+  struct ArrayClientMetaDataProvider
+  {
+    using StorageType = T;
+    using DataRef = T&;
+    using ConstDataRef = const T&;
+
+    static size_t required_count(size_t max_count)
+    {
+      return max_count;
+    }
+
+    static DataRef get(StorageType* base, size_t index)
+    {
+      return base[index];
+    }
+  };
+
   /**
    * Class containing definitions that are likely to be used by all except for
    * the most unusual back-end implementations.  This can be subclassed as a

src/snmalloc/backend_helpers/defaultpagemapentry.h

@@ -64,9 +64,14 @@ namespace snmalloc
     SNMALLOC_FAST_PATH DefaultPagemapEntryT() = default;
   };
 
-  class DefaultSlabMetadata : public FrontendSlabMetadata<DefaultSlabMetadata>
+  template<typename ClientMetaDataProvider>
+  class DefaultSlabMetadata : public FrontendSlabMetadata<
+                                DefaultSlabMetadata<ClientMetaDataProvider>,
+                                ClientMetaDataProvider>
   {};
 
-  using DefaultPagemapEntry = DefaultPagemapEntryT<DefaultSlabMetadata>;
+  template<typename ClientMetaDataProvider>
+  using DefaultPagemapEntry =
+    DefaultPagemapEntryT<DefaultSlabMetadata<ClientMetaDataProvider>>;
 
 } // namespace snmalloc

src/snmalloc/global/global.h

@@ -1,4 +1,5 @@
 #include "bounds_checks.h"
+#include "libc.h"
 #include "memcpy.h"
 #include "scopedalloc.h"
 #include "threadalloc.h"

src/snmalloc/override/libc.h → src/snmalloc/global/libc.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "../global/global.h"
+#include "threadalloc.h"
 
 #include <errno.h>
 #include <string.h>
@@ -176,4 +176,17 @@ namespace snmalloc::libc
     *memptr = p;
     return 0;
   }
+
+  inline typename snmalloc::Alloc::Config::ClientMeta::DataRef
+  get_client_meta_data(void* p)
+  {
+    return ThreadAlloc::get().get_client_meta_data(p);
+  }
+
+  inline typename snmalloc::Alloc::Config::ClientMeta::ConstDataRef
+  get_client_meta_data_const(void* p)
+  {
+    return ThreadAlloc::get().get_client_meta_data_const(p);
+  }
+
 } // namespace snmalloc::libc

src/snmalloc/global/memcpy.h

@@ -1,5 +1,4 @@
 #pragma once
-#include "../backend/globalconfig.h"
 #include "bounds_checks.h"
 
 namespace snmalloc

src/snmalloc/global/scopedalloc.h

@@ -1,5 +1,4 @@
 #pragma once
-#include "../backend/globalconfig.h"
 
 /**
  * This header requires that Alloc has been defined.

src/snmalloc/global/threadalloc.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "../backend/globalconfig.h"
-
 #if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC)
 #  define SNMALLOC_THREAD_TEARDOWN_DEFINED
 #endif

src/snmalloc/mem/corealloc.h

@@ -364,11 +364,15 @@ namespace snmalloc
         // don't touch the cache lines at this point in snmalloc_check_client.
         auto start = clear_slab(meta, sizeclass);
 
+        // Calculate the extra bytes required to store the client meta-data.
+        size_t extra_bytes = BackendSlabMetadata::get_extra_bytes(sizeclass);
+
         Config::Backend::dealloc_chunk(
           get_backend_local_state(),
           *meta,
           start,
-          sizeclass_to_slab_size(sizeclass));
+          sizeclass_to_slab_size(sizeclass),
+          extra_bytes);
       });
     }
 
@@ -401,7 +405,7 @@ namespace snmalloc
         meta->node.remove();
 
         Config::Backend::dealloc_chunk(
-          get_backend_local_state(), *meta, p, size);
+          get_backend_local_state(), *meta, p, size, 0);
 
         return;
       }
@@ -792,11 +796,15 @@ namespace snmalloc
       message<1024>("small_alloc_slow rsize={} slab size={}", rsize, slab_size);
 #endif
 
+      // Calculate the extra bytes required to store the client meta-data.
+      size_t extra_bytes = BackendSlabMetadata::get_extra_bytes(sizeclass);
+
       auto [slab, meta] = Config::Backend::alloc_chunk(
         get_backend_local_state(),
         slab_size,
         PagemapEntry::encode(
-          public_state(), sizeclass_t::from_small_class(sizeclass)));
+          public_state(), sizeclass_t::from_small_class(sizeclass)),
+        extra_bytes);
 
       if (slab == nullptr)
       {

src/snmalloc/mem/localalloc.h

@@ -816,6 +816,51 @@ namespace snmalloc
       }
     }
 
+    /**
+     * @brief Get the client meta data for the snmalloc allocation covering this
+     * pointer.
+     */
+    typename Config::ClientMeta::DataRef get_client_meta_data(void* p)
+    {
+      const PagemapEntry& entry =
+        Config::Backend::template get_metaentry(address_cast(p));
+
+      size_t index = slab_index(entry.get_sizeclass(), address_cast(p));
+
+      auto* meta_slab = entry.get_slab_metadata();
+
+      if (SNMALLOC_UNLIKELY(meta_slab == nullptr))
+      {
+        error(
+          "Cannot access meta-data for non-snmalloc object in writable form!");
+      }
+
+      return meta_slab->get_meta_for_object(index);
+    }
+
+    /**
+     * @brief Get the client meta data for the snmalloc allocation covering this
+     * pointer.
+     */
+    typename Config::ClientMeta::ConstDataRef
+    get_client_meta_data_const(void* p)
+    {
+      const PagemapEntry& entry =
+        Config::Backend::template get_metaentry<true>(address_cast(p));
+
+      size_t index = slab_index(entry.get_sizeclass(), address_cast(p));
+
+      auto* meta_slab = entry.get_slab_metadata();
+
+      if (SNMALLOC_UNLIKELY(meta_slab == nullptr))
+      {
+        static typename Config::ClientMeta::StorageType null_meta_store{};
+        return Config::ClientMeta::get(&null_meta_store, 0);
+      }
+
+      return meta_slab->get_meta_for_object(index);
+    }
+
     /**
      * Returns the number of remaining bytes in an object.
      *