Configurable client meta-data

src/snmalloc/backend/backend.h

@@ -23,9 +23,6 @@ namespace snmalloc
     using Pal = PAL;
     using SlabMetadata = typename PagemapEntry::SlabMetadata;
 
-    static constexpr size_t SizeofMetadata =
-      bits::next_pow2_const(sizeof(SlabMetadata));
-
   public:
     /**
      * Provide a block of meta-data with size and align.
@@ -91,12 +88,18 @@ namespace snmalloc
      * where slab_metadata, is the second element of the pair return.
      */
     static std::pair<capptr::Chunk<void>, SlabMetadata*>
-    alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras)
+    alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras, size_t extra_bytes = 0)
     {
       SNMALLOC_ASSERT(bits::is_pow2(size));
       SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE);
 
-      auto meta_cap = local_state.get_meta_range().alloc_range(SizeofMetadata);
+      auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes);
+
+#ifdef SNMALLOC_TRACING
+      message<1024>("Allocating metadata of size: {} ({})", meta_size, extra_bytes);
+#endif
+
+      auto meta_cap = local_state.get_meta_range().alloc_range(meta_size);
 
       auto meta = meta_cap.template as_reinterpret<SlabMetadata>().unsafe_ptr();
 
@@ -113,7 +116,7 @@ namespace snmalloc
 #endif
       if (p == nullptr)
       {
-        local_state.get_meta_range().dealloc_range(meta_cap, SizeofMetadata);
+        local_state.get_meta_range().dealloc_range(meta_cap, meta_size);
         errno = ENOMEM;
 #ifdef SNMALLOC_TRACING
         message<1024>("Out of memory");
@@ -140,7 +143,8 @@ namespace snmalloc
       LocalState& local_state,
       SlabMetadata& slab_metadata,
       capptr::Alloc<void> alloc,
-      size_t size)
+      size_t size,
+      size_t extra_bytes = 0)
     {
       /*
        * The backend takes possession of these chunks now, by disassociating
@@ -167,8 +171,9 @@ namespace snmalloc
        */
       capptr::Arena<void> arena = Authmap::amplify(alloc);
 
+      auto meta_size = bits::next_pow2(sizeof(SlabMetadata) + extra_bytes);
       local_state.get_meta_range().dealloc_range(
-        capptr::Arena<void>::unsafe_from(&slab_metadata), SizeofMetadata);
+        capptr::Arena<void>::unsafe_from(&slab_metadata), meta_size);
 
       local_state.get_object_range()->dealloc_range(arena, size);
     }

src/snmalloc/backend/fixedglobalconfig.h

@@ -8,11 +8,13 @@ namespace snmalloc
   /**
    * A single fixed address range allocator configuration
    */
-  template<SNMALLOC_CONCEPT(IsPAL) PAL>
+  template<
+    SNMALLOC_CONCEPT(IsPAL) PAL,
+    typename ClientMetaDataProvider = NoClientMetaDataProvider>
   class FixedRangeConfig final : public CommonConfig
   {
   public:
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<ClientMetaDataProvider>;
 
   private:
     using ConcretePagemap =
@@ -63,11 +65,13 @@ namespace snmalloc
      * C++, and not just its initializer fragment, to initialize a non-prefix
      * subset of the flags (in any order, at that).
      */
-    static constexpr Flags Options = []() constexpr {
+    static constexpr Flags Options = []() constexpr
+    {
       Flags opts = {};
       opts.HasDomesticate = true;
       return opts;
-    }();
+    }
+    ();
 
     // This needs to be a forward reference as the
     // thread local state will need to know about this.

src/snmalloc/backend/globalconfig.h

@@ -28,13 +28,14 @@ namespace snmalloc
    * The Configuration sets up a Pagemap for the backend to use, and the state
    * required to build new allocators (GlobalPoolState).
    */
-  class StandardConfig final : public CommonConfig
+  template <typename ClientMetaDataProvider = NoClientMetaDataProvider>
+  class StandardConfigClientMeta final : public CommonConfig
   {
-    using GlobalPoolState = PoolState<CoreAllocator<StandardConfig>>;
+    using GlobalPoolState = PoolState<CoreAllocator<StandardConfigClientMeta<ClientMetaDataProvider>>>;
 
   public:
     using Pal = DefaultPal;
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<ClientMetaDataProvider>;
 
   private:
     using ConcretePagemap =
@@ -163,6 +164,8 @@ namespace snmalloc
     }
   };
 
+  using StandardConfig = StandardConfigClientMeta<NoClientMetaDataProvider>;
+
   /**
    * Create allocator type for this configuration.
    */

src/snmalloc/backend_helpers/commonconfig.h

@@ -95,6 +95,33 @@ namespace snmalloc
     bool HasDomesticate = false;
   };
 
+  struct NoClientMetaDataProvider
+  {
+    using StorageType = Empty;
+
+    static size_t required_count(size_t) {
+      return 1;
+    }
+
+    static Empty& get(Empty* base, size_t) {
+      return *base;
+    }
+  };
+
+  template <typename T>
+  struct ArrayClientMetaDataProvider
+  {
+    using StorageType = T;
+
+    static size_t required_count(size_t max_count) {
+      return max_count;
+    }
+
+    static T& get(StorageType* base, size_t index) {
+      return base[index];
+    }
+  };
+
   /**
    * Class containing definitions that are likely to be used by all except for
    * the most unusual back-end implementations.  This can be subclassed as a

src/snmalloc/backend_helpers/defaultpagemapentry.h

@@ -64,9 +64,11 @@ namespace snmalloc
     SNMALLOC_FAST_PATH DefaultPagemapEntryT() = default;
   };
 
-  class DefaultSlabMetadata : public FrontendSlabMetadata<DefaultSlabMetadata>
+  template <typename ClientMetaDataProvider>
+  class DefaultSlabMetadata : public FrontendSlabMetadata<DefaultSlabMetadata<ClientMetaDataProvider>, ClientMetaDataProvider>
   {};
 
-  using DefaultPagemapEntry = DefaultPagemapEntryT<DefaultSlabMetadata>;
+  template <typename ClientMetaDataProvider>
+  using DefaultPagemapEntry = DefaultPagemapEntryT<DefaultSlabMetadata<ClientMetaDataProvider>>;
 
 } // namespace snmalloc

src/snmalloc/mem/corealloc.h

@@ -364,11 +364,15 @@ namespace snmalloc
         // don't touch the cache lines at this point in snmalloc_check_client.
         auto start = clear_slab(meta, sizeclass);
 
+        // Calculate the extra bytes required to store the client meta-data.
+        size_t extra_bytes = BackendSlabMetadata::get_extra_bytes(sizeclass);
+
         Config::Backend::dealloc_chunk(
           get_backend_local_state(),
           *meta,
           start,
-          sizeclass_to_slab_size(sizeclass));
+          sizeclass_to_slab_size(sizeclass),
+          extra_bytes);
       });
     }
 
@@ -401,7 +405,7 @@ namespace snmalloc
         meta->node.remove();
 
         Config::Backend::dealloc_chunk(
-          get_backend_local_state(), *meta, p, size);
+          get_backend_local_state(), *meta, p, size, 0);
 
         return;
       }
@@ -792,11 +796,16 @@ namespace snmalloc
       message<1024>("small_alloc_slow rsize={} slab size={}", rsize, slab_size);
 #endif
 
+      // Calculate the extra bytes required to store the client meta-data.
+      size_t extra_bytes = BackendSlabMetadata::get_extra_bytes(sizeclass);
+      message<1024>("extra_bytes={}, sizeclass={}", extra_bytes, sizeclass);
+
       auto [slab, meta] = Config::Backend::alloc_chunk(
         get_backend_local_state(),
         slab_size,
         PagemapEntry::encode(
-          public_state(), sizeclass_t::from_small_class(sizeclass)));
+          public_state(), sizeclass_t::from_small_class(sizeclass)),
+        extra_bytes);
 
       if (slab == nullptr)
       {

src/snmalloc/mem/metadata.h

@@ -368,7 +368,7 @@ namespace snmalloc
   class FrontendSlabMetadata_Trait
   {
   private:
-    template<typename BackendType>
+    template<typename BackendType, typename ClientMeta_>
     friend class FrontendSlabMetadata;
 
     // Can only be constructed by FrontendSlabMetadata
@@ -379,10 +379,15 @@ namespace snmalloc
    * The FrontendSlabMetadata represent the metadata associated with a single
    * slab.
    */
-  template<typename BackendType>
+  template<typename BackendType, typename ClientMeta_>
   class FrontendSlabMetadata : public FrontendSlabMetadata_Trait
   {
   public:
+    /**
+     * Type that encapsulates logic for accessing client meta-data.
+     */
+    using ClientMeta = ClientMeta_;
+
     /**
      * Used to link slab metadata together in various other data-structures.
      * This is used with `SeqSet` and so may actually hold a subclass of this
@@ -424,6 +429,13 @@ namespace snmalloc
      */
     bool large_ = false;
 
+    /**
+     * Stores client meta-data for this slab. This must be last element in the
+     * slab. The meta data will actually allocate multiple elements after this
+     * type, so that client_meta_[1] will work for the required meta-data size.
+     */
+    SNMALLOC_NO_UNIQUE_ADDRESS typename ClientMeta::StorageType client_meta_;
+
     uint16_t& needed()
     {
       return needed_;
@@ -452,6 +464,9 @@ namespace snmalloc
       set_sleeping(sizeclass, 0);
 
       large_ = false;
+
+      new (&client_meta_)
+        typename ClientMeta::StorageType[get_client_storage_count(sizeclass)];
     }
 
     /**
@@ -469,6 +484,8 @@ namespace snmalloc
 
       // Jump to slow path on first deallocation.
       needed() = 1;
+
+      new (&client_meta_) typename ClientMeta::StorageType();
     }
 
     /**
@@ -583,6 +600,28 @@ namespace snmalloc
     {
       return address_cast(free_queue.read_head(0, key));
     }
+
+    typename ClientMeta::StorageType* get_client_meta_base()
+    {
+      return &client_meta_;
+    }
+
+    static size_t get_client_storage_count(smallsizeclass_t sizeclass)
+    {
+      auto count = sizeclass_to_slab_object_count(sizeclass);
+      auto result = ClientMeta::required_count(count);
+      if (result == 0)
+        return 1;
+      return result;
+    }
+
+    static size_t get_extra_bytes(smallsizeclass_t sizeclass)
+    {
+      // We remove one from the extra-bytes as there is one in the metadata to
+      // start with.
+      return (get_client_storage_count(sizeclass) - 1) *
+        sizeof(typename ClientMeta::StorageType);
+    }
   };
 
   /**

src/test/func/domestication/domestication.cc

@@ -23,7 +23,7 @@ namespace snmalloc
   {
   public:
     using Pal = DefaultPal;
-    using PagemapEntry = DefaultPagemapEntry;
+    using PagemapEntry = DefaultPagemapEntry<NoClientMetaDataProvider>;
 
   private:
     using ConcretePagemap =