Upgrade snmalloc from 0.6.2 to 0.7.0 (#6746)

3rdparty/exported/snmalloc/CMakeLists.txt

@@ -26,6 +26,9 @@ option(SNMALLOC_NO_REALLOCARR "Build without reallocarr exported" ON)
 option(SNMALLOC_LINK_ICF "Link with Identical Code Folding" ON)
 option(SNMALLOC_IPO "Link with IPO/LTO support" OFF)
 option(SNMALLOC_BENCHMARK_INDIVIDUAL_MITIGATIONS "Build tests and ld_preload for individual mitigations" OFF)
+option(SNMALLOC_ENABLE_DYNAMIC_LOADING "Build such that snmalloc can be dynamically loaded. This is not required for LD_PRELOAD, and will harm performance if enabled." OFF)
+option(SNMALLOC_ENABLE_WAIT_ON_ADDRESS "Use wait on address backoff strategy if it is available" ON)
+option(SNMALLOC_ENABLE_FUZZING "Enable fuzzing instrumentation tests" OFF)
 # Options that apply only if we're not building the header-only library
 cmake_dependent_option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF)
 cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF)
@@ -61,6 +64,18 @@ if (SNMALLOC_SANITIZER)
   message(STATUS "Using sanitizer=${SNMALLOC_SANITIZER}")
 endif()
 
+set(SNMALLOC_MIN_ALLOC_SIZE "" CACHE STRING "Minimum allocation bytes (power of 2)")
+set(SNMALLOC_MIN_ALLOC_STEP_SIZE "" CACHE STRING "Minimum allocation step (power of 2)")
+
+set(SNMALLOC_PAGESIZE "" CACHE STRING "Page size in bytes")
+
+set(SNMALLOC_DEALLOC_BATCH_RING_ASSOC "" CACHE STRING "Associativity of deallocation batch cache; 0 to disable")
+set(SNMALLOC_DEALLOC_BATCH_RING_SET_BITS "" CACHE STRING "Logarithm of number of deallocation batch cache associativity sets")
+
+if(MSVC AND SNMALLOC_STATIC_LIBRARY AND (SNMALLOC_STATIC_LIBRARY_PREFIX STREQUAL ""))
+  message(FATAL_ERROR "Empty static library prefix not supported on MSVC")
+endif()
+
 # If CheckLinkerFlag doesn't exist then provide a dummy implementation that
 # always fails.  The fallback can be removed when we move to CMake 3.18 as the
 # baseline.
@@ -121,6 +136,9 @@ int main() {
 # this is why we check its existence here
 CHECK_INCLUDE_FILE_CXX(linux/random.h SNMALLOC_HAS_LINUX_RANDOM_H)
 
+# check if futex.h is available
+CHECK_INCLUDE_FILE_CXX(linux/futex.h SNMALLOC_HAS_LINUX_FUTEX_H)
+
 # Provide as function so other projects can reuse
 # FIXME: This modifies some variables that may or may not be the ones that
 # provide flags and so is broken by design.  It should be removed once Verona
@@ -150,7 +168,7 @@ function(clangformat_targets)
   # tool.  It does not work with older versions as AfterCaseLabel is not supported
   # in earlier versions.
   find_program(CLANG_FORMAT NAMES
-    clang-format90 clang-format-9)
+    clang-format150 clang-format-15)
 
   # If we've found a clang-format tool, generate a target for it, otherwise emit
   # a warning.
@@ -176,12 +194,27 @@ endfunction()
 add_library(snmalloc INTERFACE)
 
 if(SNMALLOC_USE_CXX17)
-  target_compile_definitions(snmalloc INTERFACE -DSNMALLOC_USE_CXX17)
   target_compile_features(snmalloc INTERFACE cxx_std_17)
 else()
   target_compile_features(snmalloc INTERFACE cxx_std_20)
 endif()
 
+if(SNMALLOC_ENABLE_WAIT_ON_ADDRESS)
+  target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_WAIT_ON_ADDRESS=1)
+else()
+  target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_WAIT_ON_ADDRESS=0)
+endif()
+
+# https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus
+if(MSVC)
+  target_compile_options(snmalloc INTERFACE "/Zc:__cplusplus")
+endif()
+
+if (CMAKE_SYSTEM_NAME STREQUAL NetBSD)
+	target_include_directories(snmalloc INTERFACE /usr/pkg/include)
+	target_link_directories(snmalloc INTERFACE /usr/pkg/lib)
+endif()
+
 # Add header paths.
 target_include_directories(snmalloc
   INTERFACE
@@ -221,18 +254,30 @@ endif()
 function(add_as_define FLAG)
   target_compile_definitions(snmalloc INTERFACE $<$<BOOL:${${FLAG}}>:${FLAG}>)
 endfunction()
+function(add_as_define_value KEY)
+  if (NOT ${${KEY}} STREQUAL "")
+    target_compile_definitions(snmalloc INTERFACE ${KEY}=${${KEY}})
+  endif ()
+endfunction()
 
 add_as_define(SNMALLOC_QEMU_WORKAROUND)
 add_as_define(SNMALLOC_TRACING)
 add_as_define(SNMALLOC_CI_BUILD)
 add_as_define(SNMALLOC_PLATFORM_HAS_GETENTROPY)
 add_as_define(SNMALLOC_HAS_LINUX_RANDOM_H)
+add_as_define(SNMALLOC_HAS_LINUX_FUTEX_H)
 if (SNMALLOC_NO_REALLOCARRAY)
   add_as_define(SNMALLOC_NO_REALLOCARRAY)
 endif()
 if (SNMALLOC_NO_REALLOCARR)
   add_as_define(SNMALLOC_NO_REALLOCARR)
 endif()
+add_as_define_value(SNMALLOC_MIN_ALLOC_SIZE)
+add_as_define_value(SNMALLOC_MIN_ALLOC_STEP_SIZE)
+add_as_define_value(SNMALLOC_DEALLOC_BATCH_RING_ASSOC)
+add_as_define_value(SNMALLOC_DEALLOC_BATCH_RING_SET_BITS)
+
+add_as_define_value(SNMALLOC_PAGESIZE)
 
 target_compile_definitions(snmalloc INTERFACE $<$<BOOL:CONST_QUALIFIED_MALLOC_USABLE_SIZE>:MALLOC_USABLE_SIZE_QUALIFIER=const>)
 
@@ -316,6 +361,9 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY)
         if(SNMALLOC_SANITIZER)
           target_compile_options(${TESTNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer)
           target_link_libraries(${TESTNAME} -fsanitize=${SNMALLOC_SANITIZER})
+          if (${SNMALLOC_SANITIZER} MATCHES "thread")
+            target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_THREAD_SANITIZER_ENABLED)
+          endif()
         endif()
 
         add_warning_flags(${TESTNAME})
@@ -386,8 +434,14 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY)
       target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))")
       target_compile_options(${name} PRIVATE
         -fomit-frame-pointer -ffunction-sections)
+
+      check_cxx_compiler_flag("-Werror -Wextra -Wall -mprfchw" SUPPORT_PREFETCH_WRITE)
+      if (SUPPORT_PREFETCH_WRITE)
+        target_compile_options(${name} PRIVATE -mprfchw)
+      endif()
       # Static TLS model is unsupported on Haiku.
-      if (NOT CMAKE_SYSTEM_NAME STREQUAL "Haiku")
+      if ((NOT CMAKE_SYSTEM_NAME STREQUAL "Haiku") AND (NOT SNMALLOC_ENABLE_DYNAMIC_LOADING))
+        message(STATUS "snmalloc: Using static TLS model")
         target_compile_options(${name} PRIVATE -ftls-model=initial-exec)
         target_compile_options(${name} PRIVATE $<$<BOOL:${SNMALLOC_CI_BUILD}>:-g>)
       endif()
@@ -426,9 +480,11 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY)
 
   endfunction()
 
-  set(SHIM_FILES src/snmalloc/override/new.cc)
+  set(SHIM_FILES src/snmalloc/override/malloc.cc src/snmalloc/override/new.cc)
   set(SHIM_FILES_MEMCPY src/snmalloc/override/memcpy.cc)
 
+  add_shim(snmalloc-new-override STATIC src/snmalloc/override/new.cc)
+
   if (SNMALLOC_STATIC_LIBRARY)
     add_shim(snmallocshim-static STATIC ${SHIM_FILES})
     target_compile_definitions(snmallocshim-static PRIVATE
@@ -549,3 +605,6 @@ install(EXPORT snmallocConfig
   DESTINATION "share/snmalloc"
 )
 
+if (SNMALLOC_ENABLE_FUZZING)
+  add_subdirectory(fuzzing)
+endif()

3rdparty/exported/snmalloc/README.md

@@ -34,7 +34,8 @@ The mechanism for returning memory to remote threads has remained, but most of t
 We recommend you read [docs/security](./docs/security/README.md) to find out about the current design, and 
 if you want to dive into the code [docs/AddressSpace.md](./docs/AddressSpace.md) provides a good overview of the allocation and deallocation paths.
 
-[![snmalloc CI](https://github.com/microsoft/snmalloc/actions/workflows/main.yml/badge.svg?branch=master)](https://github.com/microsoft/snmalloc/actions/workflows/main.yml)
+[![snmalloc CI](https://github.com/microsoft/snmalloc/actions/workflows/main.yml/badge.svg)](https://github.com/microsoft/snmalloc/actions/workflows/main.yml)
+[![snmalloc CI for Morello](https://github.com/microsoft/snmalloc/actions/workflows/morello.yml/badge.svg)](https://github.com/microsoft/snmalloc/actions/workflows/morello.yml)
 
 # Hardening
 

3rdparty/exported/snmalloc/src/snmalloc/aal/aal.h

@@ -10,13 +10,24 @@
 #include "aal_concept.h"
 #include "aal_consts.h"
 
-#include <chrono>
+#if __has_include(<time.h>)
+#  include <time.h>
+#  ifdef CLOCK_MONOTONIC
+#    define SNMALLOC_TICK_USE_CLOCK_GETTIME
+#  endif
+#endif
 #include <cstdint>
 #include <utility>
 
-#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \
+#ifndef SNMALLOC_TICK_USE_CLOCK_GETTIME
+#  include <chrono>
+#endif
+
+#if ( \
+  defined(__i386__) || defined(_M_IX86) || defined(_X86_) || \
   defined(__amd64__) || defined(__x86_64__) || defined(_M_X64) || \
-  defined(_M_AMD64)
+  defined(_M_AMD64)) && \
+  !defined(_M_ARM64EC)
 #  if defined(SNMALLOC_SGX)
 #    define PLATFORM_IS_X86_SGX
 #    define SNMALLOC_NO_AAL_BUILTINS
@@ -25,7 +36,8 @@
 #  endif
 #endif
 
-#if defined(__arm__) || defined(__aarch64__)
+#if defined(__arm__) || defined(__aarch64__) || defined(_M_ARM64) || \
+  defined(_M_ARM64EC)
 #  define PLATFORM_IS_ARM
 #endif
 
@@ -53,7 +65,7 @@ namespace snmalloc
   {
     /*
      * Provide a default specification of address_t as uintptr_t for Arch-es
-     * that support IntegerPointers.  Those Arch-es without IntegerPoihnters
+     * that support IntegerPointers.  Those Arch-es without IntegerPointers
      * must explicitly give their address_t.
      *
      * This somewhat obtuse way of spelling the defaulting is necessary so
@@ -147,7 +159,7 @@ namespace snmalloc
     static inline void prefetch(void* ptr) noexcept
     {
 #if __has_builtin(__builtin_prefetch) && !defined(SNMALLOC_NO_AAL_BUILTINS)
-      __builtin_prefetch(ptr);
+      __builtin_prefetch(ptr, 1, 3);
 #else
       Arch::prefetch(ptr);
 #endif
@@ -166,11 +178,27 @@ namespace snmalloc
       if constexpr (
         (Arch::aal_features & NoCpuCycleCounters) == NoCpuCycleCounters)
       {
+#ifdef SNMALLOC_TICK_USE_CLOCK_GETTIME
+        // the buf is populated by clock_gettime
+        SNMALLOC_UNINITIALISED timespec buf;
+        // we can skip the error checking here:
+        // * EFAULT: for out-of-bound pointers (buf is always valid stack
+        // memory)
+        // * EINVAL: for invalid clock_id (we only use CLOCK_MONOTONIC enforced
+        // by POSIX.1)
+        // Notice that clock_gettime is a usually a vDSO call, so the overhead
+        // is minimal.
+        ::clock_gettime(CLOCK_MONOTONIC, &buf);
+        return static_cast<uint64_t>(buf.tv_sec) * 1000'000'000 +
+          static_cast<uint64_t>(buf.tv_nsec);
+#  undef SNMALLOC_TICK_USE_CLOCK_GETTIME
+#else
         auto tick = std::chrono::high_resolution_clock::now();
         return static_cast<uint64_t>(
           std::chrono::duration_cast<std::chrono::nanoseconds>(
             tick.time_since_epoch())
             .count());
+#endif
       }
       else
       {
@@ -204,9 +232,6 @@ namespace snmalloc
     static SNMALLOC_FAST_PATH CapPtr<T, BOut>
     capptr_bound(CapPtr<U, BIn> a, size_t size) noexcept
     {
-      static_assert(
-        BIn::spatial > capptr::dimension::Spatial::Alloc,
-        "Refusing to re-bound Spatial::Alloc CapPtr");
       static_assert(
         capptr::is_spatial_refinement<BIn, BOut>(),
         "capptr_bound must preserve non-spatial CapPtr dimensions");

3rdparty/exported/snmalloc/src/snmalloc/aal/aal_arm.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#if defined(__aarch64__)
+#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
 #  define SNMALLOC_VA_BITS_64
 #  ifdef _MSC_VER
 #    include <arm64_neon.h>
@@ -13,6 +13,7 @@
 #endif
 
 #include <cstddef>
+
 namespace snmalloc
 {
   /**
@@ -54,7 +55,7 @@ namespace snmalloc
 #elif __has_builtin(__builtin_prefetch) && !defined(SNMALLOC_NO_AAL_BUILTINS)
       __builtin_prefetch(ptr);
 #elif defined(SNMALLOC_VA_BITS_64)
-      __asm__ volatile("prfm pldl1keep, [%0]" : "=r"(ptr));
+      __asm__ volatile("prfm pstl1keep, [%0]" : "=r"(ptr));
 #else
       __asm__ volatile("pld\t[%0]" : "=r"(ptr));
 #endif

3rdparty/exported/snmalloc/src/snmalloc/aal/aal_cheri.h

@@ -69,9 +69,6 @@ namespace snmalloc
     static SNMALLOC_FAST_PATH CapPtr<T, BOut>
     capptr_bound(CapPtr<U, BIn> a, size_t size) noexcept
     {
-      static_assert(
-        BIn::spatial > capptr::dimension::Spatial::Alloc,
-        "Refusing to re-bound Spatial::Alloc CapPtr");
       static_assert(
         capptr::is_spatial_refinement<BIn, BOut>(),
         "capptr_bound must preserve non-spatial CapPtr dimensions");
@@ -87,8 +84,11 @@ namespace snmalloc
 
       void* pb = __builtin_cheri_bounds_set_exact(a.unsafe_ptr(), size);
 
-      SNMALLOC_ASSERT(
-        __builtin_cheri_tag_get(pb) && "capptr_bound exactness failed.");
+      SNMALLOC_ASSERT_MSG(
+        __builtin_cheri_tag_get(pb),
+        "capptr_bound exactness failed. {} of size {}",
+        a.unsafe_ptr(),
+        size);
 
       return CapPtr<T, BOut>::unsafe_from(static_cast<T*>(pb));
     }