Make global/static variables thread_local; use mmap() for managing memory

include/runtime/arena.h

@@ -30,12 +30,12 @@ using memory_block_header = struct {
 // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to
 // 127.
 #define REGISTER_ARENA(name, id)                                               \
-  static struct arena name = {.allocation_semispace_id = (id)}
+  static thread_local struct arena name = {.allocation_semispace_id = (id)}
 
 #define MEM_BLOCK_START(ptr)                                                   \
   ((char *)(((uintptr_t)(ptr)-1) & ~(BLOCK_SIZE - 1)))
 
-extern bool time_for_collection;
+extern thread_local bool time_for_collection;
 
 size_t get_gc_threshold();
 

include/runtime/collect.h

@@ -26,8 +26,8 @@ using set_node = set::iterator::node_t;
 using set_impl = set::iterator::tree_t;
 
 extern "C" {
-extern size_t numBytesLiveAtCollection[1 << AGE_WIDTH];
-extern bool collect_old;
+extern thread_local size_t numBytesLiveAtCollection[1 << AGE_WIDTH];
+extern thread_local bool collect_old;
 size_t get_size(uint64_t, uint16_t);
 void migrate_static_roots(void);
 void migrate(block **block_ptr);

include/runtime/header.h

@@ -48,7 +48,7 @@ void k_hash(block *, void *);
 bool hash_enter(void);
 void hash_exit(void);
 
-extern bool gc_enabled;
+extern thread_local bool gc_enabled;
 }
 
 class k_elem {

lib/codegen/CreateTerm.cpp

@@ -782,6 +782,7 @@ llvm::Value *create_term::disable_gc() {
   llvm::Constant *global
       = module_->getOrInsertGlobal("gc_enabled", llvm::Type::getInt1Ty(ctx_));
   auto *global_var = llvm::cast<llvm::GlobalVariable>(global);
+  global_var->setThreadLocal(true);
   auto *old_val = new llvm::LoadInst(
       llvm::Type::getInt1Ty(ctx_), global_var, "was_enabled", current_block_);
   new llvm::StoreInst(
@@ -793,6 +794,7 @@ void create_term::enable_gc(llvm::Value *was_enabled) {
   llvm::Constant *global
       = module_->getOrInsertGlobal("gc_enabled", llvm::Type::getInt1Ty(ctx_));
   auto *global_var = llvm::cast<llvm::GlobalVariable>(global);
+  global_var->setThreadLocal(true);
   new llvm::StoreInst(was_enabled, global_var, current_block_);
 }
 

lib/codegen/Decision.cpp

@@ -1012,6 +1012,7 @@ std::pair<std::vector<llvm::Value *>, llvm::BasicBlock *> step_function_header(
 
   auto *collection = module->getOrInsertGlobal(
       "time_for_collection", llvm::Type::getInt1Ty(module->getContext()));
+  llvm::cast<llvm::GlobalVariable>(collection)->setThreadLocal(true);
   auto *is_collection = new llvm::LoadInst(
       llvm::Type::getInt1Ty(module->getContext()), collection, "is_collection",
       check_collect);

package/debian/changelog

@@ -1,4 +1,4 @@
-k-llvm-backend (0.1.5) unstable; urgency=medium
+k-llvm-backend (0.1.98) unstable; urgency=medium
 
   * Initial release
 

package/version

@@ -1 +1 @@
-0.1.5
+0.1.98

runtime/alloc/arena.cpp

@@ -3,6 +3,8 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <memory>
+#include <sys/mman.h>
 
 #include "runtime/alloc.h"
 #include "runtime/arena.h"
@@ -47,36 +49,65 @@ get_arena_semispace_id_of_object(void *ptr) {
   return mem_block_header(ptr)->semispace;
 }
 
-static void *first_superblock_ptr = nullptr;
-static void *superblock_ptr = nullptr;
-static char **next_superblock_ptr = nullptr;
-static unsigned blocks_left = 0;
+//
+//	We will reserve enough address space for 1 million 1MB blocks. Might want to increase this on a > 1TB server.
+//
+size_t const HYPERBLOCK_SIZE = (size_t) BLOCK_SIZE * 1024 * 1024;
+static thread_local void* hyperblock_ptr = nullptr;  // only needed for munmap()
 
-static void *megabyte_malloc() {
-  if (blocks_left == 0) {
-    blocks_left = 15;
-    if (int result
-        = posix_memalign(&superblock_ptr, BLOCK_SIZE, BLOCK_SIZE * 15)) {
-      errno = result;
-      perror("posix_memalign");
+static void*
+megabyte_malloc()
+{
+  //
+  //	Return pointer to a BLOCK_SIZE chunk of memory with BLOCK_SIZE alignment.
+  //
+  static thread_local char* currentblock_ptr = nullptr;  // char* rather than void* to permit pointer arithmetic
+  if (currentblock_ptr)
+    {
+      //
+      //	We expect an page fault due to not being able to map physical memory to this block or the
+      //	process to be killed by the OOM killer long before we run off the end of our address space.
+      //
+      currentblock_ptr += BLOCK_SIZE;
     }
-    if (!first_superblock_ptr) {
-      first_superblock_ptr = superblock_ptr;
+  else
+    {
+      //
+      //	First call - need to reserve the address space.
+      //
+      size_t request = HYPERBLOCK_SIZE;
+      void* addr = mmap(NULL,  // let OS choose the address
+			request,  // Linux and MacOS both allow up to 64TB
+			PROT_READ | PROT_WRITE,  // read, write but not execute
+			MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE,  // allocate address space only
+			-1,  // no file backing
+			0);  // no offset
+      if (addr == MAP_FAILED)
+	{
+	  perror("mmap()");
+	  abort();
+	}
+      hyperblock_ptr = addr;
+      //
+      //	We ask for one block worth of address space less than we allocated so alignment will always succeed.
+      //	We don't worry about unused address space either side of our aligned address space because there will be no
+      //	memory mapped to it.
+      //
+      currentblock_ptr = reinterpret_cast<char*>(std::align(BLOCK_SIZE, HYPERBLOCK_SIZE - BLOCK_SIZE, addr, request));
     }
-    if (next_superblock_ptr) {
-      *next_superblock_ptr = (char *)superblock_ptr;
-    }
-    auto *hdr = (memory_block_header *)superblock_ptr;
-    next_superblock_ptr = &hdr->next_superblock;
-    hdr->next_superblock = nullptr;
-  }
-  blocks_left--;
-  void *result = superblock_ptr;
-  superblock_ptr = (char *)superblock_ptr + BLOCK_SIZE;
-  return result;
+  return currentblock_ptr;
+}
+
+void
+free_all_memory()
+{
+  //
+  //	Frees all memory that was demand paged into this address range.
+  //
+  munmap(hyperblock_ptr, HYPERBLOCK_SIZE);
 }
 
-bool time_for_collection;
+thread_local bool time_for_collection;
 
 static void fresh_block(struct arena *arena) {
   char *next_block = nullptr;
@@ -122,7 +153,7 @@ static void fresh_block(struct arena *arena) {
       BLOCK_SIZE - sizeof(memory_block_header));
 }
 
-bool gc_enabled = true;
+thread_local bool gc_enabled = true;
 
 __attribute__((noinline)) void *
 do_alloc_slow(size_t requested, struct arena *arena) {
@@ -229,16 +260,3 @@ size_t arena_size(const struct arena *arena) {
               : arena->num_collection_blocks)
          * (BLOCK_SIZE - sizeof(memory_block_header));
 }
-
-void free_all_memory() {
-  auto *superblock = (memory_block_header *)first_superblock_ptr;
-  while (superblock) {
-    auto *next_superblock = (memory_block_header *)superblock->next_superblock;
-    free(superblock);
-    superblock = next_superblock;
-  }
-  first_superblock_ptr = nullptr;
-  superblock_ptr = nullptr;
-  next_superblock_ptr = nullptr;
-  blocks_left = 0;
-}

runtime/alloc/register_gc_roots_enum.cpp

@@ -3,7 +3,7 @@
 #include "runtime/collect.h"
 #include "runtime/header.h"
 
-std::vector<BlockEnumerator> block_enumerators;
+thread_local std::vector<BlockEnumerator> block_enumerators;
 
 void register_gc_roots_enumerator(BlockEnumerator f) {
   block_enumerators.push_back(f);

runtime/arithmetic/int.cpp

@@ -373,8 +373,8 @@ void int_hash(mpz_t i, void *hasher) {
   }
 }
 
-gmp_randstate_t kllvm_rand_state;
-bool kllvm_rand_state_initialized = false;
+thread_local gmp_randstate_t kllvm_rand_state;
+thread_local bool kllvm_rand_state_initialized = false;
 
 SortK hook_INT_srand(SortInt seed) {
   if (!kllvm_rand_state_initialized) {

runtime/collect/collect.cpp

@@ -16,15 +16,15 @@ char **old_alloc_ptr(void);
 char *youngspace_ptr(void);
 char *oldspace_ptr(void);
 
-static bool is_gc = false;
-bool collect_old = false;
+static thread_local bool is_gc = false;
+bool thread_local collect_old = false;
 #ifndef GC_DBG
-static uint8_t num_collection_only_young = 0;
+static thread_local uint8_t num_collection_only_young = 0;
 #else
-static char *last_alloc_ptr;
+static thread_local char *last_alloc_ptr;
 #endif
 
-size_t numBytesLiveAtCollection[1 << AGE_WIDTH];
+size_t thread_local numBytesLiveAtCollection[1 << AGE_WIDTH];
 
 bool during_gc() {
   return is_gc;

runtime/collect/migrate_static_roots.cpp

@@ -2,10 +2,10 @@
 
 #include "runtime/collect.h"
 
-extern std::vector<BlockEnumerator> block_enumerators;
+extern thread_local std::vector<BlockEnumerator> block_enumerators;
 
-extern gmp_randstate_t kllvm_rand_state;
-extern bool kllvm_rand_state_initialized;
+extern thread_local gmp_randstate_t kllvm_rand_state;
+extern thread_local bool kllvm_rand_state_initialized;
 
 extern "C" {
 

runtime/lto/alloc.cpp

@@ -159,8 +159,8 @@ static inline void *kore_alloc_collection(kllvm::sort_category cat) {
   void *mem
       = kore_alloc(sizeof(blockheader) + sizeof(collection) + sizeof(uint64_t));
   auto *hdr = (blockheader *)mem;
-  static std::string name = get_raw_symbol_name(cat) + "{}";
-  static blockheader hdr_val
+  static thread_local std::string name = get_raw_symbol_name(cat) + "{}";
+  static thread_local blockheader hdr_val
       = get_block_header_for_symbol(get_tag_for_symbol_name(name.c_str()));
   *hdr = hdr_val;
   auto *offset = (uint64_t *)(hdr + 1);

unittests/runtime-collections/lists.cpp

@@ -62,7 +62,7 @@ block D1 = {{1}};
 block *DUMMY1 = &D1;
 }
 
-bool gc_enabled;
+thread_local bool gc_enabled;
 size_t get_gc_threshold() {
   return SIZE_MAX;
 }