Use atexit to avoid program shutdown race

Summary: We cannot generally expect detached threads to run to completion once static shutdown begins -- that is undefined behavior.  Instead, we now register an atexit callback to wait for any detached threads to complete.  This avoids any compiler/linker decisions lining up static destruction in the "right" order.

Reviewed By: RomanFedotovFB

Differential Revision: D50750129

fbshipit-source-id: 29afb15a3b0b82c25594bad475f1314114120444

dispenso/schedulable.cpp

@@ -9,9 +9,31 @@
 
 namespace dispenso {
 
-NewThreadInvoker::ThreadWaiter& NewThreadInvoker::getWaiter() {
-  static ThreadWaiter waiter;
-  return waiter;
+namespace {
+std::atomic<uintptr_t> g_waiter(0);
+} // namespace
+
+NewThreadInvoker::ThreadWaiter* NewThreadInvoker::getWaiter() {
+  uintptr_t waiter;
+  // Common case check first
+  if ((waiter = g_waiter.load(std::memory_order_acquire)) > 1) {
+    return reinterpret_cast<ThreadWaiter*>(waiter);
+  }
+
+  uintptr_t exp = 0;
+  if (g_waiter.compare_exchange_strong(exp, 1, std::memory_order_acq_rel)) {
+    g_waiter.store(reinterpret_cast<uintptr_t>(new ThreadWaiter()), std::memory_order_release);
+    std::atexit(destroyThreadWaiter);
+  }
+
+  while ((waiter = g_waiter.load(std::memory_order_acquire)) < 2) {
+  }
+
+  return reinterpret_cast<ThreadWaiter*>(waiter);
+}
+
+void NewThreadInvoker::destroyThreadWaiter() {
+  delete reinterpret_cast<ThreadWaiter*>(g_waiter.load(std::memory_order_acquire));
 }
 
 } // namespace dispenso

dispenso/schedulable.h

@@ -78,11 +78,11 @@ class NewThreadInvoker {
    **/
   template <typename F>
   void schedule(F&& f, ForceQueuingTag) const {
-    auto& waiter = getWaiter();
-    waiter.add();
-    std::thread thread([f = std::move(f), &waiter]() {
+    auto* waiter = getWaiter();
+    waiter->add();
+    std::thread thread([f = std::move(f), waiter]() {
       f();
-      waiter.remove();
+      waiter->remove();
     });
     thread.detach();
   }
@@ -109,7 +109,9 @@ class NewThreadInvoker {
       impl_.wait(0);
     }
   };
-  DISPENSO_DLL_ACCESS static ThreadWaiter& getWaiter();
+  DISPENSO_DLL_ACCESS static ThreadWaiter* getWaiter();
+
+  static void destroyThreadWaiter();
 };
 
 constexpr NewThreadInvoker kNewThreadInvoker;