[wasm-mt] Support async JS interop on threadpool threads

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -2529,9 +2529,9 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.ThreadCounts.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WaitThread.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WorkerThread.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WorkerThread.NonBrowser.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WorkerThread.NonBrowser.cs" Condition="'$(TargetsBrowser)' != 'true'"/>
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WorkerTracking.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.Unix.cs" Condition="'$(TargetsUnix)' == 'true' or '$(TargetsBrowser)' == 'true' or '$(TargetsWasi)' == 'true'" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.Unix.cs" Condition="'$(TargetsUnix)' == 'true' or ('$(TargetsBrowser)' == 'true' and '$(FeatureWasmThreads)' != 'true') or '$(TargetsWasi)' == 'true'" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.Windows.cs" Condition="'$(TargetsWindows)' == 'true'" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\LowLevelLifoSemaphore.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\LowLevelLifoSemaphore.Windows.cs" Condition="'$(TargetsWindows)' == 'true'" />

src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPool.Portable.cs

@@ -19,7 +19,9 @@ public static partial class ThreadPool
     {
         // Indicates whether the thread pool should yield the thread from the dispatch loop to the runtime periodically so that
         // the runtime may use the thread for processing other work
+#if !(TARGET_BROWSER && FEATURE_WASM_THREADS)
         internal static bool YieldFromDispatchLoop => false;
+#endif
 
 #if NATIVEAOT
         private const bool IsWorkerTrackingEnabledInConfig = false;

src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Interop/LegacyExports.cs

@@ -32,18 +32,15 @@ internal static void PreventTrimming()
 
         public static void GetCSOwnedObjectByJSHandleRef(nint jsHandle, int shouldAddInflight, out JSObject? result)
         {
-            lock (JSHostImplementation.s_csOwnedObjects)
+            if (JSHostImplementation.CsOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference))
             {
-                if (JSHostImplementation.s_csOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference))
+                reference.TryGetTarget(out JSObject? jsObject);
+                if (shouldAddInflight != 0)
                 {
-                    reference.TryGetTarget(out JSObject? jsObject);
-                    if (shouldAddInflight != 0)
-                    {
-                        jsObject?.AddInFlight();
-                    }
-                    result = jsObject;
-                    return;
+                    jsObject?.AddInFlight();
                 }
+                result = jsObject;
+                return;
             }
             result = null;
         }
@@ -77,14 +74,12 @@ public static void CreateCSOwnedProxyRef(nint jsHandle, LegacyHostImplementation
 
             JSObject? res = null;
 
-            lock (JSHostImplementation.s_csOwnedObjects)
+            if (!JSHostImplementation.CsOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference) ||
+                !reference.TryGetTarget(out res) ||
+                res.IsDisposed)
             {
-                if (!JSHostImplementation.s_csOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference) ||
-                    !reference.TryGetTarget(out res) ||
-                    res.IsDisposed)
-                {
 #pragma warning disable CS0612 // Type or member is obsolete
-                    res = mappedType switch
+                res = mappedType switch
                     {
                         LegacyHostImplementation.MappedType.JSObject => new JSObject(jsHandle),
                         LegacyHostImplementation.MappedType.Array => new Array(jsHandle),
@@ -95,8 +90,7 @@ public static void CreateCSOwnedProxyRef(nint jsHandle, LegacyHostImplementation
                         _ => throw new ArgumentOutOfRangeException(nameof(mappedType))
                     };
 #pragma warning restore CS0612 // Type or member is obsolete
-                    JSHostImplementation.s_csOwnedObjects[(int)jsHandle] = new WeakReference<JSObject>(res, trackResurrection: true);
-                }
+                JSHostImplementation.CsOwnedObjects[(int)jsHandle] = new WeakReference<JSObject>(res, trackResurrection: true);
             }
             if (shouldAddInflight != 0)
             {

src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.cs

@@ -15,7 +15,20 @@ internal static partial class JSHostImplementation
         private const string TaskGetResultName = "get_Result";
         private static MethodInfo? s_taskGetResultMethodInfo;
         // we use this to maintain identity of JSHandle for a JSObject proxy
-        public static readonly Dictionary<int, WeakReference<JSObject>> s_csOwnedObjects = new Dictionary<int, WeakReference<JSObject>>();
+#if FEATURE_WASM_THREADS
+        [ThreadStatic]
+#endif
+        private static Dictionary<int, WeakReference<JSObject>>? s_csOwnedObjects;
+
+        public static Dictionary<int, WeakReference<JSObject>> CsOwnedObjects
+        {
+            get
+            {
+                s_csOwnedObjects ??= new ();
+                return s_csOwnedObjects;
+            }
+        }
+
         // we use this to maintain identity of GCHandle for a managed object
         public static Dictionary<object, IntPtr> s_gcHandleFromJSOwnedObject = new Dictionary<object, IntPtr>(ReferenceEqualityComparer.Instance);
 
@@ -24,10 +37,7 @@ public static void ReleaseCSOwnedObject(nint jsHandle)
         {
             if (jsHandle != IntPtr.Zero)
             {
-                lock (s_csOwnedObjects)
-                {
-                    s_csOwnedObjects.Remove((int)jsHandle);
-                }
+                CsOwnedObjects.Remove((int)jsHandle);
                 Interop.Runtime.ReleaseCSOwnedObject(jsHandle);
             }
         }
@@ -175,17 +185,14 @@ public static unsafe void FreeMethodSignatureBuffer(JSFunctionBinding signature)
 
         public static JSObject CreateCSOwnedProxy(nint jsHandle)
         {
-            JSObject? res = null;
+            JSObject? res;
 
-            lock (s_csOwnedObjects)
+            if (!CsOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference) ||
+                !reference.TryGetTarget(out res) ||
+                res.IsDisposed)
             {
-                if (!s_csOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference) ||
-                    !reference.TryGetTarget(out res) ||
-                    res.IsDisposed)
-                {
-                    res = new JSObject(jsHandle);
-                    s_csOwnedObjects[(int)jsHandle] = new WeakReference<JSObject>(res, trackResurrection: true);
-                }
+                res = new JSObject(jsHandle);
+                CsOwnedObjects[(int)jsHandle] = new WeakReference<JSObject>(res, trackResurrection: true);
             }
             return res;
         }

src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Legacy/LegacyHostImplementation.cs

@@ -21,10 +21,7 @@ public static void ReleaseInFlight(object obj)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static void RegisterCSOwnedObject(JSObject proxy)
         {
-            lock (JSHostImplementation.s_csOwnedObjects)
-            {
-                JSHostImplementation.s_csOwnedObjects[(int)proxy.JSHandle] = new WeakReference<JSObject>(proxy, trackResurrection: true);
-            }
+            JSHostImplementation.CsOwnedObjects[(int)proxy.JSHandle] = new WeakReference<JSObject>(proxy, trackResurrection: true);
         }
 
         public static MarshalType GetMarshalTypeFromType(Type type)

src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj

@@ -281,6 +281,9 @@
   <ItemGroup Condition="('$(TargetsBrowser)' == 'true'  or '$(TargetsWasi)' == 'true') and '$(FeatureWasmThreads)' == 'true'">
       <Compile Include="$(BclSourcesRoot)\System\Threading\ThreadPoolBoundHandle.Browser.Threads.Mono.cs" />
       <Compile Include="$(BclSourcesRoot)\System\Threading\LowLevelLifoAsyncWaitSemaphore.Browser.Threads.Mono.cs" />
+      <Compile Include="$(BclSourcesRoot)\System\Threading\PortableThreadPool.Browser.Threads.Mono.cs" />
+      <Compile Include="$(BclSourcesRoot)\System\Threading\PortableThreadPool.WorkerThread.Browser.Threads.Mono.cs" />
+      <Compile Include="$(BclSourcesRoot)\System\Threading\ThreadPool.Browser.Threads.Mono.cs" />
       <Compile Include="$(BclSourcesRoot)\System\Threading\WebWorkerEventLoop.Browser.Threads.Mono.cs" />
   </ItemGroup>
   <ItemGroup Condition="('$(TargetsBrowser)' == 'true' or '$(TargetsWasi)' == 'true') and '$(FeatureWasmThreads)' != 'true'">

src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.Browser.Threads.Mono.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Threading;
+
+internal sealed partial class PortableThreadPool
+{
+    private static partial class WorkerThread
+    {
+        private static bool IsIOPending => WebWorkerEventLoop.HasJavaScriptInteropDependents;
+    }
+
+    private struct CpuUtilizationReader
+    {
+#pragma warning disable CA1822
+        public double CurrentUtilization => 0.0; // FIXME: can we do better
+#pragma warning restore CA1822
+    }
+}

src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.Browser.Threads.Mono.cs

@@ -0,0 +1,122 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Diagnostics.Tracing;
+using System.Runtime.CompilerServices;
+
+namespace System.Threading
+{
+    internal sealed partial class PortableThreadPool
+    {
+        /// <summary>
+        /// The worker thread infastructure for the CLR thread pool.
+        /// </summary>
+        private static partial class WorkerThread
+        {
+            /// <summary>
+            /// Semaphore for controlling how many threads are currently working.
+            /// </summary>
+            private static readonly LowLevelLifoAsyncWaitSemaphore s_semaphore =
+                new LowLevelLifoAsyncWaitSemaphore(
+                    0,
+                    MaxPossibleThreadCount,
+                    AppContextConfigHelper.GetInt32Config(
+                        "System.Threading.ThreadPool.UnfairSemaphoreSpinLimit",
+                        SemaphoreSpinCountDefault,
+                        false),
+                    onWait: () =>
+                    {
+                        if (NativeRuntimeEventSource.Log.IsEnabled())
+                        {
+                            NativeRuntimeEventSource.Log.ThreadPoolWorkerThreadWait(
+                                (uint)ThreadPoolInstance._separated.counts.VolatileRead().NumExistingThreads);
+                        }
+                    });
+
+            private static readonly ThreadStart s_workerThreadStart = WorkerThreadStart;
+
+            private sealed record SemaphoreWaitState(PortableThreadPool ThreadPoolInstance, LowLevelLock ThreadAdjustmentLock, WebWorkerEventLoop.KeepaliveToken KeepaliveToken)
+            {
+                public bool SpinWait = true;
+
+                public void ResetIteration() {
+                    SpinWait = true;
+                }
+            }
+
+            private static void WorkerThreadStart()
+            {
+                Thread.CurrentThread.SetThreadPoolWorkerThreadName();
+
+                PortableThreadPool threadPoolInstance = ThreadPoolInstance;
+
+                if (NativeRuntimeEventSource.Log.IsEnabled())
+                {
+                    NativeRuntimeEventSource.Log.ThreadPoolWorkerThreadStart(
+                        (uint)threadPoolInstance._separated.counts.VolatileRead().NumExistingThreads);
+                }
+
+                LowLevelLock threadAdjustmentLock = threadPoolInstance._threadAdjustmentLock;
+                var keepaliveToken = WebWorkerEventLoop.KeepalivePush();
+                SemaphoreWaitState state = new(threadPoolInstance, threadAdjustmentLock, keepaliveToken) { SpinWait = true };
+                // set up the callbacks for semaphore waits, tell
+                // emscripten to keep the thread alive, and return to
+                // the JS event loop.
+                WaitForWorkLoop(s_semaphore, state);
+                // return from thread start with keepalive - the thread will stay alive in the JS event loop
+            }
+
+            private static readonly Action<LowLevelLifoAsyncWaitSemaphore, object?> s_WorkLoopSemaphoreSuccess = new(WorkLoopSemaphoreSuccess);
+            private static readonly Action<LowLevelLifoAsyncWaitSemaphore, object?> s_WorkLoopSemaphoreTimedOut = new(WorkLoopSemaphoreTimedOut);
+
+            private static void WaitForWorkLoop(LowLevelLifoAsyncWaitSemaphore semaphore, SemaphoreWaitState state)
+            {
+                semaphore.PrepareAsyncWait(ThreadPoolThreadTimeoutMs, s_WorkLoopSemaphoreSuccess, s_WorkLoopSemaphoreTimedOut, state);
+                // thread should still be kept alive
+                Debug.Assert(state.KeepaliveToken.Valid);
+            }
+
+            private static void WorkLoopSemaphoreSuccess(LowLevelLifoAsyncWaitSemaphore semaphore, object? stateObject)
+            {
+                SemaphoreWaitState state = (SemaphoreWaitState)stateObject!;
+                WorkerDoWork(state.ThreadPoolInstance, ref state.SpinWait);
+                // Go around the loop one more time, keeping existing mutated state
+                WaitForWorkLoop(semaphore, state);
+            }
+
+            private static void WorkLoopSemaphoreTimedOut(LowLevelLifoAsyncWaitSemaphore semaphore, object? stateObject)
+            {
+                SemaphoreWaitState state = (SemaphoreWaitState)stateObject!;
+                if (ShouldExitWorker(state.ThreadPoolInstance, state.ThreadAdjustmentLock)) {
+                    // we're done, kill the thread.
+
+                    // we're wrapped in an emscripten eventloop handler which will consult the
+                    // keepalive count, destroy the thread and run the TLS dtor which will
+                    // unregister the thread from Mono
+                    state.KeepaliveToken.Pop();
+                    return;
+                } else {
+                    // more work showed up while we were shutting down, go around one more time
+                    state.ResetIteration();
+                    WaitForWorkLoop(semaphore, state);
+                }
+            }
+
+            private static void CreateWorkerThread()
+            {
+                // Thread pool threads must start in the default execution context without transferring the context, so
+                // using captureContext: false.
+                Thread workerThread = new Thread(s_workerThreadStart);
+                workerThread.IsThreadPoolThread = true;
+                workerThread.IsBackground = true;
+                // thread name will be set in thread proc
+
+                // This thread will return to the JS event loop - tell the runtime not to cleanup
+                // after the start function returns, if the Emscripten keepalive is non-zero.
+                WebWorkerEventLoop.StartExitable(workerThread, captureContext: false);
+            }
+        }
+    }
+}

src/mono/System.Private.CoreLib/src/System/Threading/ThreadPool.Browser.Threads.Mono.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Threading
+{
+    public static partial class ThreadPool
+    {
+        // Indicates that the threadpool should yield the thread from the dispatch loop to the
+        // runtime periodically.  We use this to return back to the JS event loop so that the JS
+        // event queue can be drained
+        internal static bool YieldFromDispatchLoop => true;
+    }
+}

src/mono/mono/metadata/threads-types.h

@@ -78,6 +78,10 @@ typedef enum {
 	MONO_THREAD_CREATE_FLAGS_DEBUGGER		= 0x02,
 	MONO_THREAD_CREATE_FLAGS_FORCE_CREATE	= 0x04,
 	MONO_THREAD_CREATE_FLAGS_SMALL_STACK	= 0x08,
+	// "external eventloop" means the thread main function can return without killing the thread
+	// and the thread will continue to be attached to the runtime and may invoke embedding APIs
+	// and managed calls.  There is usually some platform-specific way to shut down the thread.
+	MONO_THREAD_CREATE_FLAGS_EXTERNAL_EVENTLOOP = 0x10,
 } MonoThreadCreateFlags;
 
 MONO_COMPONENT_API MonoInternalThread*

src/mono/mono/metadata/threads.c

@@ -1088,6 +1088,7 @@ typedef struct {
 	MonoThreadStart start_func;
 	gpointer start_func_arg;
 	gboolean force_attach;
+	gboolean external_eventloop;
 	gboolean failed;
 	MonoCoopSem registered;
 } StartInfo;
@@ -1173,6 +1174,8 @@ start_wrapper_internal (StartInfo *start_info, gsize *stack_ptr)
 	/* Let the thread that called Start() know we're ready */
 	mono_coop_sem_post (&start_info->registered);
 
+	gboolean external_eventloop = start_info->external_eventloop;
+
 	if (mono_atomic_dec_i32 (&start_info->ref) == 0) {
 		mono_coop_sem_destroy (&start_info->registered);
 		g_free (start_info);
@@ -1240,6 +1243,12 @@ start_wrapper_internal (StartInfo *start_info, gsize *stack_ptr)
 
 	THREAD_DEBUG (g_message ("%s: (%" G_GSIZE_FORMAT ") Start wrapper terminating", __func__, mono_native_thread_id_get ()));
 
+	if (G_UNLIKELY (external_eventloop)) {
+		/* if the thread wants to stay alive in an external eventloop, don't clean up after it */
+		if (mono_thread_platform_external_eventloop_keepalive_check ())
+			return 0;
+	}
+
 	/* Do any cleanup needed for apartment state. This
 	 * cannot be done in mono_thread_detach_internal since
 	 * mono_thread_detach_internal could be  called for a
@@ -1266,9 +1275,16 @@ start_wrapper (gpointer data)
 	info = mono_thread_info_attach ();
 	info->runtime_thread = TRUE;
 
+	gboolean external_eventloop = start_info->external_eventloop;
 	/* Run the actual main function of the thread */
 	res = start_wrapper_internal (start_info, (gsize*)info->stack_end);
 
+	if (G_UNLIKELY (external_eventloop)) {
+		/* if the thread wants to stay alive, don't clean up after it */
+		if (mono_thread_platform_external_eventloop_keepalive_check ())
+			return 0;
+	}
+
 	mono_thread_info_exit (res);
 
 	g_assert_not_reached ();
@@ -1355,6 +1371,7 @@ create_thread (MonoThread *thread, MonoInternalThread *internal, MonoThreadStart
 	start_info->start_func_arg = start_func_arg;
 	start_info->force_attach = flags & MONO_THREAD_CREATE_FLAGS_FORCE_CREATE;
 	start_info->failed = FALSE;
+	start_info->external_eventloop = (flags & MONO_THREAD_CREATE_FLAGS_EXTERNAL_EVENTLOOP) != 0;
 	mono_coop_sem_init (&start_info->registered, 0);
 
 	if (flags != MONO_THREAD_CREATE_FLAGS_SMALL_STACK)
@@ -4913,7 +4930,15 @@ ves_icall_System_Threading_Thread_StartInternal (MonoThreadObjectHandle thread_h
 		return;
 	}
 
-	res = create_thread (internal, internal, NULL, NULL, stack_size, MONO_THREAD_CREATE_FLAGS_NONE, error);
+	MonoThreadCreateFlags create_flags = MONO_THREAD_CREATE_FLAGS_NONE;
+#if defined(HOST_BROWSER) && !defined(DISABLE_THREADS)
+	// HACK: threadpool threads can return to the JS event loop
+	// WISH: support this for other threads, too
+	if (internal->threadpool_thread)
+		create_flags |= MONO_THREAD_CREATE_FLAGS_EXTERNAL_EVENTLOOP;
+#endif
+
+	res = create_thread (internal, internal, NULL, NULL, stack_size, create_flags, error);
 	if (!res) {
 		UNLOCK_THREAD (internal);
 		return;