Merge pull request #550 from agoode/wascally

Add WASI (WebAssembly) cross compile support

basis-library/mlton/platform.sig

@@ -10,7 +10,7 @@ signature MLTON_PLATFORM =
       structure Arch:
          sig
             datatype t = Alpha | AMD64 | ARM | ARM64 | HPPA | IA64 | LoongArch64 | m68k |
-                         MIPS | PowerPC | PowerPC64 | RISCV | S390 | Sparc | X86
+                         MIPS | PowerPC | PowerPC64 | RISCV | S390 | Sparc | Wasm32 | X86
 
             val fromString: string -> t option
             val host: t
@@ -29,7 +29,7 @@ signature MLTON_PLATFORM =
       structure OS:
          sig
             datatype t = AIX | Cygwin | Darwin | FreeBSD | Hurd | HPUX
-                       | Linux | MinGW | NetBSD | OpenBSD | Solaris
+                       | Linux | MinGW | NetBSD | OpenBSD | Solaris | WASI
 
             val fromString: string -> t option
             val host: t

basis-library/mlton/platform.sml

@@ -44,6 +44,7 @@ structure MLtonPlatform: MLTON_PLATFORM =
              | RISCV
              | S390
              | Sparc
+             | Wasm32
              | X86
 
             val all =
@@ -61,6 +62,7 @@ structure MLtonPlatform: MLTON_PLATFORM =
                (RISCV, "RISCV")::
                (S390, "S390")::
                (Sparc, "Sparc")::
+               (Wasm32, "Wasm32")::
                (X86, "X86")::
                nil
 
@@ -83,6 +85,7 @@ structure MLtonPlatform: MLTON_PLATFORM =
                 | "riscv" => RISCV
                 | "s390" => S390
                 | "sparc" => Sparc
+                | "wasm32" => Wasm32
                 | "x86" => X86
                 | _ => raise Fail "strange MLton_Platform_Arch_host"
          end
@@ -128,6 +131,7 @@ structure MLtonPlatform: MLTON_PLATFORM =
              | NetBSD
              | OpenBSD
              | Solaris
+             | WASI
 
             val all =
                (AIX, "AIX")::
@@ -141,6 +145,7 @@ structure MLtonPlatform: MLTON_PLATFORM =
                (NetBSD, "NetBSD")::
                (OpenBSD, "OpenBSD")::
                (Solaris, "Solaris")::
+               (WASI, "WASI")::
                nil
 
             val (fromString, toString) = fromString_toString all
@@ -159,6 +164,7 @@ structure MLtonPlatform: MLTON_PLATFORM =
                 | "netbsd" => NetBSD
                 | "openbsd" => OpenBSD
                 | "solaris" => Solaris
+                | "wasi" => WASI
                 | _ => raise Fail "strange MLton_Platform_OS_host"
          end
    end

basis-library/sml-nj/sml-nj.sml

@@ -42,6 +42,7 @@ structure SMLofNJ: SML_OF_NJ =
                    | NetBSD => UNIX
                    | OpenBSD => UNIX
                    | Solaris => UNIX
+                   | WASI => UNIX
                end
 
             fun getOSName () = MLton.Platform.OS.toString MLton.Platform.OS.host

bin/mlton-script

@@ -118,6 +118,9 @@ for arg in "$@"; do
                -target-cc-opt ia64-hpux "-mlp64"                        \
                -target-cc-opt ia64 "-mtune=itanium2"                    \
                -target-cc-opt sparc '-m32 -mcpu=v8 -Wa,-xarch=v8plusa'  \
+               -target-cc-opt wasi '-D_WASI_EMULATED_SIGNAL'            \
+               -target-cc-opt wasi '-D_WASI_EMULATED_PROCESS_CLOCKS'    \
+               -target-cc-opt wasi '-D_WASI_EMULATED_GETPID'            \
                -target-cc-opt x86 '-m32'                                \
                -target-link-opt aix '-maix64'                           \
                -target-link-opt alpha                                   \
@@ -134,6 +137,9 @@ for arg in "$@"; do
                -target-link-opt mingw '-Wl,--enable-stdcall-fixup'      \
                -target-link-opt openbsd '-Wl,--no-execute-only'         \
                -target-link-opt solaris '-lnsl -lsocket -lrt'           \
+               -target-link-opt wasi '-lwasi-emulated-signal'           \
+               -target-link-opt wasi '-lwasi-emulated-process-clocks'   \
+               -target-link-opt wasi '-lwasi-emulated-getpid'           \
                -target-link-opt x86 '-m32'                              \
                -profile-exclude '\$\(SML_LIB\)'
     fi

doc/guide/src/PlatformSpecificNotes.adoc

@@ -14,6 +14,7 @@ Here are notes about using MLton on the following platforms.
 * <<RunningOnNetBSD#,NetBSD>>
 * <<RunningOnOpenBSD#,OpenBSD>>
 * <<RunningOnSolaris#,Solaris>>
+* <<RunningOnWASI#,WASI>>
 
 == Architectures
 
@@ -22,6 +23,7 @@ Here are notes about using MLton on the following platforms.
 * <<RunningOnPowerPC#,PowerPC>>
 * <<RunningOnPowerPC64#,PowerPC64>>
 * <<RunningOnSparc#,Sparc>>
+* <<RunningOnWASI#,WebAssembly>>
 * <<RunningOnX86#,X86>>
 
 == Also see

doc/guide/src/RunningOnWASI.adoc

@@ -0,0 +1,102 @@
+= RunningOnWASI
+
+https://wasi.dev[WASI] is a somewhat POSIX-compatible system interface
+for writing https://webassembly.org[WebAssembly] (Wasm) programs.
+
+MLton does not run on WASI directly (since it requires unimplemented
+functionality), but it can cross-compile to this target, resuling in
+binaries that can run in the browser (using a shim) and from the
+command line (using something like https://wasmtime.dev[Wasmtime]).
+
+== Notes
+
+* To compile MLton for WASI:
+
+** You will need a WASI compiler and sysroot. These are both included
+   in https://github.com/WebAssembly/wasi-sdk[wasi-sdk]. Follow the
+   directions to download and unpack a copy.
+
+** The <<GMP#>> library is required. It must be compiled for
+   WASI. Download it from https://gmplib.org, build it and install it
+   to a local prefix. An example configuration command is below
+   (substitute your own wasi-sdk paths):
++
+[,shell]
+----
+./configure --host=wasm32-wasi \
+            --prefix=$HOME/gmp-wasi-INSTALL \
+            CC=$HOME/wasi-sdk-22.0/bin/clang \
+            RANLIB=$HOME/wasi-sdk-22.0/bin/ranlib \
+            CFLAGS=-D_WASI_EMULATED_SIGNAL LDFLAGS=-lwasi-emulated-signal
+----
+
+** During compilation of MLton, some files are generated, compiled, and
+   ran. When compiling the MLton runtime for WASI, you will need a way
+   of executing these binaries. Install https://wasmtime.dev[Wasmtime]
+   to do this.
+
+** MLton must be compiled and installed, then the MLton WASI runtime
+   target must be compiled and installed. Build and install MLton normally,
+   then clean the source directory and build the WASI runtime. An example
+   to build and install the WASI runtime is:
++
+[,shell]
+----
+gmake CC=$HOME/wasi-sdk-22.0/bin/clang \
+      AR=$HOME/wasi-sdk-22.0/bin/ar \
+      RANLIB=$HOME/wasi-sdk-22.0/bin/ranlib \
+      TARGET_OS=wasi \
+      TARGET_ARCH=wasm32 \
+      TARGET=wasm32-wasi \
+      WITH_GMP_DIR=$HOME/gmp-wasi-INSTALL \
+      PREFIX=$HOME/mlton-INSTALL \
+      dirs runtime install-runtime
+----
++
+Once this is complete, you should have a functional MLton that can
+cross-compile to WASI.
+
+* Compiling WASI binaries with MLton:
+
+** Once installed, you can run MLton with the
+   ``-target wasm32-wasi`` flag, to build WASI binaries:
++
+[,shell]
+----
+mlton -target wasm32-wasi hello-world.sml
+----
+
+** Using Wasmtime, you can run your WASI binaries locally:
++
+[,shell]
+----
+wasmtime hello-world
+----
++
+----
+Hello, World!
+----
+
+== Known issues
+
+* Many functions are unimplemented and will cause link errors during
+  compilation, allowing for early detection of incompatibilities. See
+  https://github.com/WebAssembly/wasi-libc[wasi-libc] for what is
+  available. In general, anything listed in ``platform/wasi.h`` is
+  unavailable for use.
+
+* Disk-backed heap is not supported. Do not set ``may-page-heap`` to
+  ``true``.
+
+* The maximum amount of available memory is fixed. See
+  ``platform/wasi.c`` for the current value.
+
+* The WASI runtime is restrictive regarding file IO and other
+  operations. You will need to map directories for use by compiled
+  binaries. See the Wasmtime help for details on how to do this.
+
+* MLton currently only supports WASI, which requires a shim to run in
+  browsers. A few options are available:
+
+** https://github.com/bjorn3/browser_wasi_shim
+** https://runno.dev/wasi

mlton/main/main.fun

@@ -1239,7 +1239,8 @@ fun commandLine (_: string, args: string list): unit =
                       | MinGW => true
                       | NetBSD => true
                       | OpenBSD => true
-                      | Solaris => true)
+                      | Solaris => true
+                      | WASI => false)
                     then ()
                     else usage (concat ["can't use -profile time on ",
                                         MLton.Platform.OS.toString targetOS])

runtime/Makefile

@@ -20,6 +20,8 @@ endif
 
 EXE :=
 
+WASMTIME :=
+
 MDVARIANTS := OPT DBG
 PIVARIANTS := DPI NPI PIC PIE
 
@@ -110,6 +112,16 @@ ifeq ($(TARGET_OS), cygwin)
 EXE := .exe
 endif
 
+ifeq ($(TARGET_OS), wasi)
+WASMTIME := wasmtime
+XCPPFLAGS += -D_WASI_EMULATED_SIGNAL \
+             -D_WASI_EMULATED_PROCESS_CLOCKS \
+             -D_WASI_EMULATED_GETPID
+XLDFLAGS += -lwasi-emulated-signal \
+            -lwasi-emulated-process-clocks \
+            -lwasi-emulated-getpid
+endif
+
 ifeq ($(TARGET_OS), mingw)
 EXE := .exe
 # GCC doesn't recognize the %I64 format specifier which means %ll on windows
@@ -239,11 +251,11 @@ ml-types.h: gen/ml-types.h
 	$(CP) $< $@
 
 gen/ml-types.h: gen/gen-types$(EXE)
-	./gen/gen-types$(EXE) ml-types.h > gen/ml-types.h
+	$(WASMTIME) ./gen/gen-types$(EXE) ml-types.h > gen/ml-types.h
 gen/c-types.h: gen/gen-types$(EXE)
-	./gen/gen-types$(EXE) c-types.h > gen/c-types.h
+	$(WASMTIME) ./gen/gen-types$(EXE) c-types.h > gen/c-types.h
 gen/c-types.sml: gen/gen-types$(EXE)
-	./gen/gen-types$(EXE) c-types.sml > gen/c-types.sml
+	$(WASMTIME) ./gen/gen-types$(EXE) c-types.sml > gen/c-types.sml
 
 ifneq ($(MAKECMDGOALS),clean)
 -include gen/gen-types.d
@@ -262,7 +274,7 @@ gen/$(1): gen/gen-basis-ffi.sml gen/basis-ffi.def
 	touch gen/$(1)
 else
 gen/$(1): gen/gen-basis-ffi$(EXE) gen/basis-ffi.def
-	./gen/gen-basis-ffi$(EXE) $(1) < gen/basis-ffi.def > gen/$(1)
+	$(WASMTIME) ./gen/gen-basis-ffi$(EXE) $(1) < gen/basis-ffi.def > gen/$(1)
 	$(CAT) gen/gen-basis-ffi.sml gen/basis-ffi.def gen/$(1) | $(SHA1DGST) > gen/$(1).chk
 endif
 endef
@@ -307,7 +319,7 @@ gdtoa/README: gdtoa.tgz gdtoa.may_alias-unions.patch gdtoa.rename-public-fns.pat
 	@touch $@
 
 gdtoa/arith.h: gdtoa/arithchk$(EXE)
-	./gdtoa/arithchk$(EXE) > gdtoa/arith.h
+	$(WASMTIME) ./gdtoa/arithchk$(EXE) > gdtoa/arith.h
 
 gdtoa/arithchk.c: gdtoa/README
 	@touch $@
@@ -319,7 +331,7 @@ ifneq ($(MAKECMDGOALS),clean)
 endif
 
 gdtoa/gd_qnan.h: gdtoa/qnan$(EXE)
-	./gdtoa/qnan$(EXE) > gdtoa/gd_qnan.h
+	$(WASMTIME) ./gdtoa/qnan$(EXE) > gdtoa/gd_qnan.h
 
 gdtoa/qnan.c: gdtoa/README
 	@touch $@
@@ -348,7 +360,7 @@ $(eval $(call A_TEMPLATE,mlton,MLTON))
 ### gen/constants ###
 
 gen/constants: gen/gen-constants$(EXE)
-	./gen/gen-constants > gen/constants
+	$(WASMTIME) ./gen/gen-constants > gen/constants
 
 ifneq ($(MAKECMDGOALS),clean)
 -include gen/gen-constants.d

runtime/cenv.h

@@ -82,6 +82,8 @@ COMPILE_TIME_ASSERT(sizeof_double__is_eight, sizeof(double) == 8);
 #include "platform/openbsd.h"
 #elif (defined (__sun__))
 #include "platform/solaris.h"
+#elif (defined (__wasi__))
+#include "platform/wasi.h"
 #else
 #error unknown platform os
 #endif
@@ -117,6 +119,8 @@ COMPILE_TIME_ASSERT(sizeof_double__is_eight, sizeof(double) == 8);
 #include "platform/x86.h"
 #elif (defined (__loongarch64))
 #include "platform/loongarch64.h"
+#elif (defined (__wasm32))
+#include "platform/wasm32.h"
 #else
 #error unknown platform arch
 #endif

runtime/gc/heap.c

@@ -183,7 +183,9 @@ void shrinkHeap (GC_state s, GC_heap h, size_t keepSize) {
     }
     assert (isAligned (keepWithMapsSize, s->sysvals.pageSize));
     assert (keepWithMapsSize <= h->withMapsSize);
+#if HAS_SHRINK_HEAP
     GC_release (h->start + keepWithMapsSize, h->withMapsSize - keepWithMapsSize);
+#endif
     h->size = keepSize;
     h->withMapsSize = keepWithMapsSize;
   }

runtime/platform.h

@@ -35,6 +35,10 @@
 #error HAS_REMAP not defined
 #endif
 
+#ifndef HAS_SHRINK_HEAP
+#error HAS_SHRINK_HEAP not defined
+#endif
+
 #ifndef HAS_SIGALTSTACK
 #error HAS_SIGALTSTACK not defined
 #else

runtime/platform/aix.h

@@ -34,6 +34,7 @@
 #define HAS_MSG_DONTWAIT FALSE
 #define HAS_PTRACE FALSE
 #define HAS_REMAP FALSE
+#define HAS_SHRINK_HEAP TRUE
 #define HAS_SIGALTSTACK TRUE
 #define NEEDS_SIGALTSTACK_EXEC FALSE
 #define HAS_SPAWN FALSE

runtime/platform/cygwin.h

@@ -35,6 +35,7 @@
 
 #define HAS_FEROUND FALSE
 #define HAS_REMAP TRUE
+#define HAS_SHRINK_HEAP TRUE
 #define HAS_SIGALTSTACK FALSE
 #define NEEDS_SIGALTSTACK_EXEC FALSE
 #define HAS_SPAWN TRUE

runtime/platform/darwin.h

@@ -34,6 +34,7 @@
 #define HAS_FEROUND TRUE
 #define HAS_MSG_DONTWAIT TRUE
 #define HAS_REMAP FALSE
+#define HAS_SHRINK_HEAP TRUE
 #define HAS_SIGALTSTACK TRUE
 #define NEEDS_SIGALTSTACK_EXEC FALSE
 #define HAS_SPAWN FALSE

runtime/platform/freebsd.h

@@ -30,6 +30,7 @@
 #define HAS_FEROUND TRUE
 #define HAS_MSG_DONTWAIT TRUE
 #define HAS_REMAP FALSE
+#define HAS_SHRINK_HEAP TRUE
 #define HAS_SIGALTSTACK TRUE
 #define NEEDS_SIGALTSTACK_EXEC FALSE
 #define HAS_SPAWN FALSE

runtime/platform/hpux.h

@@ -52,6 +52,7 @@
 #define HAS_FEROUND TRUE
 #define HAS_MSG_DONTWAIT FALSE
 #define HAS_REMAP FALSE
+#define HAS_SHRINK_HEAP TRUE
 #define HAS_SIGALTSTACK TRUE
 #define NEEDS_SIGALTSTACK_EXEC FALSE
 #define HAS_SPAWN FALSE

runtime/platform/hurd.h

@@ -30,6 +30,7 @@
 #define HAS_FEROUND TRUE
 #define HAS_MSG_DONTWAIT TRUE
 #define HAS_REMAP TRUE
+#define HAS_SHRINK_HEAP TRUE
 #define HAS_SIGALTSTACK TRUE
 #define NEEDS_SIGALTSTACK_EXEC FALSE
 #define HAS_SPAWN FALSE

runtime/platform/linux.h

@@ -38,6 +38,7 @@
 #endif
 #define HAS_MSG_DONTWAIT TRUE
 #define HAS_REMAP TRUE
+#define HAS_SHRINK_HEAP TRUE
 #define HAS_SIGALTSTACK TRUE
 #if (defined (__hppa__))
 #define NEEDS_SIGALTSTACK_EXEC TRUE