From fdd022f66492b9c20c6a1af916e2070006803f7d Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Fri, 29 Sep 2023 14:05:45 -0600 Subject: [PATCH] add shared library support This adds support for building WASI shared libraries per https://github.com/WebAssembly/tool-conventions/blob/main/DynamicLinking.md. For the time being, the goal is to allow "pseudo-dynamic" linking using the Component Model per https://github.com/WebAssembly/component-model/blob/main/design/mvp/examples/SharedEverythingDynamicLinking.md. This requires all libraries to be available when the component is created, but still allows runtime symbol resolution via `dlopen`/`dlsym` backed by a static lookup table. This is sufficient to support Python native extensions, for example. A complete demo using `wit-component` is available at https://github.com/dicej/component-linking-demo. This requires https://reviews.llvm.org/D153293, which we will need to backport to LLVM 16 we're ready to upgrade to LLVM 17, hence the llvm-D153293-backport.patch file. Signed-off-by: Joel Dice --- .gitmodules | 2 +- Makefile | 32 +++++++++++++++++++------------- src/wasi-libc | 2 +- wasi-sdk-pthread.cmake | 2 ++ wasi-sdk.cmake | 2 ++ 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.gitmodules b/.gitmodules index e8b2eace4..0691635ce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ url = https://github.com/llvm/llvm-project [submodule "src/wasi-libc"] path = src/wasi-libc - url = https://github.com/CraneStation/wasi-libc + url = https://github.com/WebAssembly/wasi-libc [submodule "src/config"] path = src/config url = https://git.savannah.gnu.org/git/config.git diff --git a/Makefile b/Makefile index 319cabd90..899428d3a 100644 --- a/Makefile +++ b/Makefile @@ -107,21 +107,24 @@ build/llvm.BUILT: llvm-config touch build/llvm.BUILT -build/wasi-libc.BUILT: build/llvm.BUILT +build/wasi-libc.BUILT: build/compiler-rt.BUILT $(MAKE) -C $(ROOT_DIR)/src/wasi-libc \ CC=$(BUILD_PREFIX)/bin/clang \ AR=$(BUILD_PREFIX)/bin/llvm-ar \ NM=$(BUILD_PREFIX)/bin/llvm-nm \ - SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot + SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ + BUILTINS_LIB=$(BUILD_PREFIX)/lib/clang/$(CLANG_VERSION)/lib/wasi/libclang_rt.builtins-wasm32.a \ + default libc_so $(MAKE) -C $(ROOT_DIR)/src/wasi-libc \ CC=$(BUILD_PREFIX)/bin/clang \ AR=$(BUILD_PREFIX)/bin/llvm-ar \ NM=$(BUILD_PREFIX)/bin/llvm-nm \ SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ + BUILTINS_LIB=$(BUILD_PREFIX)/lib/clang/$(CLANG_VERSION)/lib/wasi/libclang_rt.builtins-wasm32.a \ THREAD_MODEL=posix touch build/wasi-libc.BUILT -build/compiler-rt.BUILT: build/llvm.BUILT build/wasi-libc.BUILT +build/compiler-rt.BUILT: build/llvm.BUILT # Do the build, and install it. mkdir -p build/compiler-rt cd build/compiler-rt && cmake -G Ninja \ @@ -151,6 +154,8 @@ build/compiler-rt.BUILT: build/llvm.BUILT build/wasi-libc.BUILT touch build/compiler-rt.BUILT # Flags for libcxx and libcxxabi. +# $(1): pthreads ON or OFF +# $(2): shared libraries ON or OFF LIBCXX_CMAKE_FLAGS = \ -DCMAKE_C_COMPILER_WORKS=ON \ -DCMAKE_CXX_COMPILER_WORKS=ON \ @@ -161,38 +166,39 @@ LIBCXX_CMAKE_FLAGS = \ -DLLVM_CONFIG_PATH=$(ROOT_DIR)/build/llvm/bin/llvm-config \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DCXX_SUPPORTS_CXX11=ON \ - -DLIBCXX_ENABLE_THREADS:BOOL=@PTHREAD@ \ - -DLIBCXX_HAS_PTHREAD_API:BOOL=@PTHREAD@ \ + -DLIBCXX_ENABLE_THREADS:BOOL=$(1) \ + -DLIBCXX_HAS_PTHREAD_API:BOOL=$(1) \ -DLIBCXX_HAS_EXTERNAL_THREAD_API:BOOL=OFF \ -DLIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL=OFF \ -DLIBCXX_HAS_WIN32_THREAD_API:BOOL=OFF \ -DLLVM_COMPILER_CHECKED=ON \ -DCMAKE_BUILD_TYPE=RelWithDebugInfo \ - -DLIBCXX_ENABLE_SHARED:BOOL=OFF \ + -DLIBCXX_ENABLE_SHARED:BOOL=$(2) \ -DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY:BOOL=OFF \ -DLIBCXX_ENABLE_EXCEPTIONS:BOOL=OFF \ -DLIBCXX_ENABLE_FILESYSTEM:BOOL=OFF \ + -DLIBCXX_ENABLE_ABI_LINKER_SCRIPT:BOOL=OFF \ -DLIBCXX_CXX_ABI=libcxxabi \ -DLIBCXX_CXX_ABI_INCLUDE_PATHS=$(LLVM_PROJ_DIR)/libcxxabi/include \ -DLIBCXX_HAS_MUSL_LIBC:BOOL=ON \ -DLIBCXX_ABI_VERSION=2 \ -DLIBCXXABI_ENABLE_EXCEPTIONS:BOOL=OFF \ - -DLIBCXXABI_ENABLE_SHARED:BOOL=OFF \ + -DLIBCXXABI_ENABLE_SHARED:BOOL=$(2) \ -DLIBCXXABI_SILENT_TERMINATE:BOOL=ON \ - -DLIBCXXABI_ENABLE_THREADS:BOOL=@PTHREAD@ \ - -DLIBCXXABI_HAS_PTHREAD_API:BOOL=@PTHREAD@ \ + -DLIBCXXABI_ENABLE_THREADS:BOOL=$(1) \ + -DLIBCXXABI_HAS_PTHREAD_API:BOOL=$(1) \ -DLIBCXXABI_HAS_EXTERNAL_THREAD_API:BOOL=OFF \ -DLIBCXXABI_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL=OFF \ -DLIBCXXABI_HAS_WIN32_THREAD_API:BOOL=OFF \ - -DLIBCXXABI_ENABLE_PIC:BOOL=OFF \ + -DLIBCXXABI_ENABLE_PIC:BOOL=$(2) \ -DWASI_SDK_PREFIX=$(BUILD_PREFIX) \ -DUNIX:BOOL=ON \ --debug-trycompile -build/libcxx.BUILT: build/llvm.BUILT build/compiler-rt.BUILT build/wasi-libc.BUILT +build/libcxx.BUILT: build/llvm.BUILT build/wasi-libc.BUILT # Do the build. mkdir -p build/libcxx - cd build/libcxx && cmake -G Ninja $(LIBCXX_CMAKE_FLAGS:@PTHREAD@=OFF) \ + cd build/libcxx && cmake -G Ninja $(call LIBCXX_CMAKE_FLAGS,OFF,ON) \ -DCMAKE_SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ -DCMAKE_C_FLAGS="$(DEBUG_PREFIX_MAP) $(EXTRA_CFLAGS)" \ -DCMAKE_CXX_FLAGS="$(DEBUG_PREFIX_MAP) $(EXTRA_CXXFLAGS)" \ @@ -202,7 +208,7 @@ build/libcxx.BUILT: build/llvm.BUILT build/compiler-rt.BUILT build/wasi-libc.BUI $(LLVM_PROJ_DIR)/runtimes ninja $(NINJA_FLAGS) -C build/libcxx mkdir -p build/libcxx-threads - cd build/libcxx-threads && cmake -G Ninja $(LIBCXX_CMAKE_FLAGS:@PTHREAD@=ON) \ + cd build/libcxx-threads && cmake -G Ninja $(call LIBCXX_CMAKE_FLAGS,ON,OFF) \ -DCMAKE_SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ -DCMAKE_C_FLAGS="$(DEBUG_PREFIX_MAP) -pthread $(EXTRA_CFLAGS)" \ -DCMAKE_CXX_FLAGS="$(DEBUG_PREFIX_MAP) -pthread $(EXTRA_CXXFLAGS)" \ diff --git a/src/wasi-libc b/src/wasi-libc index ec4566bea..d4dae8964 160000 --- a/src/wasi-libc +++ b/src/wasi-libc @@ -1 +1 @@ -Subproject commit ec4566beae84e54952637f0bf61bee4b4cacc087 +Subproject commit d4dae89648d654cdcad6b0aa8b265bf957f07531 diff --git a/wasi-sdk-pthread.cmake b/wasi-sdk-pthread.cmake index ba660234b..afcbc8a36 100644 --- a/wasi-sdk-pthread.cmake +++ b/wasi-sdk-pthread.cmake @@ -41,3 +41,5 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_POSITION_INDEPENDENT_CODE On) diff --git a/wasi-sdk.cmake b/wasi-sdk.cmake index f6a0059c8..a77d20a05 100644 --- a/wasi-sdk.cmake +++ b/wasi-sdk.cmake @@ -38,3 +38,5 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_POSITION_INDEPENDENT_CODE On)