0.0.10-pre

.github/workflows/push.yml

@@ -19,6 +19,7 @@ jobs:
       - name: compile runtime
         run: |
           make ARCH=${{ matrix.platform }} lo
+#          ./lo build binding core
       - name: check
         if: ${{ matrix.platform == 'x64' }}
         run: |
@@ -45,6 +46,7 @@ jobs:
       - name: compile
         run: |
           make lo
+#          ./lo build binding core
       - name: check
         run: |
           make check

Makefile

@@ -5,7 +5,7 @@ CCARGS=-std=c++17 -c -fno-omit-frame-pointer -fno-rtti -fno-exceptions
 CARGS=-c -fno-omit-frame-pointer
 WARN=-Werror -Wpedantic -Wall -Wextra -Wno-unused-parameter
 OPT=-O3
-VERSION=0.0.9-pre
+VERSION=0.0.10-pre
 V8_VERSION=1.0.0
 RUNTIME=lo
 LO_HOME=$(shell pwd)
@@ -45,6 +45,13 @@ ifneq ($(os),win)
 	rm -f v8-include.tar.gz
 endif
 
+v8/src: ## download the v8 source code for debugging
+	curl -L -o v8-src.tar.gz https://github.com/just-js/v8/releases/download/${V8_VERSION}/src.tar.gz
+	tar -xf v8-src.tar.gz
+ifneq ($(os),win)
+	rm -f v8-src.tar.gz
+endif
+
 v8/libv8_monolith.a: ## download the v8 static libary for linux/macos
 	curl -L -o v8/libv8_monolith.a.gz https://github.com/just-js/v8/releases/download/${V8_VERSION}/libv8_monolith-${os}-${ARCH}.a.gz
 	gzip -d v8/libv8_monolith.a.gz

TODO.md

@@ -59,6 +59,9 @@
 - [ ] **bug**: for static libraries, we need to compile without -fPIC: https://stackoverflow.com/questions/28187163/how-do-you-link-a-static-library-to-a-shared-library
 - [ ] **todo**: change inflate builder to download the depencies rather than having them embedded in the runtime
 - [ ] **todo**: we need an SSE4 alternative (NEON?) for arm64 for picohttpparser
+- [ ] **todo**: handle bindings methods that are optional based on defines
+- [ ] **todo**: think about how we handle fork. can we handle it?
+- [ ] **bug**: we open file twice with the LO_HOME core loader - refactor this so it works better
 
 ## features
 

builtins.S

@@ -33,16 +33,16 @@ __binary_lib_path_js_start:
         .incbin "lib/path.js"
         .global __binary_lib_path_js_end
 __binary_lib_path_js_end:
-.global __binary_lib_inflate_js_start
-__binary_lib_inflate_js_start:
-        .incbin "lib/inflate.js"
-        .global __binary_lib_inflate_js_end
-__binary_lib_inflate_js_end:
 .global __binary_lib_curl_js_start
 __binary_lib_curl_js_start:
         .incbin "lib/curl.js"
         .global __binary_lib_curl_js_end
 __binary_lib_curl_js_end:
+.global __binary_lib_inflate_js_start
+__binary_lib_inflate_js_start:
+        .incbin "lib/inflate.js"
+        .global __binary_lib_inflate_js_end
+__binary_lib_inflate_js_end:
 .global __binary_lib_build_js_start
 __binary_lib_build_js_start:
         .incbin "lib/build.js"
@@ -118,16 +118,6 @@ __binary_lib_lz4_api_js_start:
         .incbin "lib/lz4/api.js"
         .global __binary_lib_lz4_api_js_end
 __binary_lib_lz4_api_js_end:
-.global __binary_lib_inflate_em_inflate_c_start
-__binary_lib_inflate_em_inflate_c_start:
-        .incbin "lib/inflate/em_inflate.c"
-        .global __binary_lib_inflate_em_inflate_c_end
-__binary_lib_inflate_em_inflate_c_end:
-.global __binary_lib_inflate_em_inflate_h_start
-__binary_lib_inflate_em_inflate_h_start:
-        .incbin "lib/inflate/em_inflate.h"
-        .global __binary_lib_inflate_em_inflate_h_end
-__binary_lib_inflate_em_inflate_h_end:
 .global __binary_lib_mbedtls_api_js_start
 __binary_lib_mbedtls_api_js_start:
         .incbin "lib/mbedtls/api.js"

builtins_linux.S

@@ -33,16 +33,16 @@ _binary_lib_path_js_start:
         .incbin "lib/path.js"
         .global _binary_lib_path_js_end
 _binary_lib_path_js_end:
-.global _binary_lib_inflate_js_start
-_binary_lib_inflate_js_start:
-        .incbin "lib/inflate.js"
-        .global _binary_lib_inflate_js_end
-_binary_lib_inflate_js_end:
 .global _binary_lib_curl_js_start
 _binary_lib_curl_js_start:
         .incbin "lib/curl.js"
         .global _binary_lib_curl_js_end
 _binary_lib_curl_js_end:
+.global _binary_lib_inflate_js_start
+_binary_lib_inflate_js_start:
+        .incbin "lib/inflate.js"
+        .global _binary_lib_inflate_js_end
+_binary_lib_inflate_js_end:
 .global _binary_lib_build_js_start
 _binary_lib_build_js_start:
         .incbin "lib/build.js"
@@ -118,16 +118,6 @@ _binary_lib_lz4_api_js_start:
         .incbin "lib/lz4/api.js"
         .global _binary_lib_lz4_api_js_end
 _binary_lib_lz4_api_js_end:
-.global _binary_lib_inflate_em_inflate_c_start
-_binary_lib_inflate_em_inflate_c_start:
-        .incbin "lib/inflate/em_inflate.c"
-        .global _binary_lib_inflate_em_inflate_c_end
-_binary_lib_inflate_em_inflate_c_end:
-.global _binary_lib_inflate_em_inflate_h_start
-_binary_lib_inflate_em_inflate_h_start:
-        .incbin "lib/inflate/em_inflate.h"
-        .global _binary_lib_inflate_em_inflate_h_end
-_binary_lib_inflate_em_inflate_h_end:
 .global _binary_lib_mbedtls_api_js_start
 _binary_lib_mbedtls_api_js_start:
         .incbin "lib/mbedtls/api.js"

globals.d.ts

@@ -33,6 +33,15 @@ interface RuntimeVersion {
 
 interface Core {
   open(path: string, flags: number, mode: number);
+  dlsym(handle: number, name: string): number;
+  dlopen(path: string, flags: number): number;
+  strnlen(str: string, size: number);
+  readFile(path: string): Uint8Array;
+}
+
+declare class CString extends Uint8Array {
+  ptr: number;
+  size: number;
 }
 
 interface Runtime {
@@ -46,7 +55,7 @@ interface Runtime {
   libraries(): Array<string>;
   builtins(): Array<string>;
   assert(expression: any, message?: string | Function): any;
-  cstr(str: string): Uint8Array;
+  cstr(str: string): CString;
   load(name: string): any;
   library(name: string): any;
   /**
@@ -63,11 +72,10 @@ interface Runtime {
   utf8EncodeInto(str: string, buf: TypedArray): number;
   utf8EncodeIntoAtOffset(str: string, buf: TypedArray, off: number): number;
   utf8Decode(address: number, len?: number): string;
+  latin1Decode(address: number, len?: number): string;
   utf8Encode(str: sring): TypedArray;
   wrap(handle: TypedArray, fn: Function, plen: number): function;
   addr(handle: TypedArray): number;
-  dlsym(handle: number, name: string): number;
-  dlopen(path: string, flags: number): number;
   version: RuntimeVersion;
   args: Array<string>;
   workerSource: string;

lib/build.js

@@ -80,6 +80,7 @@ async function compile_bindings (lib, verbose = false) {
   const lib_dir = `lib/${lib}`
   const binding_path = `${lib_dir}/api.js`
 
+  // todo: download the lib if we don't have it internally
   console.log(`${AM}compile binding${AD} ${lib} ${AY}in${AD} ${lib_dir}`)
   if (!isDir(lib_dir) && lo.builtins().includes(binding_path)) {
     console.log(`${AM}create dir${AD} ${lib_dir}`)
@@ -95,12 +96,16 @@ async function compile_bindings (lib, verbose = false) {
   console.log(`${AY}create ${AD} ${lib_dir}/${def.name}.cc`)
   writeFile(`${lib_dir}/${def.name}.cc`, encoder.encode(src))
 
-  console.log(`${AY}change dir to ${AD} ${lib_dir}`)
-  assert(chdir(lib_dir) === 0)
-
-  if (def.build) {
+  const binding_build_path = `${lib_dir}/build.js`
+  if (isFile(binding_build_path)) {
     console.log(`${AM}building dependencies${AD} ${lib} ${AY}in${AD} ${lib_dir}`)
-    await def.build(C, CC)
+    const { build } = await import (binding_build_path)
+    console.log(`${AY}change dir to ${AD} ${lib_dir}`)
+    assert(chdir(lib_dir) === 0)
+    await build(C, CC)
+  } else {
+    console.log(`${AY}change dir to ${AD} ${lib_dir}`)
+    assert(chdir(lib_dir) === 0)
   }
 
   console.log(`${AY}compile${AD} ${def.name}.cc ${AY}with${AG} ${CC}${AD}`)
@@ -112,7 +117,8 @@ async function compile_bindings (lib, verbose = false) {
   console.log(`${AY}static lib ${AD} ${def.name}.a`)
   unlink(`${def.name}.a`)
   if (def.obj && def.obj.length) {
-    exec2(['ar', 'crsT', `${def.name}.a`, `${def.name}.o`, ...def.obj.filter(f => extName(f) === 'o')], verbose)
+    exec2(['ar', 'crsT', `${def.name}.a`, `${def.name}.o`, 
+      ...def.obj.filter(f => extName(f) === 'o')], verbose)
   } else {
     exec2(['ar', 'crsT', `${def.name}.a`, `${def.name}.o`], verbose)
   }
@@ -123,14 +129,16 @@ async function compile_bindings (lib, verbose = false) {
   unlink(`${def.name}.so`)
   if (os === 'mac') {
     exec2([...LINK.split(' '), ...LARGS, OPT, '-bundle', ...WARN, '-o', 
-      `${def.name}.so`, `${def.name}.o`, ...(def.obj || []).filter(f => extName(f) === 'a'), 
+      `${def.name}.so`, `${def.name}.o`, 
+      ...(def.obj || []).filter(f => extName(f) === 'a'), 
       ...(def.libs || []).map(l => `-l${l}`)],
       verbose)
   } else if (os === 'linux') {
     // todo: why do we not include .o obj files here?
     // confused by this. why does linking against the .a file not work?
     exec2([...LINK.split(' '), ...LARGS, OPT, '-shared', ...WARN, '-o', 
-      `${def.name}.so`, `${def.name}.o`, ...(def.obj || []).filter(f => extName(f) === 'a'), 
+      `${def.name}.so`, `${def.name}.o`, 
+      ...(def.obj || []).filter(f => extName(f) === 'a'), 
       ...(def.libs || []).map(l => `-l${l}`)],
       verbose)
   }
@@ -145,28 +153,26 @@ function create_builtins (libs = [], os) {
 //  config.os = 'win'
 //  writeFile(`${LO_HOME}/builtins.h`, encoder.encode([`main.js`, ...libs].map(linkerScript).join('')))
   config.os = 'linux'
-  writeFile(`${LO_HOME}/builtins_linux.S`, encoder.encode([`main.js`, ...libs].map(linkerScript).join('')))
+  writeFile(`${LO_HOME}/builtins_linux.S`, 
+    encoder.encode([`main.js`, ...libs].map(linkerScript).join('')))
   config.os = 'mac'
-  writeFile(`${LO_HOME}/builtins.S`, encoder.encode([`main.js`, ...libs].map(linkerScript).join('')))
+  writeFile(`${LO_HOME}/builtins.S`, 
+    encoder.encode([`main.js`, ...libs].map(linkerScript).join('')))
   config.os = os
 }
 
-/*
-function create_header (libs = [], bindings = [], opts) {
-  const main_h = headerFile([...libs, ...bindings.map(n => `lib/${n}/${n}.a`)], opts)
-  writeFile(`${LO_HOME}/main.h`, encoder.encode(main_h))
-}
-*/
-
 function create_header (libs = [], bindings = [], opts) {
   const os = config.os
   config.os = 'win'
-  writeFile(`${LO_HOME}/main_win.h`, encoder.encode(headerFile([...libs, ...bindings.map(n => `lib/${n}/${n}.a`)], opts)))
+  writeFile(`${LO_HOME}/main_win.h`, encoder.encode(headerFile([...libs, 
+    ...bindings.map(n => `lib/${n}/${n}.a`)], opts)))
   config.os = os
-  writeFile(`${LO_HOME}/main.h`, encoder.encode(headerFile([...libs, ...bindings.map(n => `lib/${n}/${n}.a`)], opts)))
+  writeFile(`${LO_HOME}/main.h`, encoder.encode(headerFile([...libs, 
+    ...bindings.map(n => `lib/${n}/${n}.a`)], opts)))
 }
 
-async function build_runtime ({ libs = lo.builtins(), bindings = lo.libraries(), embeds = [] }, verbose = false) {
+async function build_runtime ({ libs = lo.builtins(), bindings = lo.libraries(), 
+  embeds = [] }, verbose = false) {
   const cwd = getcwd()
   console.log(`${AY}create${AD} builtins`)
   create_builtins([...libs, ...embeds], config.os)
@@ -232,7 +238,7 @@ const encoder = new TextEncoder()
 const status = new Int32Array(2)
 
 // todo: clean up api so we can pass a config in and run builds through api
-const VERSION = getenv('VERSION') || '"0.0.9pre"'
+const VERSION = getenv('VERSION') || '"0.0.10pre"'
 const RUNTIME = getenv('RUNTIME') || '"lo"'
 const TARGET = getenv('TARGET') || 'lo'
 const LINK_TYPE = (getenv('LINK_TYPE') || '-rdynamic').split(' ')
@@ -263,22 +269,31 @@ config.os = os
 const so_ext = (os === 'linux' ? 'so' : (os === 'mac' ? 'dylib' : 'dll'))
 
 const runtimes = {
-  custom: {
-    bindings: [],
-    libs: []
-  },
   core: {
     bindings: [
       'core'
     ],
     libs: []
   },
   builder: {
-    bindings: ['core', 'inflate', 'curl'],
+    bindings: [
+      'core', 
+      'inflate', 
+      'curl', 
+    ],
     libs: [
-      'lib/bench.js', 'lib/gen.js', 'lib/fs.js', 'lib/untar.js', 'lib/proc.js', 
-      'lib/path.js', 'lib/inflate.js', 'lib/curl.js', 'lib/build.js', 
-      'lib/asm.js', 'lib/ffi.js', 'lib/binary.js'
+      'lib/bench.js', 
+      'lib/gen.js', 
+      'lib/fs.js', 
+      'lib/untar.js', 
+      'lib/proc.js', 
+      'lib/path.js', 
+      'lib/curl.js',
+      'lib/inflate.js', 
+      'lib/build.js', 
+      'lib/asm.js', 
+      'lib/ffi.js', 
+      'lib/binary.js'
     ],
     embeds: [
       'main.cc',
@@ -292,8 +307,6 @@ const runtimes = {
       'lib/libffi/api.js',
       'lib/libssl/api.js',
       'lib/lz4/api.js',
-      'lib/inflate/em_inflate.c',
-      'lib/inflate/em_inflate.h',
       'lib/mbedtls/api.js',
       'lib/net/api.js',
       'lib/pico/api.js',
@@ -304,7 +317,7 @@ const runtimes = {
       'lib/tcc/api.js',
       'lib/wireguard/api.js',
       'lib/zlib/api.js',
-      'lib/duckdb/api.js',
+      'lib/duckdb/api.js'
     ]
   },
   full: {
@@ -365,7 +378,7 @@ const runtimes = {
       'lib/tcc/api.js',
       'lib/wireguard/api.js',
       'lib/zlib/api.js',
-      'lib/duckdb/api.js',
+      'lib/duckdb/api.js'
     ]
   },
   mbedtls: {
@@ -392,7 +405,6 @@ async function build (args) {
     args = args.filter(a => a !== '-v')
     verbose = true
   }
-  // ./lo eval "console.log(parseInt(lo.utf8Decode(lo.ptr(lo.core.readFile('/proc/self/stat', 0, 1024)).ptr, -1).match(/(\d+)\s/g)[21], 10) * 4096)"
   // it's 11 ms versus 7ms for ```hyperfine "lo eval 1"``` for curl build versus mbedtls 
   // use ```lo LINK="mold -run g++" CC="ccache g++" build.js```  for fast builds
   await create_lo_home(LO_HOME)

lib/duckdb/api.js

@@ -165,30 +165,4 @@ int set_config (duckdb_config* config, const char* key, const char* value) {
 }
 `
 
-import { isDir, isFile } from 'lib/fs.js'
-import { exec } from 'lib/proc.js'
-
-async function build (C = 'gcc', CC = 'g++') {
-  const { assert } = lo
-  const { chdir, mkdir, S_IRWXU, S_IRWXG, S_IROTH, S_IXOTH } = lo.core
-  const status = new Int32Array(2)
-
-  if (!isDir('deps/duckdb')) {
-    mkdir('deps', S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)
-    assert(chdir('deps') === 0)
-    exec('git', ['clone', '--depth', '1', '--single-branch', '-b', 'v0.9.2', 'https://github.com/duckdb/duckdb.git'], status)
-    assert(status[0] === 0)
-    assert(chdir('../') === 0)
-  }
-
-  if (obj.some(o => !isFile(o))) {
-    assert(chdir('deps/duckdb') === 0)
-    exec('python3', ['scripts/amalgamation.py'], status)
-    exec(CC.split(' ')[0], [...CC.split(' ').slice(1), '-fPIC', '-c', '-O3', '-o', 'duckdb.o', '-Isrc/amalgamation', 'src/amalgamation/duckdb.cpp'], status)
-    assert(status[0] === 0)
-    assert(chdir('../../') === 0)
-  }
-
-}
-
-export { name, api, constants, structs, obj, build, include_paths, includes, preamble }
+export { name, api, constants, structs, obj, include_paths, includes, preamble }