Merge pull request #19 from billywhizz/main

0.0.11-pre

.github/workflows/push.yml

@@ -24,6 +24,8 @@ jobs:
         if: ${{ matrix.platform == 'x64' }}
         run: |
           make check
+          brew install libffi lz4 zstd
+          ./lo test/build.js
       - name: upload runtime artifact
         uses: actions/upload-artifact@v3
         with:
@@ -42,6 +44,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: install libcurl
         run: |
+          sudo apt-get update -y
           sudo apt-get install -qy libcurl4-openssl-dev
       - name: compile
         run: |
@@ -50,6 +53,8 @@ jobs:
       - name: check
         run: |
           make check
+          sudo apt-get install -qy libffi-dev liblz4-dev libseccomp-dev libsqlite3-dev libtcc-dev zlib1g-dev tcc
+          ./lo test/build.js
       - name: upload artifact
         uses: actions/upload-artifact@v3
         with:

.gitignore

@@ -8,8 +8,11 @@ v8
 *.exp
 *.lib
 *.so
+*.dylib
 lo
 .vscode
 scratch
 deps
 notes.md
+lib/inflate/em_inflate.c
+lib/inflate/em_inflate.h

Makefile

@@ -5,7 +5,7 @@ CCARGS=-std=c++17 -c -fno-omit-frame-pointer -fno-rtti -fno-exceptions
 CARGS=-c -fno-omit-frame-pointer
 WARN=-Werror -Wpedantic -Wall -Wextra -Wno-unused-parameter
 OPT=-O3
-VERSION=0.0.10-pre
+VERSION=0.0.11-pre
 V8_VERSION=1.0.0
 RUNTIME=lo
 LO_HOME=$(shell pwd)
@@ -25,6 +25,7 @@ else
 			LARGS+=-s
     else ifeq ($(UNAME_S),Darwin)
 			os=mac
+			LARGS+=-s -w
 			ifeq ($(ARCH),arm64)
 				LARGS+=-arch arm64
 				CARGS+=-arch arm64

TODO.md

@@ -62,6 +62,12 @@
 - [ ] **todo**: handle bindings methods that are optional based on defines
 - [ ] **todo**: think about how we handle fork. can we handle it?
 - [ ] **bug**: we open file twice with the LO_HOME core loader - refactor this so it works better
+- [ ] **todo**: implement a way of building with a subset of bindings that work cross-platform for builder config - e.g. linux/mac
+                maybe just put ifdef macros in the bindings cpp source for different platforms? it's auto-generate so that's ok
+- [ ] **todo**: add ability to download and build dependencies and build our own version of a binding (e.g. libssl) or dynamically link to system library - switch on command line?
+- [ ] **todo**: fix build.js for libssl so it works for compiling openssl from scratch
+- [ ] **todo**: add a safe_wrap method to main.js so we can wrap pointers with a check for maxsafeinteger
+- [ ] **todo**: when i do ```lo build binding <new_binding>``` generate binding definition with all cases covered and lots of comments, or not...
 
 ## features
 

globals.d.ts

@@ -1,3 +1,33 @@
+const EPOLL_CLOEXEC = 524288
+const EPOLLIN = 0x1
+
+interface eventCallback { ( fd: number, events: number ): void };
+
+declare class Loop {
+  constructor(nevents?: 4096, flags?: EPOLL_CLOEXEC);
+  readonly size: Number;
+  static readonly Writable: number;
+  static readonly EdgeTriggered: number;
+  static readonly Readable: number;
+  static readonly Blocked: number;
+
+  /**
+   * Add a system resource file descriptor to the event loop with a related callback
+   * @param fd file descriptor for socket/file/timer system resource
+   * @param callback callback function that is called when an event occurs on fd
+   * @param flags optional flags - Loop.Readable | Loop.Writable | Loop.EdgeTriggered, default = Loop.Readable + Level Triggered
+   * @param errHandler optional callback called when we get ERR or HUP on fd
+   */
+  add(fd: number, callback: eventCallback, flags?: EPOLLIN, errHandler?: function): number;
+  modify(fd: number, flags?: EPOLLIN, callback: eventCallback, errHandler?: function): number;
+  remove(fd: number): number;
+  poll(timeout?: number): number;
+};
+
+export interface bindings {
+  Loop: Loop;
+};
+
 declare class TextEncoder {
   /**
    * The encoding supported by the `TextEncoder` instance. Always set to `'utf-8'`.

lib/ansi.js

@@ -0,0 +1,35 @@
+const AD = '\u001b[0m' // ANSI Default
+const A0 = '\u001b[30m' // ANSI Black
+const AR = '\u001b[31m' // ANSI Red
+const AG = '\u001b[32m' // ANSI Green
+const AY = '\u001b[33m' // ANSI Yellow
+const AB = '\u001b[34m' // ANSI Blue
+const AM = '\u001b[35m' // ANSI Magenta
+const AC = '\u001b[36m' // ANSI Cyan
+const AW = '\u001b[37m' // ANSI White
+
+const colors = { AD, A0, AR, AG, AY, AB, AM, AC, AW }
+
+const HOME = '\u001b[0;0H' // set cursor to 0,0
+const CLS = '\u001b[2J' // clear screen
+const EL = '\u001b[K' // erase line
+const SAVECUR = '\u001b[s' // save cursor
+const RESTCUR = '\u001b[u' // restore cursor
+const HIDE = '\u001b[?25l' // hide cursor
+const SHOW = '\u001b[?25h' // show cursor
+
+const control = {
+  home: () => HOME,
+  move: (x = 0, y = 0) => `\u001b[${x};${y}H`,
+  column: x => `\u001b[${x}G`,
+  cls: () => CLS,
+  eraseLine: () => EL,
+  cursor: {
+    hide: () => HIDE,
+    show: () => SHOW,
+    save: () => SAVECUR,
+    restore: () => RESTCUR
+  }
+}
+
+export { colors, control }

lib/build.js

@@ -75,6 +75,7 @@ async function create_lo_home (path) {
   chdir(cwd)
 } 
 
+// todo: change these methods to accept objects, not filenames
 async function compile_bindings (lib, verbose = false) {
   const cwd = getcwd()
   const lib_dir = `lib/${lib}`
@@ -128,25 +129,28 @@ async function compile_bindings (lib, verbose = false) {
   console.log(`${AY}shared lib ${AD} ${def.name}.so ${AY}with${AG} ${CC}${AD}`)
   unlink(`${def.name}.so`)
   if (os === 'mac') {
-    exec2([...LINK.split(' '), ...LARGS, OPT, '-bundle', ...WARN, '-o', 
+    exec2([...LINK.split(' '), '-bundle_loader', '../../lo', ...LARGS, OPT, '-bundle', ...WARN, '-o', 
       `${def.name}.so`, `${def.name}.o`, 
       ...(def.obj || []).filter(f => extName(f) === 'a'), 
-      ...(def.libs || []).map(l => `-l${l}`)],
+      ...(def.libs || []).map(l => `-l${l}`), 
+      ...(def.lib_paths || []).map(l => `-L${l}`)],
       verbose)
   } else if (os === 'linux') {
     // todo: why do we not include .o obj files here?
     // confused by this. why does linking against the .a file not work?
     exec2([...LINK.split(' '), ...LARGS, OPT, '-shared', ...WARN, '-o', 
       `${def.name}.so`, `${def.name}.o`, 
-      ...(def.obj || []).filter(f => extName(f) === 'a'), 
-      ...(def.libs || []).map(l => `-l${l}`)],
+      ...(def.obj || []).filter(f => extName(f) === 'o'), 
+      ...(def.libs || []).map(l => `-l${l}`), 
+      ...(def.lib_paths || []).map(l => `-L${l}`)],
       verbose)
   }
   console.log(`${AY}change dir to ${AD} ${cwd}`)
   assert(chdir(cwd) === 0)
 
   if (!def.obj) return []
-  return def.obj.filter(f => extName(f) === 'a').map(f => `${lib_dir}/${f}`)
+  // todo: this is kind of a hack - if it is an absolute path, don't prefix it with binding path
+  return def.obj.filter(f => extName(f) === 'a').map(f => f.slice(0, 1) === '/' ? f : `${lib_dir}/${f}`)
 }
 
 function create_builtins (libs = [], os) {
@@ -206,10 +210,9 @@ async function build_runtime ({ libs = lo.builtins(), bindings = lo.libraries(),
     static_libs = static_libs.concat(await compile_bindings(binding, verbose))
   }
   const dynamic_libs = await linkArgs(bindings.map(n => `lib/${n}/api.js`))
-  const mbed_tls = []
   exec2([...LINK.split(' '), ...LARGS, OPT, ...LINK_TYPE, ...WARN, '-o', 
     `${TARGET}`, `${TARGET}.o`, 'main.o', 'builtins.o', 'v8/libv8_monolith.a', 
-    ...static_libs, ...mbed_tls, ...dynamic_libs], verbose) 
+    ...static_libs, ...dynamic_libs], verbose) 
   assert(chdir(cwd) === 0)
 }
 
@@ -238,18 +241,13 @@ const encoder = new TextEncoder()
 const status = new Int32Array(2)
 
 // todo: clean up api so we can pass a config in and run builds through api
-const VERSION = getenv('VERSION') || '"0.0.10pre"'
+const VERSION = getenv('VERSION') || '"0.0.11pre"'
 const RUNTIME = getenv('RUNTIME') || '"lo"'
 const TARGET = getenv('TARGET') || 'lo'
 const LINK_TYPE = (getenv('LINK_TYPE') || '-rdynamic').split(' ')
-const C = getenv('C') || 'gcc'
-const CC = getenv('CC') || 'g++'
-const LINK = getenv('LINK') || 'g++'
 const OPT = getenv('OPT') || '-O3'
-const CFLAGS = (getenv('CFLAGS') || '-fPIC -std=c++17 -c -DV8_COMPRESS_POINTERS -DV8_TYPED_ARRAY_MAX_SIZE_IN_HEAP=0').split(' ')
 const WARN = (getenv('WARN') || 
   '-Werror -Wpedantic -Wall -Wextra -Wno-unused-parameter').split(' ')
-const LARGS = (getenv('LARGS') || '-s').split(' ')
 const LO_HOME = getenv('LO_HOME') || getcwd()
 const v8 = getenv('V8_VERSION') || '1.0.0'
 const os = getenv('LO_OS') || lo.os()
@@ -263,11 +261,40 @@ const defaultOpts = {
   v8_cleanup: 0, v8_threads: 2, on_exit: 0,
   v8flags: '--stack-trace-limit=10 --use-strict --turbo-fast-api-calls --no-freeze-flags-after-init'
 }
+// todo: clean this up
+let c_compiler = 'gcc'
+if (os === 'mac') {
+  if (arch === 'arm64') {
+    c_compiler = 'clang -arch arm64'
+  } else {
+    c_compiler = 'clang'
+  }
+}
+let cc_compiler = 'g++'
+if (os === 'mac') {
+  if (arch === 'arm64') {
+    cc_compiler = 'clang++ -arch arm64'
+  } else {
+    cc_compiler = 'clang++'
+  }
+}
+let link_args = '-s'
+if (os === 'mac') {
+  if (arch === 'arm64') {
+    link_args = '-s -arch arm64 -w'
+  } else {
+    link_args = '-s -w'
+  }
+}
 
+const LINK = getenv('LINK') || cc_compiler
+const C = getenv('C') || c_compiler
+const CC = getenv('CC') || cc_compiler
+const CFLAGS = (getenv('CFLAGS') || `-fPIC -std=c++17 -c -DV8_COMPRESS_POINTERS -DV8_TYPED_ARRAY_MAX_SIZE_IN_HEAP=0`).split(' ')
+const LARGS = (getenv('LARGS') || link_args).split(' ')
+const so_ext = (os === 'linux' ? 'so' : (os === 'mac' ? 'so' : 'dll'))
 config.os = os
 
-const so_ext = (os === 'linux' ? 'so' : (os === 'mac' ? 'dylib' : 'dll'))
-
 const runtimes = {
   core: {
     bindings: [
@@ -321,23 +348,37 @@ const runtimes = {
     ]
   },
   full: {
-    bindings: [
+    bindings: (os === 'linux' ? [
       'core', 
+      'curl',
+      'duckdb',
+      'encode', 
+      'epoll', 
+      'inflate',
+      'libffi',
+      'libssl',
+      'lz4', 
+      'net', 
+      'pico', 
+      'pthread', 
+      'sqlite', 
+      'system', 
+      'zlib'
+    ] : [
+      'core', 
+      'curl',
+      'encode', 
       'inflate', 
-      'zlib', 
+      'libffi', 
       'libssl', 
+      'net', 
       'pico', 
+      'pthread', 
       'sqlite', 
       'system', 
-      'pthread', 
-      'net', 
-      'lz4', 
-      'epoll', 
-      'encode', 
-      'duckdb',
-      'curl'
-    ],
-    libs: [
+      'zlib'      
+    ]),
+    libs: (os === 'linux' ? [
       'lib/bench.js', 
       'lib/gen.js', 
       'lib/fs.js', 
@@ -353,8 +394,23 @@ const runtimes = {
       'lib/tcc.js', 
       'lib/zlib.js',
       'lib/sqlite.js'
-    ],
-    embeds: [
+    ]: [
+      'lib/bench.js', 
+      'lib/gen.js', 
+      'lib/fs.js', 
+      'lib/untar.js', 
+      'lib/proc.js', 
+      'lib/path.js', 
+      'lib/inflate.js', 
+      'lib/curl.js', 
+      'lib/build.js', 
+      'lib/asm.js', 
+      'lib/ffi.js', 
+      'lib/binary.js', 
+      'lib/zlib.js',
+      'lib/sqlite.js'
+    ]),
+    embeds: (os === 'linux' ? [
       'main.cc',
       'lo.cc', 
       'lo.h',
@@ -379,7 +435,27 @@ const runtimes = {
       'lib/wireguard/api.js',
       'lib/zlib/api.js',
       'lib/duckdb/api.js'
-    ]
+    ] : [
+      'main.cc',
+      'lo.cc', 
+      'lo.h',
+      'lib/core/api.js',
+      'lib/curl/api.js',
+      'lib/encode/api.js',
+      'lib/inflate/api.js',
+      'lib/libffi/api.js',
+      'lib/libssl/api.js',
+      'lib/inflate/em_inflate.c',
+      'lib/inflate/em_inflate.h',
+      'lib/mbedtls/api.js',
+      'lib/net/api.js',
+      'lib/pico/api.js',
+      'lib/pthread/api.js',
+      'lib/sqlite/api.js',
+      'lib/system/api.js',
+      'lib/zlib/api.js',
+      'lib/duckdb/api.js'
+    ])
   },
   mbedtls: {
     bindings: [