Skip to content

Commit

Permalink
bytecode cross-compiled on dev machine (#481)
Browse files Browse the repository at this point in the history 
* using cli tools to convert to c header files

* add string lengths for ii files too

* bytecode cross-compilation & loading working

* cleanup makefile and document lua bc cross-compilation process

* fix dependencies in makefile

* remove lua 5.2 compatibility. unused anyway

* tracking lua globals for env clear

* fix buffer name when loading scripts

* support crow which doesn't have require

* lua env no longer loads some useless modules
  • Loading branch information
@trentgill
trentgill authored Apr 3, 2023
1 parent 862b013 commit f0291e4
Show file tree
Hide file tree
Showing 10 changed files with 110 additions and 117 deletions.
32 changes: 23 additions & 9 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ LUAS=submodules/lua/src
BOOTLOADER=submodules/dfu-stm32f7
BUILD_DIR := build
PRJ_DIR=crow
LUAC_CROSS=util/luacc

CC=arm-none-eabi-gcc
LD=arm-none-eabi-gcc
Expand Down Expand Up @@ -53,7 +54,8 @@ CFLAGS += $(MCFLAGS)
CFLAGS += $(OPTIMIZE)
CFLAGS += $(DEFS) -I. -I./ $(STM32_INCLUDES)
CFLAGS += -fsingle-precision-constant -Wdouble-promotion
CFLAGS += -DLUA_32BITS -DLUA_COMPAT_5_2
CFLAGS += -DLUA_32BITS
# CFLAGS += -DLUA_32BITS -DLUA_COMPAT_5_2
CFLAGS += -fno-common
CFLAGS += -DVERSION=\"$(GIT_VERSION)\"
CFLAGS += -ffunction-sections -fdata-sections # provides majority of LTO binary size reduction
Expand Down Expand Up @@ -141,19 +143,19 @@ $(II_TARGET): util/ii_lua_module.lua

$(BUILD_DIR)/ii_%.lua: $(II_SRCD)/%.lua util/ii_lua_module.lua | $(BUILD_DIR)
@lua util/ii_lua_module.lua $< $@
@echo lua $@
@echo "ii-lua-module $< -> $@"

$(BUILD_DIR)/iihelp.lua: $(II_SRC) util/ii_lua_help.lua | $(BUILD_DIR)
@lua util/ii_lua_help.lua $(II_SRCD) $@
@echo lua $@
@echo "ii-lua-help $@"

$(BUILD_DIR)/ii_c_layer.h: $(II_SRC) util/ii_c_layer.lua | $(BUILD_DIR)
@lua util/ii_c_layer.lua $(II_SRCD) $@
@echo lua $@
@echo "ii-c-layer $@"

$(BUILD_DIR)/ii_lualink.h: $(II_SRC) util/ii_lualinker.lua | $(BUILD_DIR)
@lua util/ii_lualinker.lua $(II_SRCD) $@
@echo lua $@
@echo "ii-lualinker $@"


### destination sources
Expand All @@ -180,6 +182,7 @@ LUA_SRC += $(II_TARGET)

LUA_PP = $(LUA_SRC:%.lua=%.lua.h)
LUA_PP: $(LUA_SRC)
@echo "pre-compiling lua sources to bytecode wrapped in c headers"

LUACORE_OBJS= lapi.o lcode.o lctype.o ldebug.o ldo.o ldump.o lfunc.o lgc.o llex.o \
lmem.o lobject.o lopcodes.o lparser.o lstate.o lstring.o ltable.o \
Expand All @@ -194,6 +197,8 @@ OBJS = $(SRC:%.c=$(OBJDIR)/%.o)
OBJS += $(addprefix $(LUAS)/,$(LUACORE_OBJS) $(LUALIB_OBJS) )
OBJS += Startup.o

$(OBJS): $(LUA_PP)

# specific objects that require built dependencies (ii)
$(OBJDIR)/lib/l_bootstrap.o: $(LUA_PP) $(BUILD_DIR)/ii_lualink.h
# $(OBJDIR)/lib/lualink.o: $(LUA_PP) $(BUILD_DIR)/ii_lualink.h
Expand Down Expand Up @@ -301,10 +306,19 @@ zip: $(BIN) $(TARGET).dfu
@echo f2l $< "->" $@
@$(FENNEL) --compile $< > $@

%.lua.h: %.lua util/l2h.lua
@luac -p $<
@echo l2h $< "->" $@
@lua util/l2h.lua $<
# a bunch of gnarly make-functions to massage the intermediate stage filenames
# everything goes into /build now, and we have to save output of LUAC_CROSS into
# named files for (xxd -i) to build include files with valid names
# could be avoided by a more complicated pass in (sed), but this was easier
# 1. cross-compile all .lua files into .lc bytecode for stm32-arm-cortex-m7 format
# 2. wrap the .lc binary files into .h headers with auto-generated names
# 3. add const qualifiers to headers to satisfy C99 struct initializer requirement

%.lua.h: %.lua $(BUILD_DIR)
@echo l2h $< "->" $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua.h,.h,$@)))
@$(LUAC_CROSS) -s -o $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua,.lc,$<))) $<
@xxd -i $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua,.lc,$<))) $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua.h,.h,$@)))
@sed -i 's/unsigned int/const unsigned int/g' $(addprefix $(BUILD_DIR)/, $(notdir $(subst .lua.h,.h,$@)))

Startup.o: $(STARTUP)
@$(CC) $(CFLAGS) -c $< -o $@
Expand Down
141 changes: 57 additions & 84 deletions lib/l_bootstrap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,46 +4,46 @@

#include "l_crowlib.h"

#include "lua/crowlib.lua.h"
#include "lua/asl.lua.h"
#include "lua/asllib.lua.h"
#include "lua/clock.lua.h"
#include "lua/metro.lua.h"
#include "lua/public.lua.h"
#include "lua/input.lua.h"
#include "lua/output.lua.h"
#include "lua/ii.lua.h"
#include "build/iihelp.lua.h" // generated lua stub for loading i2c modules
#include "lua/calibrate.lua.h"
#include "lua/sequins.lua.h"
#include "lua/quote.lua.h"
#include "lua/timeline.lua.h"
#include "lua/hotswap.lua.h"
// Lua libs wrapped in C-headers
#include "build/crowlib.h"
#include "build/asl.h"
#include "build/asllib.h"
#include "build/clock.h"
#include "build/metro.h"
#include "build/public.h"
#include "build/input.h"
#include "build/output.h"
#include "build/ii.h"
#include "build/iihelp.h" // generated lua stub for loading i2c modules
#include "build/calibrate.h"
#include "build/sequins.h"
#include "build/quote.h"
#include "build/timeline.h"
#include "build/hotswap.h"

#include "build/ii_lualink.h" // generated C header for linking to lua

static int _writer(lua_State *L, const void *p, size_t sz, void *ud);
static int _load_chunk(lua_State* L, const char* code, int strip);
static int _open_lib( lua_State *L, const struct lua_lib_locator* lib, const char* name );
static void lua_full_gc(lua_State* L);

// mark the 3rd arg 'false' if you need to debug that library
const struct lua_lib_locator Lua_libs[] =
{ { "lua_crowlib" , lua_crowlib , true}
, { "lua_asl" , lua_asl , true}
, { "lua_asllib" , lua_asllib , true}
, { "lua_clock" , lua_clock , true}
, { "lua_metro" , lua_metro , true}
, { "lua_input" , lua_input , true}
, { "lua_output" , lua_output , true}
, { "lua_public" , lua_public , true}
, { "lua_ii" , lua_ii , true}
, { "build_iihelp" , build_iihelp , true}
, { "lua_calibrate" , lua_calibrate , true}
, { "lua_sequins" , lua_sequins , true}
, { "lua_quote" , lua_quote , true}
, { "lua_timeline" , lua_timeline , true}
, { "lua_hotswap" , lua_hotswap , true}
, { NULL , NULL , true}
{ { "lua_crowlib" , build_crowlib_lc , true, build_crowlib_lc_len}
, { "lua_asl" , build_asl_lc , true, build_asl_lc_len}
, { "lua_asllib" , build_asllib_lc , true, build_asllib_lc_len}
, { "lua_clock" , build_clock_lc , true, build_clock_lc_len}
, { "lua_metro" , build_metro_lc , true, build_metro_lc_len}
, { "lua_input" , build_input_lc , true, build_input_lc_len}
, { "lua_output" , build_output_lc , true, build_output_lc_len}
, { "lua_public" , build_public_lc , true, build_public_lc_len}
, { "lua_ii" , build_ii_lc , true, build_ii_lc_len}
, { "build_iihelp" , build_iihelp_lc , true, build_iihelp_lc_len}
, { "lua_calibrate" , build_calibrate_lc , true, build_calibrate_lc_len}
, { "lua_sequins" , build_sequins_lc , true, build_sequins_lc_len}
, { "lua_quote" , build_quote_lc , true, build_quote_lc_len}
, { "lua_timeline" , build_timeline_lc , true, build_timeline_lc_len}
, { "lua_hotswap" , build_hotswap_lc , true, build_hotswap_lc_len}
, { NULL , NULL , true, 0}
};


Expand All @@ -69,9 +69,18 @@ void l_bootstrap_init(lua_State* L){
// crowlib C extensions
l_crowlib_init(L);

// track all user-created globals
luaL_dostring(L,
"_user={}\n"
"local function trace(t,k,v)\n"
"_user[k]=true\n"
"rawset(t,k,v)\n"
"end\n"
"setmetatable(_G,{ __newindex = trace })\n"
);

// perform two full garbage collection cycles for full cleanup
lua_gc(L, LUA_GCCOLLECT, 1);
lua_gc(L, LUA_GCCOLLECT, 1);
lua_full_gc(L);
}


Expand All @@ -92,19 +101,22 @@ int l_bootstrap_dofile(lua_State* L)
default:{ cname[p++] = l_name[i]; } break;
}
}
// goto fail; // no match was found, so error out (silently?)

strcomplete:
lua_pop( L, 1 );
switch( _open_lib( L, Lua_libs, cname ) ){
case -1: goto fail;
case 1: return 1;
case 1: lua_full_gc(L); return 1;
default: break;
}
switch( _open_lib( L, Lua_ii_libs, cname ) ){
case -1: goto fail;
case 1: return 1;
case 1: lua_full_gc(L); return 1;
default: break;
}
printf("can't open library: %s\n", (char*)cname);

fail:
lua_pushnil(L);
return 1;
Expand All @@ -117,58 +129,14 @@ int l_bootstrap_dofile(lua_State* L)

/////////// private defns

// this is a somewhat arbitrary size. must be big enough for the biggest library. 8kB was too small
#define SIZED_STRING_LEN 0x4000 // (16kB)
struct sized_string{
char data[SIZED_STRING_LEN];
int len;
};

#define UNUSED(x) (void)(sizeof(x))
static int _writer(lua_State *L, const void *p, size_t sz, void *ud)
{
UNUSED(L);
struct sized_string* chunkstr = ud; /// need explicit cast?
if(chunkstr->len + sz >= SIZED_STRING_LEN){
printf("chunkstr too small.\n");
return 1;
}
memcpy(&chunkstr->data[chunkstr->len], p, sz);
chunkstr->len += sz;
return 0;
}
#undef UNUSED

static int _load_chunk(lua_State* L, const char* code, int strip)
{
int retval = 0;
struct sized_string chunkstr = {.len = 0};
{ // scope lua_State to destroy it asap
lua_State* LL=luaL_newstate();
if( !LL ){ printf("luaL_newstate failed\n"); return 1; }
if( luaL_loadstring(LL, code) ){
printf("loadstring error\n");
retval = 1;
goto close_LL;
}
if( lua_dump(LL, _writer, &chunkstr, strip) ){
printf("dump error\n");
retval = 1;
goto close_LL;
}
close_LL:
lua_close(LL);
}
luaL_loadbuffer(L, chunkstr.data, chunkstr.len, chunkstr.data); // load our compiled chunk
return retval;
}

static int _open_lib( lua_State *L, const struct lua_lib_locator* lib, const char* name )
{
uint8_t i = 0;
while( lib[i].addr_of_luacode != NULL ){
if( !strcmp( name, lib[i].name ) ){ // if the strings match
if( _load_chunk(L, lib[i].addr_of_luacode, lib[i].stripped) ){
if( luaL_loadbuffer(L, (const char*)lib[i].addr_of_luacode
, lib[i].len
, lib[i].name) ){
printf("can't load library: %s\n", (char*)lib[i].name );
printf( "%s\n", (char*)lua_tostring( L, -1 ) );
lua_pop( L, 1 );
Expand All @@ -186,3 +154,8 @@ static int _open_lib( lua_State *L, const struct lua_lib_locator* lib, const cha
}
return 0; // not found
}

static void lua_full_gc(lua_State* L){
lua_gc(L, LUA_GCCOLLECT, 1);
lua_gc(L, LUA_GCCOLLECT, 1);
}
27 changes: 18 additions & 9 deletions lib/lualink.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@
#define WATCHDOG_FREQ 0x100000 // ~1s how often we run the watchdog
#define WATCHDOG_COUNT 2 // how many watchdogs before 'frozen'


// Basic crow script
#include "lua/First.lua.h"
#include "build/First.h"

// Private prototypes
static void Lua_linkctolua( lua_State* L );
Expand Down Expand Up @@ -80,10 +81,17 @@ lua_State* Lua_Init(void)
luaL_openlibs(L);
Lua_linkctolua(L);
l_bootstrap_init(L); // redefine dofile(), print(), load crowlib
// Lua_eval(L, lua_bootstrap
// , strlen(lua_bootstrap)
// , "=lib"
// );
return L;
}

lua_State* Lua_ReInit_Environment(lua_State* L){
// clear user-created globals
luaL_dostring(L, "for k,_ in pairs(_user) do\n"
"_G[k] = nil\n"
"end\n"
"_G._user = {}\n");
lua_gc(L, LUA_GCCOLLECT, 1);
lua_gc(L, LUA_GCCOLLECT, 1);
return L;
}

Expand All @@ -99,14 +107,15 @@ lua_State* Lua_Reset( void )
}
events_clear();
clock_cancel_coro_all();
Lua_DeInit();
return Lua_Init();
return Lua_ReInit_Environment(L);
// Lua_DeInit();
// return Lua_Init();
}

void Lua_load_default_script( void )
{
Lua_eval(L, lua_First
, strlen(lua_First)
Lua_eval(L, (const char*)build_First_lc
, build_First_lc_len
, "=First.lua"
);
}
Expand Down
3 changes: 2 additions & 1 deletion lib/lualink.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@
typedef void (*ErrorHandler_t)(char* error_message);
struct lua_lib_locator{
const char* name;
const char* addr_of_luacode;
const unsigned char* addr_of_luacode;
const bool stripped;
const unsigned int len;
};

extern volatile int CPU_count; // count from main.c
Expand Down
11 changes: 1 addition & 10 deletions lib/repl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ typedef enum{ REPL_normal
// global variables
lua_State* Lua;
L_repl_mode repl_mode = REPL_normal;
char* new_script;
char new_script[USER_SCRIPT_SIZE]; // static alloc avoids malloc failures
uint16_t new_script_len;
static bool running_from_mem;
static char running_script_name[64];
Expand Down Expand Up @@ -48,7 +48,6 @@ void REPL_init( lua_State* lua )
printf("failed to load user script\n");
Caw_send_luachunk("failed to load user script");
}
free(new_script);
break;
}
case USERSCRIPT_Clear:
Expand Down Expand Up @@ -97,7 +96,6 @@ void REPL_upload( int flash )
} else {
Caw_send_luachunk("User script evaluation failed.");
}
free(new_script);
}
repl_mode = REPL_normal;
}
Expand Down Expand Up @@ -194,7 +192,6 @@ static void REPL_receive_script( char* buf, uint32_t len, ErrorHandler_t errfn )
if( new_script_len + len >= USER_SCRIPT_SIZE ){
Caw_send_luachunk("!ERROR! Script is too long.");
repl_mode = REPL_discard;
free(new_script);
} else {
memcpy( &new_script[new_script_len], buf, len );
new_script_len += len;
Expand All @@ -203,12 +200,6 @@ static void REPL_receive_script( char* buf, uint32_t len, ErrorHandler_t errfn )

static bool REPL_new_script_buffer( uint32_t len )
{
new_script = calloc( sizeof(char), len);
if( new_script == NULL ){
printf("malloc failed. REPL\n");
Caw_send_luachunk("!ERROR! Out of memory.");
return false;
}
new_script_len = 0; // reset counter as the buffer is empty
return true;
}
Expand Down
1 change: 1 addition & 0 deletions lua/hotswap.lua
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
--- hotswap library

--- globals are available on crow, otherwise require for norns
local require = require or function() end
local s = sequins or require 'lib/sequins'
local tl = timeline or require 'lib/timeline'

Expand Down
Loading

0 comments on commit f0291e4

Please sign in to comment.