flamenco, ci: support choosing page sz for sol compat wksp

firedancer-io · Jan 8, 2025 · 2565d94 · 2565d94
1 parent 6fe5c25
commit 2565d94
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 27 deletions.
diff --git a/contrib/test/run_test_vectors.sh b/contrib/test/run_test_vectors.sh
@@ -34,31 +34,31 @@ git checkout -q $GIT_REF
 cd ../..
 
 LOG=$LOG_PATH/test_exec_syscall
-cat contrib/test/test-vectors-fixtures/syscall-fixtures/*.list | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
+cat contrib/test/test-vectors-fixtures/syscall-fixtures/*.list | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
 
 LOG=$LOG_PATH/test_exec_interp
-cat contrib/test/test-vectors-fixtures/vm-interp-fixtures/*.list | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
-find dump/test-vectors/vm_interp/fixtures/v0 -type f -name '*.fix' | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
-find dump/test-vectors/vm_interp/fixtures/v1 -type f -name '*.fix' | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
-find dump/test-vectors/vm_interp/fixtures/v2 -type f -name '*.fix' | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
+cat contrib/test/test-vectors-fixtures/vm-interp-fixtures/*.list | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+find dump/test-vectors/vm_interp/fixtures/v0 -type f -name '*.fix' | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+find dump/test-vectors/vm_interp/fixtures/v1 -type f -name '*.fix' | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+find dump/test-vectors/vm_interp/fixtures/v2 -type f -name '*.fix' | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
 
 LOG=$LOG_PATH/test_exec_precompiles
-cat contrib/test/test-vectors-fixtures/precompile-fixtures/*.list | xargs -P 4 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
+cat contrib/test/test-vectors-fixtures/precompile-fixtures/*.list | xargs -P 4 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
 
 LOG=$LOG_PATH/test_exec_txn
-cat contrib/test/test-vectors-fixtures/txn-fixtures/*.list | xargs -P 4 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
+cat contrib/test/test-vectors-fixtures/txn-fixtures/*.list | xargs -P 4 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
 
 zstd -df dump/test-vectors/elf_loader/fixtures/*.zst
 LOG=$LOG_PATH/test_elf_loader
-cat contrib/test/test-vectors-fixtures/elf-loader-fixtures/*.list | xargs -P 4 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
+cat contrib/test/test-vectors-fixtures/elf-loader-fixtures/*.list | xargs -P 4 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
 
 LOG=$LOG_PATH/test_exec_instr
-cat contrib/test/test-vectors-fixtures/instr-fixtures/*.list | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG
+cat contrib/test/test-vectors-fixtures/instr-fixtures/*.list | xargs -P 8 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
 
 # check if ./$OBJDIR/unit-test/test_exec_sol_compat_stubbed exists
 if [ -f ./$OBJDIR/unit-test/test_exec_sol_compat_stubbed ]; then
   LOG=$LOG_PATH/test_exec_cpi
-  cat contrib/test/test-vectors-fixtures/cpi-fixtures/*.list | xargs -P 4 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat_stubbed --log-path $LOG
+  cat contrib/test/test-vectors-fixtures/cpi-fixtures/*.list | xargs -P 4 -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat_stubbed --log-path $LOG --wksp-page-sz 1073741824
 else
   # skip stubbed tests if the binary does not exist
   echo -e "\e[38;5;214mNOTICE:\e[0m Skipping stubbed tests due to missing stub binary"

diff --git a/src/flamenco/runtime/tests/fd_exec_sol_compat.c b/src/flamenco/runtime/tests/fd_exec_sol_compat.c
@@ -33,13 +33,14 @@ typedef struct {
 
 static sol_compat_features_t features;
 static       uchar *     smem;
-static const ulong       smax = 1UL<<27;
+static const ulong       smax = 256UL<<20;
 
 static       uchar *     spad_mem;
 
 static       fd_wksp_t * wksp = NULL;
 
-#define WKSP_TAG 2
+#define WKSP_EXECUTE_ALLOC_TAG (2UL)
+#define WKSP_INIT_ALLOC_TAG    (3UL)
 
 void
 sol_compat_init( int log_level ) {
@@ -56,25 +57,35 @@ sol_compat_init( int log_level ) {
   fd_flamenco_boot( NULL, NULL );
   fd_log_level_core_set(4);  /* abort on FD_LOG_ERR */
 
-  sol_compat_wksp_init();
+  sol_compat_wksp_init( FD_SHMEM_NORMAL_PAGE_SZ );
 }
 
 void
-sol_compat_wksp_init( void ) {
+sol_compat_wksp_init( ulong wksp_page_sz ) {
   ulong cpu_idx = fd_tile_cpu_id( fd_tile_idx() );
   if( cpu_idx>=fd_shmem_cpu_cnt() ) cpu_idx = 0UL;
-  wksp = fd_wksp_new_anonymous( FD_SHMEM_NORMAL_PAGE_SZ, 65536UL * 18UL, fd_shmem_cpu_idx( fd_shmem_numa_idx( cpu_idx ) ), "wksp", 0UL );
+  switch( wksp_page_sz )
+  {
+  case FD_SHMEM_GIGANTIC_PAGE_SZ:
+    wksp = fd_wksp_new_anonymous( FD_SHMEM_GIGANTIC_PAGE_SZ, 5UL, fd_shmem_cpu_idx( fd_shmem_numa_idx( cpu_idx ) ), "wksp", 0UL );
+    break;
+  case FD_SHMEM_NORMAL_PAGE_SZ:
+    wksp = fd_wksp_new_anonymous( FD_SHMEM_NORMAL_PAGE_SZ, 512UL * 512UL * 5UL, fd_shmem_cpu_idx( fd_shmem_numa_idx( cpu_idx ) ), "wksp", 0UL );
+    break;
+  default:
+    FD_LOG_ERR(( "Unsupported page size %lu", wksp_page_sz ));
+  }
   assert( wksp );
 
-  spad_mem = fd_wksp_alloc_laddr( wksp, FD_SPAD_ALIGN, FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_FUZZ, 3 ); /* 4738713960 B */
+  spad_mem = fd_wksp_alloc_laddr( wksp, FD_SPAD_ALIGN, FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_FUZZ, WKSP_INIT_ALLOC_TAG ); /* 4738713960 B */
   assert( spad_mem );
 
-  smem = malloc( smax );  /* 128 MB */
+  smem = fd_wksp_alloc_laddr( wksp, FD_SCRATCH_SMEM_ALIGN, smax, WKSP_INIT_ALLOC_TAG );  /* 256 MB */
   assert( smem );
 
   features.struct_size         = sizeof(sol_compat_features_t);
-  features.cleaned_up_features = malloc( FD_FEATURE_ID_CNT * sizeof(ulong) );
-  features.supported_features  = malloc( FD_FEATURE_ID_CNT * sizeof(ulong) );
+  features.cleaned_up_features = fd_wksp_alloc_laddr( wksp, 8UL, FD_FEATURE_ID_CNT * sizeof(ulong), WKSP_INIT_ALLOC_TAG );
+  features.supported_features  = fd_wksp_alloc_laddr( wksp, 8UL, FD_FEATURE_ID_CNT * sizeof(ulong), WKSP_INIT_ALLOC_TAG );
 
   for( const fd_feature_id_t * current_feature = fd_feature_iter_init(); !fd_feature_iter_done( current_feature ); current_feature = fd_feature_iter_next( current_feature ) ) {
     // Skip reverted features
@@ -91,10 +102,10 @@ sol_compat_wksp_init( void ) {
 void
 sol_compat_fini( void ) {
   fd_wksp_free_laddr( spad_mem );
+  fd_wksp_free_laddr( smem );
+  fd_wksp_free_laddr( features.cleaned_up_features );
+  fd_wksp_free_laddr( features.supported_features );
   fd_wksp_delete_anonymous( wksp );
-  free( smem );
-  free( features.cleaned_up_features );
-  free( features.supported_features );
   wksp     = NULL;
   smem     = NULL;
   spad_mem = NULL;
@@ -103,7 +114,7 @@ sol_compat_fini( void ) {
 void
 sol_compat_check_wksp_usage( void ) {
   fd_wksp_usage_t usage[1];
-  ulong tags[1] = { WKSP_TAG };
+  ulong tags[1] = { WKSP_EXECUTE_ALLOC_TAG };
   fd_wksp_usage( wksp, tags, 1, usage );
   if( usage->used_sz ) {
     FD_LOG_ERR(( "%lu bytes leaked in %lu allocations", usage->used_sz, usage->used_cnt ));
@@ -123,8 +134,8 @@ sol_compat_setup_scratch_and_runner( void * fmem ) {
   fd_scratch_push();
 
   // Setup test runner
-  void * runner_mem = fd_wksp_alloc_laddr( wksp, fd_exec_instr_test_runner_align(), fd_exec_instr_test_runner_footprint(), WKSP_TAG );
-  fd_exec_instr_test_runner_t * runner = fd_exec_instr_test_runner_new( runner_mem, spad_mem, WKSP_TAG );
+  void * runner_mem = fd_wksp_alloc_laddr( wksp, fd_exec_instr_test_runner_align(), fd_exec_instr_test_runner_footprint(), WKSP_EXECUTE_ALLOC_TAG );
+  fd_exec_instr_test_runner_t * runner = fd_exec_instr_test_runner_new( runner_mem, spad_mem, WKSP_EXECUTE_ALLOC_TAG );
   return runner;
 }
 

diff --git a/src/flamenco/runtime/tests/fd_exec_sol_compat.h b/src/flamenco/runtime/tests/fd_exec_sol_compat.h
@@ -9,7 +9,7 @@
 FD_PROTOTYPES_BEGIN
 
 void
-sol_compat_wksp_init( void );
+sol_compat_wksp_init( ulong wksp_page_sz );
 
 void
 sol_compat_fini( void );

diff --git a/src/flamenco/runtime/tests/test_exec_sol_compat.c b/src/flamenco/runtime/tests/test_exec_sol_compat.c
@@ -60,7 +60,11 @@ main( int     argc,
       char ** argv ) {
   fd_boot( &argc, &argv );
   fd_flamenco_boot( NULL, NULL );
-  sol_compat_wksp_init();
+  ulong wksp_page_sz = fd_env_strip_cmdline_ulong( &argc, &argv, "--wksp-page-sz", "SOL_COMPAT_WKSP_PAGE_SZ", ULONG_MAX );
+  if( wksp_page_sz == ULONG_MAX ) {
+    wksp_page_sz = FD_SHMEM_NORMAL_PAGE_SZ;
+  }
+  sol_compat_wksp_init( wksp_page_sz );
   ulong fmem[ 64 ];
 
   ulong fail_cnt = 0UL;