Merge branch 'main' into asiegel/http-bench

firedancer-io · Nov 8, 2024 · e320e7a · e320e7a
2 parents fd7c9b5 + ad535ed
commit e320e7a
Show file tree

Hide file tree

Showing 22 changed files with 76 additions and 44 deletions.
diff --git a/contrib/test/test-vectors-fixtures/cpi-fixtures/cpi.list b/contrib/test/test-vectors-fixtures/cpi-fixtures/cpi.list
@@ -3,3 +3,4 @@ dump/test-vectors/cpi/fixtures/cpi_prepare_missing_acct.fix
 dump/test-vectors/cpi/fixtures/18387a7e499df53e20db216f4520bca9ce0727e7_1582213.fix
 dump/test-vectors/cpi/fixtures/dead_non_executable_callee.fix
 dump/test-vectors/cpi/fixtures/850b8f0286d8f4793a4f447c917164247e74a803_302825.fix
+dump/test-vectors/cpi/fixtures/b06b5c4824c38f9bd935756a2d383c01d4dad885_228827.fix
diff --git a/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c b/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c
@@ -640,14 +640,33 @@ VM_SYSCALL_CPI_ENTRYPOINT( void *  _vm,
     return err;
   }
 
+  /* Create the instruction to execute (in the input format the FD runtime expects) from
+     the translated CPI ABI inputs. */
+  fd_instr_info_t * instruction_to_execute = &vm->instr_ctx->txn_ctx->instr_infos[ vm->instr_ctx->txn_ctx->instr_info_cnt ];
+
+  vm->instr_ctx->txn_ctx->instr_info_cnt++;
+  if( FD_UNLIKELY( vm->instr_ctx->txn_ctx->instr_info_cnt>FD_MAX_INSTRUCTION_TRACE_LENGTH ) ) {
+     return FD_EXECUTOR_INSTR_ERR_MAX_INSN_TRACE_LENS_EXCEEDED;;
+  }
+
+  err = VM_SYSCALL_CPI_INSTRUCTION_TO_INSTR_FUNC( vm, cpi_instruction, cpi_account_metas, program_id, data, instruction_to_execute );
+  if( FD_UNLIKELY( err ) ) return err;
+
+  /* Prepare the instruction for execution in the runtime. This is required by the runtime
+     before we can pass an instruction to the executor. */
+  fd_instruction_account_t instruction_accounts[256];
+  ulong instruction_accounts_cnt;
+  err = fd_vm_prepare_instruction( vm->instr_ctx->instr, instruction_to_execute, vm->instr_ctx, instruction_accounts, &instruction_accounts_cnt, signers, signers_seeds_cnt );
+  if( FD_UNLIKELY( err ) ) return err;
+
   /* Translate account infos ******************************************/
   /* This is the equivalent of translate_slice in translate_account_infos:
      https://github.com/firedancer-io/agave/blob/838c1952595809a31520ff1603a13f2c9123aa51/programs/bpf_loader/src/syscalls/cpi.rs#L816 */
   VM_SYSCALL_CPI_ACC_INFO_T * acc_infos =
     FD_VM_MEM_SLICE_HADDR_ST( vm, 
                               acct_infos_va,
                               VM_SYSCALL_CPI_ACC_INFO_ALIGN,
-                              acct_info_cnt*VM_SYSCALL_CPI_ACC_INFO_SIZE );
+                              fd_ulong_sat_mul( acct_info_cnt, VM_SYSCALL_CPI_ACC_INFO_SIZE ) );
   /* Right after translating, Agave checks the number of account infos:
      https://github.com/firedancer-io/agave/blob/838c1952595809a31520ff1603a13f2c9123aa51/programs/bpf_loader/src/syscalls/cpi.rs#L822 */
   if( FD_FEATURE_ACTIVE( vm->instr_ctx->slot_ctx, loosen_cpi_size_restriction ) ) {
@@ -665,25 +684,6 @@ VM_SYSCALL_CPI_ENTRYPOINT( void *  _vm,
     }
   }
 
-  /* Create the instruction to execute (in the input format the FD runtime expects) from
-     the translated CPI ABI inputs. */
-  fd_instr_info_t * instruction_to_execute = &vm->instr_ctx->txn_ctx->instr_infos[ vm->instr_ctx->txn_ctx->instr_info_cnt ];
-
-  vm->instr_ctx->txn_ctx->instr_info_cnt++;
-  if( FD_UNLIKELY( vm->instr_ctx->txn_ctx->instr_info_cnt>FD_MAX_INSTRUCTION_TRACE_LENGTH ) ) {
-     return FD_EXECUTOR_INSTR_ERR_MAX_INSN_TRACE_LENS_EXCEEDED;;
-  }
-
-  err = VM_SYSCALL_CPI_INSTRUCTION_TO_INSTR_FUNC( vm, cpi_instruction, cpi_account_metas, program_id, data, instruction_to_execute );
-  if( FD_UNLIKELY( err ) ) return err;
-
-  /* Prepare the instruction for execution in the runtime. This is required by the runtime
-     before we can pass an instruction to the executor. */
-  fd_instruction_account_t instruction_accounts[256];
-  ulong instruction_accounts_cnt;
-  err = fd_vm_prepare_instruction( vm->instr_ctx->instr, instruction_to_execute, vm->instr_ctx, instruction_accounts, &instruction_accounts_cnt, signers, signers_seeds_cnt );
-  if( FD_UNLIKELY( err ) ) return err;
-
   /* Update the callee accounts with any changes made by the caller prior to this CPI execution */
   ulong callee_account_keys[256];
   ulong caller_accounts_to_update[256];

diff --git a/src/waltz/quic/Local.mk b/src/waltz/quic/Local.mk
@@ -16,9 +16,6 @@ $(call add-hdrs,fd_quic_conn_map.h)
 $(call add-hdrs,fd_quic_pkt_meta.h)
 $(call add-objs,fd_quic_pkt_meta,fd_quic)
 
-$(call add-hdrs,fd_quic_proto.h fd_quic_proto_structs.h fd_quic_types.h)
-$(call add-objs,fd_quic_proto,fd_quic)
-
 $(call add-hdrs,fd_quic_retry.h)
 $(call add-objs,fd_quic_retry,fd_quic)
 

diff --git a/src/waltz/quic/fd_quic.c b/src/waltz/quic/fd_quic.c
@@ -7,6 +7,7 @@
 #include "fd_quic_conn.h"
 #include "fd_quic_conn_map.h"
 #include "fd_quic_proto.h"
+#include "fd_quic_proto.c"
 #include "fd_quic_retry.h"
 
 #include "templ/fd_quic_frame_handler_decl.h"
@@ -3016,6 +3017,11 @@ typedef struct fd_quic_pkt_hdr fd_quic_pkt_hdr_t;
 /* encode packet header into buffer */
 ulong
 fd_quic_pkt_hdr_encode( uchar * cur_ptr, ulong cur_sz, fd_quic_pkt_hdr_t * pkt_hdr, uint enc_level ) {
+  /* optimize for the common case */
+  if( FD_LIKELY( enc_level == fd_quic_enc_level_appdata_id ) ) {
+    return fd_quic_encode_one_rtt( cur_ptr, cur_sz, &pkt_hdr->quic_pkt.one_rtt );
+  }
+
   switch( enc_level ) {
     case fd_quic_enc_level_initial_id:;
       return fd_quic_encode_initial( cur_ptr, cur_sz, &pkt_hdr->quic_pkt.initial );
@@ -5335,6 +5341,10 @@ fd_quic_frame_handle_stream_frame(
   /* ack-eliciting */
   pkt->ack_flag |= ACK_FLAG_RQD;
 
+  /* This eliminates a warning without appearing to affect the optimization */
+  if( !data->length_opt ) __asm__( "#NOP" : "=rm" (data->length) );
+  if( !data->offset_opt ) __asm__( "#NOP" : "=rm" (data->offset) );
+
   /* offset field is optional, implied 0 */
   ulong offset      = fd_ulong_if( data->offset_opt, data->offset, 0UL );
   ulong stream_id   = data->stream_id;

diff --git a/src/waltz/quic/fd_quic_ack_tx.c b/src/waltz/quic/fd_quic_ack_tx.c
@@ -1,6 +1,9 @@
 #include "fd_quic_ack_tx.h"
 #include "fd_quic_private.h"
 
+#include "fd_quic_proto.h"
+#include "fd_quic_proto.c"
+
 static inline int
 fd_quic_range_can_extend( fd_quic_range_t const * range,
                           ulong                   idx ) {

diff --git a/src/waltz/quic/fd_quic_pcap_main.c b/src/waltz/quic/fd_quic_pcap_main.c
@@ -7,6 +7,7 @@
 #include "fd_quic.h"
 #include "../../ballet/hex/fd_hex.h"
 #include "../../waltz/quic/fd_quic_proto.h"
+#include "../../waltz/quic/fd_quic_proto.c"
 #include "../../waltz/quic/templ/fd_quic_parse_util.h"
 #include "../../util/net/fd_pcap.h"
 #include "../../util/net/fd_pcapng.h"

diff --git a/src/waltz/quic/fd_quic_proto.c b/src/waltz/quic/fd_quic_proto.c
@@ -13,12 +13,9 @@
 #include "fd_quic_types.h"
 #include "fd_quic_common.h"
 
-#include "templ/fd_quic_parse_util.h"
+#include "fd_quic_proto.h"
 
-#include "templ/fd_quic_defs.h"
-#include "templ/fd_quic_templ.h"
-#include "templ/fd_quic_frames_templ.h"
-#include "templ/fd_quic_undefs.h"
+#include "templ/fd_quic_parse_util.h"
 
 #include "templ/fd_quic_parsers.h"
 #include "templ/fd_quic_templ.h"

diff --git a/src/waltz/quic/fd_quic_retry_private.h b/src/waltz/quic/fd_quic_retry_private.h
@@ -1,5 +1,7 @@
 #include "fd_quic_retry.h"
+
 #include "fd_quic_proto.h"
+#include "fd_quic_proto.c"
 
 /* FD_QUIC_RETRY_MAX_PSEUDO_SZ is the max encoded size of a Retry pseudo
    header. */

diff --git a/src/waltz/quic/templ/fd_quic_encoders.h b/src/waltz/quic/templ/fd_quic_encoders.h
@@ -32,9 +32,11 @@
    frame is not const, as it may be mutated, for example to store offsets
    to particular bytes in the encoded data */
 #define FD_TEMPL_DEF_STRUCT_BEGIN(NAME)                                \
-  ulong fd_quic_encode_##NAME( uchar *              buf,               \
-                               ulong                sz,                \
-                               fd_quic_##NAME##_t * frame ) {          \
+  static inline                                                        \
+  ulong                                                                \
+  fd_quic_encode_##NAME( uchar *              buf,                     \
+                         ulong                sz,                      \
+                         fd_quic_##NAME##_t * frame ) {                \
     (void)frame;                                                       \
     uchar *  orig_buf = buf;                                           \
     uchar *  buf_end  = buf + sz;                                      \

diff --git a/src/waltz/quic/templ/fd_quic_encoders_decl.h b/src/waltz/quic/templ/fd_quic_encoders_decl.h
@@ -1,10 +1,14 @@
 /* QUIC encoders + footprints */
 
 #define FD_TEMPL_DEF_STRUCT_BEGIN(NAME)                       \
-  ulong fd_quic_encode_##NAME( uchar *              buf,      \
-                               ulong                sz ,      \
-                               fd_quic_##NAME##_t * frame );  \
-  ulong fd_quic_encode_footprint_##NAME( fd_quic_##NAME##_t * frame );
+  static inline                                               \
+  ulong                                                       \
+  fd_quic_encode_##NAME( uchar *              buf,            \
+                         ulong                sz ,            \
+                         fd_quic_##NAME##_t * frame );        \
+  static inline                                               \
+  ulong                                                       \
+  fd_quic_encode_footprint_##NAME( fd_quic_##NAME##_t * frame );
 
 #include "fd_quic_dft.h"
 
diff --git a/src/waltz/quic/templ/fd_quic_parse_util.h b/src/waltz/quic/templ/fd_quic_parse_util.h
@@ -1,3 +1,6 @@
+#ifndef FD_QUIC_WALTZ_QUIC_TEMPL_FD_QUIC_PARSE_UTIL_H
+#define FD_QUIC_WALTZ_QUIC_TEMPL_FD_QUIC_PARSE_UTIL_H
+
 #include <stddef.h>
 
 #include "../fd_quic_common.h"
@@ -128,3 +131,5 @@ fd_quic_pktnum_decode( uchar const * buf,
   }
   return FD_LOAD( uint, scratch );
 }
+
+#endif
diff --git a/src/waltz/quic/templ/fd_quic_parsers.h b/src/waltz/quic/templ/fd_quic_parsers.h
@@ -66,6 +66,7 @@
 // remaining bits are all data bits
 #define FD_TEMPL_MBR_ELEM_VARINT(NAME,TYPE)                            \
   do {                                                                 \
+    out->NAME = 0;                                                     \
     if( FD_UNLIKELY( cur_byte >= sz ) ) return FD_QUIC_PARSE_FAIL;     \
     uint msb2 = buf[cur_byte] >> 6u;                                   \
     uint vsz  = 1U<<msb2;                                              \
@@ -132,9 +133,6 @@
       cur_byte += FD_TEMPL_PARSE(TYPE,out->NAME[j],buf+cur_byte);      \
     }
 
-#if 0
-#define FD_TEMPL_MBR_OPT(STRUCT,NAME,MASK,TEMPL)
-#else
 #define FD_TEMPL_MBR_OPT(TYPE_NAME,NAME,MASK,...)   \
     do {                                            \
       _Bool cond = out->TYPE_NAME & (MASK);         \
@@ -143,7 +141,6 @@
         __VA_ARGS__                                 \
       }                                             \
     } while(0);
-#endif
 
 
 // at end, return the number of bytes consumed

diff --git a/src/waltz/quic/templ/fd_quic_parsers_decl.h b/src/waltz/quic/templ/fd_quic_parsers_decl.h
@@ -5,7 +5,9 @@
    forces caller to check the result */
 #define FD_TEMPL_DEF_STRUCT_BEGIN(NAME) \
   FD_WARN_UNUSED \
-  ulong fd_quic_decode_##NAME( fd_quic_##NAME##_t * FD_RESTRICT out, uchar const * FD_RESTRICT buf, ulong sz );
+  static inline \
+  ulong \
+  fd_quic_decode_##NAME( fd_quic_##NAME##_t * FD_RESTRICT out, uchar const * FD_RESTRICT buf, ulong sz );
 
 #include "fd_quic_dft.h"
 
diff --git a/src/waltz/quic/templ/fd_quic_templ_dump.h b/src/waltz/quic/templ/fd_quic_templ_dump.h
@@ -1,5 +1,7 @@
-#define FD_TEMPL_DEF_STRUCT_BEGIN(NAME)                                \
-  void fd_quic_dump_struct_##NAME( fd_quic_##NAME##_t const * data ) { \
+#define FD_TEMPL_DEF_STRUCT_BEGIN(NAME)                           \
+  static inline                                                   \
+  void                                                            \
+  fd_quic_dump_struct_##NAME( fd_quic_##NAME##_t const * data ) { \
     (void)data;
 
 #define FD_QUIC_FMT_uchar  "u"

diff --git a/src/waltz/quic/templ/fd_quic_templ_dump_decl.h b/src/waltz/quic/templ/fd_quic_templ_dump_decl.h
@@ -1,5 +1,7 @@
-#define FD_TEMPL_DEF_STRUCT_BEGIN(NAME) \
-  void fd_quic_dump_struct_##NAME( fd_quic_##NAME##_t const * data );
+#define FD_TEMPL_DEF_STRUCT_BEGIN(NAME)                          \
+  static inline                                                  \
+  void                                                           \
+  fd_quic_dump_struct_##NAME( fd_quic_##NAME##_t const * data );
 
 #include "fd_quic_dft.h"
 
diff --git a/src/waltz/quic/templ/test_quic_transport_params.c b/src/waltz/quic/templ/test_quic_transport_params.c
@@ -2,6 +2,7 @@
 #include "../fd_quic_conn_id.h"
 #include "../fd_quic_enum.h"
 #include "../fd_quic_proto.h"
+#include "../fd_quic_proto.c"
 #include "fd_quic_transport_params.h"
 
 static int

diff --git a/src/waltz/quic/tests/fuzz_quic_wire.c b/src/waltz/quic/tests/fuzz_quic_wire.c
@@ -20,6 +20,7 @@
 #include "../../../util/net/fd_ip4.h"
 #include "../../../util/net/fd_udp.h"
 #include "../fd_quic_proto.h"
+#include "../fd_quic_proto.c"
 #include "../fd_quic_private.h"
 
 #include <assert.h>

diff --git a/src/waltz/quic/tests/test_quic_ack_tx.c b/src/waltz/quic/tests/test_quic_ack_tx.c
@@ -1,5 +1,6 @@
 #include "../fd_quic_ack_tx.h"
 #include "../fd_quic_proto.h"
+#include "../fd_quic_proto.c"
 
 int
 main( int     argc,

diff --git a/src/waltz/quic/tests/test_quic_concurrency.c b/src/waltz/quic/tests/test_quic_concurrency.c
@@ -8,6 +8,7 @@
 
 #include "fd_quic_sandbox.h"
 #include "../fd_quic_proto.h"
+#include "../fd_quic_proto.c"
 #include "../fd_quic_private.h"
 #include "../../../tango/fd_tango.h"
 

diff --git a/src/waltz/quic/tests/test_quic_conformance.c b/src/waltz/quic/tests/test_quic_conformance.c
@@ -3,6 +3,7 @@
 
 #include "fd_quic_sandbox.h"
 #include "../fd_quic_proto.h"
+#include "../fd_quic_proto.c"
 #include "../fd_quic_private.h"
 
 /* RFC 9000 Section 4.1. Data Flow Control

diff --git a/src/waltz/quic/tests/test_quic_layout.c b/src/waltz/quic/tests/test_quic_layout.c
@@ -1,5 +1,6 @@
 #include <stdio.h>
 #include "../fd_quic_proto.h"
+#include "../fd_quic_proto.c"
 #include "../templ/fd_quic_parse_util.h"
 
 int

diff --git a/src/waltz/quic/tests/test_quic_proto.c b/src/waltz/quic/tests/test_quic_proto.c
@@ -1,5 +1,6 @@
 #include "../fd_quic_common.h"
 #include "../fd_quic_proto.h"
+#include "../fd_quic_proto.c"
 #include "../templ/fd_quic_parse_util.h"
 
 /* Test the varint parser generator */