From f8191f77605bb0358bd1a2c50f110a8707010fae Mon Sep 17 00:00:00 2001 From: Tom Pointon Date: Tue, 7 Jan 2025 19:02:24 +0000 Subject: [PATCH] flamenco, repair: cache good repair peers to file --- src/app/fdctl/config.h | 1 + src/app/fdctl/config_parse.c | 1 + src/app/fdctl/run/tiles/fd_repair.c | 12 +- .../run/tiles/generated/repair_seccomp.h | 32 ++- src/app/fdctl/run/tiles/repair.seccomppolicy | 12 +- src/app/fdctl/run/topos/fd_firedancer.c | 2 + src/disco/topo/fd_topo.h | 2 + src/flamenco/repair/fd_repair.c | 194 +++++++++++++++++- src/flamenco/repair/fd_repair.h | 1 + 9 files changed, 239 insertions(+), 18 deletions(-) diff --git a/src/app/fdctl/config.h b/src/app/fdctl/config.h index a73fccb5ea..a7e659a310 100644 --- a/src/app/fdctl/config.h +++ b/src/app/fdctl/config.h @@ -278,6 +278,7 @@ typedef struct { struct { ushort repair_intake_listen_port; ushort repair_serve_listen_port; + char good_peer_cache_file[ PATH_MAX ]; } repair; struct { diff --git a/src/app/fdctl/config_parse.c b/src/app/fdctl/config_parse.c index b283ad1ab8..330fa0039e 100644 --- a/src/app/fdctl/config_parse.c +++ b/src/app/fdctl/config_parse.c @@ -367,6 +367,7 @@ fdctl_pod_to_cfg( config_t * config, CFG_POP ( ushort, tiles.repair.repair_intake_listen_port ); CFG_POP ( ushort, tiles.repair.repair_serve_listen_port ); + CFG_POP ( cstr, tiles.repair.good_peer_cache_file ); CFG_POP ( cstr, tiles.replay.capture ); CFG_POP ( cstr, tiles.replay.funk_checkpt ); diff --git a/src/app/fdctl/run/tiles/fd_repair.c b/src/app/fdctl/run/tiles/fd_repair.c index b2b69ff924..3e85e71e0f 100644 --- a/src/app/fdctl/run/tiles/fd_repair.c +++ b/src/app/fdctl/run/tiles/fd_repair.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "../../../../util/net/fd_net_headers.h" @@ -485,6 +486,12 @@ privileged_init( fd_topo_t * topo, ctx->repair_config.private_key = ctx->identity_private_key; ctx->repair_config.public_key = &ctx->identity_public_key; + tile->repair.good_peer_cache_file_fd = open( tile->repair.good_peer_cache_file, O_RDWR | O_CREAT, 0644 ); + if( FD_UNLIKELY(tile->repair.good_peer_cache_file_fd==-1 ) ) { + FD_LOG_WARNING(( "Failed to open the good peer cache file (%i-%s)", errno, fd_io_strerror( errno ) )); + } + ctx->repair_config.good_peer_cache_file_fd = tile->repair.good_peer_cache_file_fd; + FD_TEST( sizeof(ulong) == getrandom( &ctx->repair_seed, sizeof(ulong), 0 ) ); } @@ -655,7 +662,8 @@ populate_allowed_seccomp( fd_topo_t const * topo, (void)topo; (void)tile; - populate_sock_filter_policy_repair( out_cnt, out, (uint)fd_log_private_logfile_fd() ); + populate_sock_filter_policy_repair( + out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)tile->repair.good_peer_cache_file_fd ); return sock_filter_policy_repair_instr_cnt; } @@ -673,6 +681,8 @@ populate_allowed_fds( fd_topo_t const * topo, out_fds[ out_cnt++ ] = 2; /* stderr */ if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) ) out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */ + if( FD_LIKELY( -1!=tile->repair.good_peer_cache_file_fd ) ) + out_fds[ out_cnt++ ] = tile->repair.good_peer_cache_file_fd; /* good peer cache file */ return out_cnt; } diff --git a/src/app/fdctl/run/tiles/generated/repair_seccomp.h b/src/app/fdctl/run/tiles/generated/repair_seccomp.h index 579068167e..2a0b8e674e 100644 --- a/src/app/fdctl/run/tiles/generated/repair_seccomp.h +++ b/src/app/fdctl/run/tiles/generated/repair_seccomp.h @@ -21,34 +21,44 @@ #else # error "Target architecture is unsupported by seccomp." #endif -static const unsigned int sock_filter_policy_repair_instr_cnt = 14; +static const unsigned int sock_filter_policy_repair_instr_cnt = 19; -static void populate_sock_filter_policy_repair( ulong out_cnt, struct sock_filter * out, unsigned int logfile_fd) { - FD_TEST( out_cnt >= 14 ); - struct sock_filter filter[14] = { +static void populate_sock_filter_policy_repair( ulong out_cnt, struct sock_filter * out, unsigned int logfile_fd, unsigned int good_peer_cache_file_fd) { + FD_TEST( out_cnt >= 19 ); + struct sock_filter filter[19] = { /* Check: Jump to RET_KILL_PROCESS if the script's arch != the runtime arch */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) ) ), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 10 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 15 ), /* loading syscall number in accumulator */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, nr ) ) ), /* allow write based on expression */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_write, /* check_write */ 2, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_write, /* check_write */ 3, 0 ), /* allow fsync based on expression */ - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fsync, /* check_fsync */ 5, 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fsync, /* check_fsync */ 8, 0 ), + /* allow read based on expression */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_read, /* check_read */ 9, 0 ), /* none of the syscalls matched */ - { BPF_JMP | BPF_JA, 0, 0, /* RET_KILL_PROCESS */ 6 }, + { BPF_JMP | BPF_JA, 0, 0, /* RET_KILL_PROCESS */ 10 }, // check_write: /* load syscall argument 0 in accumulator */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 2, /* RET_ALLOW */ 5, /* lbl_1 */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 2, /* RET_ALLOW */ 9, /* lbl_1 */ 0 ), // lbl_1: /* load syscall argument 0 in accumulator */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 3, /* RET_KILL_PROCESS */ 2 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 7, /* lbl_2 */ 0 ), +// lbl_2: + /* load syscall argument 0 in accumulator */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, good_peer_cache_file_fd, /* RET_ALLOW */ 5, /* RET_KILL_PROCESS */ 4 ), // check_fsync: /* load syscall argument 0 in accumulator */ BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), - BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 1, /* RET_KILL_PROCESS */ 0 ), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, logfile_fd, /* RET_ALLOW */ 3, /* RET_KILL_PROCESS */ 2 ), +// check_read: + /* load syscall argument 0 in accumulator */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, good_peer_cache_file_fd, /* RET_ALLOW */ 1, /* RET_KILL_PROCESS */ 0 ), // RET_KILL_PROCESS: /* KILL_PROCESS is placed before ALLOW since it's the fallthrough case. */ BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), diff --git a/src/app/fdctl/run/tiles/repair.seccomppolicy b/src/app/fdctl/run/tiles/repair.seccomppolicy index efb7dec4f4..8572a94ed5 100644 --- a/src/app/fdctl/run/tiles/repair.seccomppolicy +++ b/src/app/fdctl/run/tiles/repair.seccomppolicy @@ -1,18 +1,26 @@ # logfile_fd: It can be disabled by configuration, but typically tiles # will open a log file on boot and write all messages there. -unsigned int logfile_fd +# good_peer_cache_file_fd: The list of known good repair peers, which we read +# at boot and then periodically write to. +unsigned int logfile_fd, unsigned int good_peer_cache_file_fd # logging: all log messages are written to a file and/or pipe # # 'WARNING' and above are written to the STDERR pipe, while all messages # are always written to the log file. # +# good_peer_cache_file: we periodically write out the good peers to the file +# # arg 0 is the file descriptor to write to. The boot process ensures # that descriptor 2 is always STDERR. write: (or (eq (arg 0) 2) - (eq (arg 0) logfile_fd)) + (eq (arg 0) logfile_fd) + (eq (arg 0) good_peer_cache_file_fd)) # logging: 'WARNING' and above fsync the logfile to disk immediately # # arg 0 is the file descriptor to fsync. fsync: (eq (arg 0) logfile_fd) + +# good_peer_cache_file: we read in the good peer cache file on boot +read: (eq (arg 0) good_peer_cache_file_fd) diff --git a/src/app/fdctl/run/topos/fd_firedancer.c b/src/app/fdctl/run/topos/fd_firedancer.c index 06280ab737..dfb7774bdb 100644 --- a/src/app/fdctl/run/topos/fd_firedancer.c +++ b/src/app/fdctl/run/topos/fd_firedancer.c @@ -591,6 +591,8 @@ fd_topo_initialize( config_t * config ) { tile->repair.repair_intake_listen_port = config->tiles.repair.repair_intake_listen_port; tile->repair.repair_serve_listen_port = config->tiles.repair.repair_serve_listen_port; tile->repair.ip_addr = config->tiles.net.ip_addr; + strncpy( tile->repair.good_peer_cache_file, + config->tiles.repair.good_peer_cache_file, sizeof(config->tiles.repair.good_peer_cache_file) ); memcpy( tile->repair.src_mac_addr, config->tiles.net.mac_addr, 6 ); strncpy( tile->repair.identity_key_path, config->consensus.identity_path, sizeof(tile->repair.identity_key_path) ); diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index dd44c07404..f886ff5cda 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -296,11 +296,13 @@ typedef struct { struct { ushort repair_intake_listen_port; ushort repair_serve_listen_port; + char good_peer_cache_file[ PATH_MAX ]; /* non-config */ uint ip_addr; uchar src_mac_addr[ 6 ]; + int good_peer_cache_file_fd; char identity_key_path[ PATH_MAX ]; } repair; diff --git a/src/flamenco/repair/fd_repair.c b/src/flamenco/repair/fd_repair.c index 185b9e949b..99bcc4c2dd 100644 --- a/src/flamenco/repair/fd_repair.c +++ b/src/flamenco/repair/fd_repair.c @@ -8,8 +8,11 @@ #include "../../util/rng/fd_rng.h" #include #include +#include +#include #include #include +#include #include #pragma GCC diagnostic ignored "-Wstrict-aliasing" @@ -225,6 +228,8 @@ struct fd_repair { long last_decay; /* Last statistics printout */ long last_print; + /* Last write to good peer cache file */ + long last_good_peer_cache_file_write; /* Random number generator */ fd_rng_t rng[1]; /* RNG seed */ @@ -232,6 +237,8 @@ struct fd_repair { /* Stake weights */ ulong stake_weights_cnt; fd_stake_weight_t * stake_weights; + /* Path to the file where we write the cache of known good repair peers, to make cold booting faster */ + int good_peer_cache_file_fd; }; ulong @@ -268,6 +275,7 @@ fd_repair_new ( void * shmem, ulong seed ) { glob->last_sends = 0; glob->last_decay = 0; glob->last_print = 0; + glob->last_good_peer_cache_file_write = 0; glob->oldest_nonce = glob->current_nonce = glob->next_nonce = 0; fd_rng_new(glob->rng, (uint)seed, 0UL); @@ -345,6 +353,7 @@ fd_repair_set_config( fd_repair_t * glob, const fd_repair_config_t * config ) { glob->sign_fun = config->sign_fun; glob->sign_arg = config->sign_arg; glob->deliver_fail_fun = config->deliver_fail_fun; + glob->good_peer_cache_file_fd = config->good_peer_cache_file_fd; return 0; } @@ -559,17 +568,133 @@ fd_repair_decay_stats( fd_repair_t * glob ) { } } +/** + * read_line() reads characters one by one from 'fd' until: + * - it sees a newline ('\n') + * - it reaches 'max_len - 1' characters + * - or EOF (read returns 0) + * It stores the line in 'buf' and null-terminates it. + * + * Returns the number of characters read (not counting the null terminator), + * or -1 on error. + */ +long read_line(int fd, char *buf) { + long i = 0; + + while (i < 255) { + char c; + long n = read(fd, &c, 1); + + if (n < 0) { + if (errno == EINTR) continue; + return -1; + } else if (n == 0) { + break; + } + + buf[i++] = c; + + if (c == '\n') { + break; + } + } + + buf[i] = '\0'; + return i; +} + +static int +fd_read_in_good_peer_cache_file( fd_repair_t * repair ) { + if ( repair->good_peer_cache_file_fd==-1 ) { + FD_LOG_NOTICE(( "No repair good_peer_cache_file specified, not loading cached peers" )); + return 0; + } + + long seek = lseek( repair->good_peer_cache_file_fd, 0UL, SEEK_SET ); + if( FD_UNLIKELY( seek!=0L ) ) { + FD_LOG_WARNING(( "Failed to seek to the beginning of the good peer cache file" )); + return 1; + } + + int loaded_peers = 0; + char line[256]; + char *saveptr = NULL; + + long len; + while ((len = read_line(repair->good_peer_cache_file_fd, line)) > 0) { + + /* Strip newline if present */ + size_t len = strlen( line ); + if( len>0 && line[len-1]=='\n' ) { + line[len-1] = '\0'; + len--; + } + + /* Skip empty or comment lines */ + if( !len || line[0]=='#' ) continue; + + /* Parse: base58EncodedPubkey/ipAddr/port */ + char * base58_str = strtok_r( line, "/", &saveptr ); + char * ip_str = strtok_r( NULL, "/", &saveptr ); + char * port_str = strtok_r( NULL, "/", &saveptr ); + + if( FD_UNLIKELY( !base58_str || !ip_str || !port_str ) ) { + FD_LOG_WARNING(( "Malformed line, skipping" )); + continue; + } + + /* Decode the base58 public key */ + fd_pubkey_t pubkey; + if( !fd_base58_decode_32( base58_str, pubkey.uc ) ) { + FD_LOG_WARNING(( "Failed to decode base58 public key '%s', skipping", base58_str )); + continue; + } + + /* Convert IP address */ + struct in_addr addr_parsed; + if( inet_aton( ip_str, &addr_parsed )==0 ) { + FD_LOG_WARNING(( "Invalid IPv4 address '%s', skipping", ip_str )); + continue; + } + uint ip_addr = (uint)addr_parsed.s_addr; + + /* Convert the port */ + char * endptr = NULL; + long port = strtol( port_str, &endptr, 10 ); + if( (port<=0L) || (port>65535L) || (endptr && *endptr!='\0') ) { + FD_LOG_WARNING(( "Invalid port '%s', skipping", port_str )); + continue; + } + + /* Create the peer address struct (byte-swap the port to network order). */ + fd_repair_peer_addr_t peer_addr; + /* already in network byte order from inet_aton */ + peer_addr.addr = ip_addr; + /* Flip to big-endian for network order */ + peer_addr.port = fd_ushort_bswap( (ushort)port ); + + /* Add to active peers in the repair tile. */ + fd_repair_add_active_peer( repair, &peer_addr, &pubkey ); + + loaded_peers++; + } + + FD_LOG_INFO(( "Loaded %d peers from good peer cache file", loaded_peers )); + return 0; +} + /* Start timed events and other protocol behavior */ int fd_repair_start( fd_repair_t * glob ) { glob->last_sends = glob->now; glob->last_decay = glob->now; glob->last_print = glob->now; - return 0; + return fd_read_in_good_peer_cache_file( glob ); } static void fd_repair_print_all_stats( fd_repair_t * glob ); static void fd_actives_shuffle( fd_repair_t * repair ); +static int fd_write_good_peer_cache_file( fd_repair_t * repair ); /* Dispatch timed events and other protocol behavior. This should be * called inside the main spin loop. */ @@ -590,6 +715,9 @@ fd_repair_continue( fd_repair_t * glob ) { fd_actives_shuffle( glob ); fd_repair_decay_stats( glob ); glob->last_decay = glob->now; + } else if ( glob->now - glob->last_good_peer_cache_file_write > (long)60e9 ) { /* 1 minute */ + fd_write_good_peer_cache_file( glob ); + glob->last_good_peer_cache_file_write = glob->now; } fd_repair_unlock( glob ); return 0; @@ -850,9 +978,10 @@ fd_actives_shuffle( fd_repair_t * repair ) { static fd_active_elem_t * actives_sample( fd_repair_t * repair ) { ulong seed = repair->actives_random_seed; - while( repair->actives_sticky_cnt ) { + ulong actives_sticky_cnt = repair->actives_sticky_cnt; + while( actives_sticky_cnt ) { seed += 774583887101UL; - fd_pubkey_t * id = &repair->actives_sticky[seed % repair->actives_sticky_cnt]; + fd_pubkey_t * id = &repair->actives_sticky[seed % actives_sticky_cnt]; fd_active_elem_t * peer = fd_active_table_query( repair->actives, id, NULL ); if( NULL != peer ) { if( peer->first_request_time == 0U ) peer->first_request_time = repair->now; @@ -865,7 +994,7 @@ actives_sample( fd_repair_t * repair ) { } peer->sticky = 0; } - *id = repair->actives_sticky[--( repair->actives_sticky_cnt )]; + *id = repair->actives_sticky[--( actives_sticky_cnt )]; } return NULL; } @@ -920,6 +1049,63 @@ fd_repair_create_needed_request( fd_repair_t * glob, int type, ulong slot, uint return 0; } +static int +fd_write_good_peer_cache_file( fd_repair_t * repair ) { + // return 0; + + if ( repair->good_peer_cache_file_fd == -1 ) { + return 0; + } + + if ( repair->actives_sticky_cnt == 0 ) { + return 0; + } + + /* Truncate the file before we write it */ + int err = ftruncate( repair->good_peer_cache_file_fd, 0UL ); + if( FD_UNLIKELY( err==-1 ) ) { + FD_LOG_WARNING(( "Failed to truncate the good peer cache file (%i-%s)", errno, fd_io_strerror( errno ) )); + return 1; + } + long seek = lseek( repair->good_peer_cache_file_fd, 0UL, SEEK_SET ); + if( FD_UNLIKELY( seek!=0L ) ) { + FD_LOG_WARNING(( "Failed to seek to the beginning of the good peer cache file" )); + return 1; + } + + /* Write the active sticky peers to file in the format: + "base58EncodedPubkey/ipAddr/port" + + Where ipAddr is in dotted-decimal (e.g. "1.2.3.4") + and port is decimal, in host order (e.g. "8001"). + */ + for( ulong i = 0UL; i < repair->actives_sticky_cnt; i++ ) { + fd_pubkey_t * id = &repair->actives_sticky[ i ]; + fd_active_elem_t * peer = fd_active_table_query( repair->actives, id, NULL ); + if ( peer == NULL ) { + continue; + } + + /* Convert the public key to base58 */ + char base58_str[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( peer->key.uc, NULL, base58_str ); + + /* Convert the IP address to dotted-decimal string. The address + in peer->addr.addr is already in network byte order. */ + struct in_addr addr_parsed; + addr_parsed.s_addr = peer->addr.addr; /* net-order -> struct in_addr */ + char * ip_str = inet_ntoa( addr_parsed ); + + /* Convert port from network byte order to host byte order. */ + ushort port = fd_ushort_bswap( peer->addr.port ); + + /* Write out line: base58EncodedPubkey/ipAddr/port */ + dprintf( repair->good_peer_cache_file_fd, "%s/%s/%u\n", base58_str, ip_str, (uint)port ); + } + + return 0; +} + int fd_repair_need_window_index( fd_repair_t * glob, ulong slot, uint shred_index ) { // FD_LOG_INFO( ( "[repair] need window %lu, shred_index %lu", slot, shred_index ) ); diff --git a/src/flamenco/repair/fd_repair.h b/src/flamenco/repair/fd_repair.h index ffe046cefa..c23bd43dd4 100644 --- a/src/flamenco/repair/fd_repair.h +++ b/src/flamenco/repair/fd_repair.h @@ -59,6 +59,7 @@ struct fd_repair_config { void * fun_arg; fd_repair_sign_fun sign_fun; void * sign_arg; + int good_peer_cache_file_fd; }; typedef struct fd_repair_config fd_repair_config_t;