diff --git a/protocol/client/CMakeLists.txt b/protocol/client/CMakeLists.txt index 7d461de477a..bee5454e104 100644 --- a/protocol/client/CMakeLists.txt +++ b/protocol/client/CMakeLists.txt @@ -15,6 +15,7 @@ add_executable(FractalClient peercursor.c ringbuffer.c sync_packets.c + bitrate.c ) # On Windows during client app packaging, FractalClient should have icon set diff --git a/protocol/client/bitrate.c b/protocol/client/bitrate.c new file mode 100644 index 00000000000..04fd0f014d0 --- /dev/null +++ b/protocol/client/bitrate.c @@ -0,0 +1,93 @@ +/** + * Copyright Fractal Computers, Inc. 2021 + * @file bitrate.h + * @brief This file contains the client's adaptive bitrate code. Any algorithms we are using for +predicting bitrate should be stored here. +============================ +Usage +============================ +Place to put any predictive/adaptive bitrate algorithms. In the current setup, each algorithm is a +function that takes in inputs through a BitrateStatistics struct. The function is responsible for +maintaining any internal state necessary for the algorithm (possibly through the use of custom types +or helpers), and should update max_bitrate when necessary. To change the algorithm the client uses, +set calculate_new_bitrate to the desired algorithm. +*/ + +/* +============================ +Includes +============================ +*/ +#include "bitrate.h" + +volatile int max_bitrate = STARTING_BITRATE; +volatile int max_burst_bitrate = STARTING_BURST_BITRATE; +#define BAD_BITRATE 10400000 +#define BAD_BURST_BITRATE 31800000 + +/* +============================ +Private Functions +============================ +*/ +Bitrates fallback_bitrate(BitrateStatistics stats); +Bitrates ewma_bitrate(BitrateStatistics stats); + +/* +============================ +Private Function Implementations +============================ +*/ +Bitrates fallback_bitrate(BitrateStatistics stats) { + /* + Switches between two sets of bitrate/burst bitrate: the default of 16mbps/100mbps and a + fallback of 10mbps/30mbps. We fall back if we've nacked a lot in the last second. + + Arguments: + stats (BitrateStatistics): struct containing the average number of nacks per second + since the last time this function was called + */ + static Bitrates bitrates; + if (stats.num_nacks_per_second > 6 && max_bitrate != BAD_BITRATE) { + bitrates.bitrate = BAD_BITRATE; + bitrates.burst_bitrate = BAD_BURST_BITRATE; + } else { + bitrates.bitrate = max_bitrate; + bitrates.burst_bitrate = max_burst_bitrate; + } + return bitrates; +} + +Bitrates ewma_bitrate(BitrateStatistics stats) { + /* + Keeps an exponentially weighted moving average of the throughput per second the client is + getting, and uses that to predict a good bitrate to ask the server for. + + Arguments: + stats (BitrateStatistics): struct containing the throughput per second of the client + */ + static const double alpha = 0.8; + // because the max bitrate of the encoder is usually larger than the actual amount of data we + // get from the server + static const double bitrate_throughput_ratio = 1.25; + static int throughput = -1; + static Bitrates bitrates; + if (throughput == -1) { + throughput = (int)(STARTING_BITRATE / bitrate_throughput_ratio); + } + // sanity check to make sure we're not sending it negative bitrate + if (stats.throughput_per_second >= 0) { + throughput = (int)(alpha * throughput + (1 - alpha) * stats.throughput_per_second); + bitrates.bitrate = (int)(bitrate_throughput_ratio * throughput); + } + bitrates.burst_bitrate = max_burst_bitrate; + return bitrates; +} + +/* +============================ +Public Function Implementations +============================ +*/ + +BitrateCalculator calculate_new_bitrate = fallback_bitrate; diff --git a/protocol/client/bitrate.h b/protocol/client/bitrate.h new file mode 100644 index 00000000000..24a222832db --- /dev/null +++ b/protocol/client/bitrate.h @@ -0,0 +1,54 @@ +#ifndef CLIENT_BITRATE_H +#define CLIENT_BITRATE_H +/** + * Copyright Fractal Computers, Inc. 2021 + * @file bitrate.h + * @brief This file contains the client's adaptive bitrate code. Any algorithms we are using for +predicting bitrate should be stored here. +============================ +Usage +============================ +The client should periodically call calculate_new_bitrate with any data rrequested, such as +throughput or nack data. This will update max_bitrate and max_burst_bitrate if needed. +*/ + +/* +============================ +Includes +============================ +*/ + +#include + +/* +============================ +Custom Types +============================ +*/ + +typedef struct BitrateStatistics { + int num_nacks_per_second; + int throughput_per_second; +} BitrateStatistics; + +typedef struct Bitrates { + int bitrate; + int burst_bitrate; +} Bitrates; + +typedef Bitrates (*BitrateCalculator)(BitrateStatistics); + +/* +============================ +Public Functions +============================ +*/ + +/** + * @brief Update max_bitrate and max_burst_bitrate with the latest client data. This function + * does not trigger any client bitrate updates. + * + * @param stats A struct containing any information we might need to update bitrate. + */ +extern BitrateCalculator calculate_new_bitrate; +#endif diff --git a/protocol/client/main.c b/protocol/client/main.c index 0f57750f0a0..93bb56973c7 100644 --- a/protocol/client/main.c +++ b/protocol/client/main.c @@ -61,8 +61,8 @@ volatile int server_height = -1; volatile CodecType server_codec_type = CODEC_TYPE_UNKNOWN; // maximum mbps -volatile int max_bitrate = STARTING_BITRATE; -volatile int max_burst_bitrate = STARTING_BURST_BITRATE; +extern volatile int max_bitrate; +extern volatile int max_burst_bitrate; volatile bool update_bitrate = false; // Global state variables diff --git a/protocol/client/video.c b/protocol/client/video.c index d6c28b7f671..d1202773a64 100644 --- a/protocol/client/video.c +++ b/protocol/client/video.c @@ -32,6 +32,7 @@ Includes #include "sdlscreeninfo.h" #include "native_window_utils.h" #include "network.h" +#include "bitrate.h" #define USE_HARDWARE true #define NO_NACKS_DURING_IFRAME false @@ -698,21 +699,25 @@ void calculate_statistics() { static clock t; static bool init_t = false; + static BitrateStatistics stats; + static Bitrates new_bitrates; if (!init_t) { start_timer(&t); init_t = true; } + // do some calculation // Update mbps every 5 seconds if (get_timer(t) > 5.0) { - // Note: Uncomment to oscillate between bitrates every 5 seconds - if (max_bitrate > STARTING_BITRATE) { - max_bitrate = STARTING_BITRATE; - max_burst_bitrate = STARTING_BURST_BITRATE; - } else { - // max_bitrate += 1000000; - // max_burst_bitrate += 1000000; + stats.num_nacks_per_second = video_ring_buffer->num_nacked / 5; + stats.throughput_per_second = -1; + new_bitrates = calculate_new_bitrate(stats); + if (new_bitrates.bitrate != max_bitrate || + new_bitrates.burst_bitrate != max_burst_bitrate) { + max_bitrate = new_bitrates.bitrate; + max_burst_bitrate = new_bitrates.burst_bitrate; + update_bitrate = true; } - // update_bitrate = true; + video_ring_buffer->num_nacked = 0; start_timer(&t); } } diff --git a/protocol/fractal/core/fractal.h b/protocol/fractal/core/fractal.h index ff7662816ef..069a39582f2 100644 --- a/protocol/fractal/core/fractal.h +++ b/protocol/fractal/core/fractal.h @@ -118,8 +118,8 @@ Defines #define MINIMUM_BITRATE 2000000 #define ACK_REFRESH_MS 50 -#define STARTING_BITRATE 10400000 -#define STARTING_BURST_BITRATE 31800000 +#define STARTING_BITRATE 15400000 +#define STARTING_BURST_BITRATE 100000000 // 16:10 aspect ratio for minimum screen, which is the Mac aspect ratio #define MIN_SCREEN_WIDTH 480 @@ -128,7 +128,8 @@ Defines #define MAX_SCREEN_HEIGHT 4096 #define AUDIO_BITRATE 128000 -#define FPS 45 +// We would like to set this higher, but mac displays have a max refresh rate of 60 Hz. +#define FPS 60 #define MIN_FPS 10 #define OUTPUT_WIDTH 1280 #define OUTPUT_HEIGHT 720 diff --git a/protocol/server/handle_client_message.c b/protocol/server/handle_client_message.c index e646c21010c..4213300a0e8 100644 --- a/protocol/server/handle_client_message.c +++ b/protocol/server/handle_client_message.c @@ -307,7 +307,7 @@ static int handle_ping_message(FractalClientMessage *fmsg, int client_id, bool i fmsg_response.ping_id = fmsg->ping_id; int ret = 0; if (send_udp_packet(&(clients[client_id].UDP_context), PACKET_MESSAGE, - (uint8_t *)&fmsg_response, sizeof(fmsg_response), 1, STARTING_BURST_BITRATE, + (uint8_t *)&fmsg_response, sizeof(fmsg_response), 1, max_burst_bitrate, NULL, NULL) < 0) { LOG_WARNING("Could not send Ping to Client ID: %d", client_id); ret = -1; diff --git a/protocol/server/video.c b/protocol/server/video.c index 468c68220a6..64f93346805 100644 --- a/protocol/server/video.c +++ b/protocol/server/video.c @@ -492,7 +492,7 @@ int32_t multithreaded_send_video(void* opaque) { // LOG_INFO("Size: %d, MBPS: %f, VS MAX MBPS: %f, Time: // %f, Transmit Time: %f, Delay: %f", - // previous_frame_size, mbps, max_mbps, frame_time, + // previous_frame_size, mbps, max_bitrate, frame_time, // transmit_time, delay); if ((current_fps < worst_fps || ideal_bitrate > current_bitrate) &&