Adaptive bitrate infrastructure (#3544)

* Push burst bitrate to 100mbps, and stop clamping fps to 45.

* Fix FPS logging and increase maximum instantaneous framerate to 90

* Change burst bitrate to 100mbps

* fallback to old settings if nacking a lot

* change to good starting bitrate

* change to good starting bitrate

* change more thing sto good bitrate

* more renaming of bitrate

* everything in bits not megabits

* good bitrate

* bitrate.c file with some bitrate code

* rename bitrate function

* remove extraneous typo

* video ring buffer

* bad rebase bitrate

* clang-formta

* divide by int

* move the bad bitrate to bitrate.c

* documentation

* actually, don't switch back to 16/100

* clang-format?

* more infra

* changed return value

* typos

* sorry variables now extern

* initialize throughput to -1, I guess

* remove burst bitrate refs

* remove unused variable

Co-authored-by: Nicholas Pipitone <npip99@gmail.com>
Co-authored-by: Yoel Hawa <yoelhawa@gmail.com>

protocol/client/CMakeLists.txt

@@ -15,6 +15,7 @@ add_executable(FractalClient
         peercursor.c
         ringbuffer.c
         sync_packets.c
+        bitrate.c
         )
 
 # On Windows during client app packaging, FractalClient should have icon set

protocol/client/bitrate.c

@@ -0,0 +1,93 @@
+/**
+ * Copyright Fractal Computers, Inc. 2021
+ * @file bitrate.h
+ * @brief This file contains the client's adaptive bitrate code. Any algorithms we are using for
+predicting bitrate should be stored here.
+============================
+Usage
+============================
+Place to put any predictive/adaptive bitrate algorithms. In the current setup, each algorithm is a
+function that takes in inputs through a BitrateStatistics struct. The function is responsible for
+maintaining any internal state necessary for the algorithm (possibly through the use of custom types
+or helpers), and should update max_bitrate when necessary. To change the algorithm the client uses,
+set calculate_new_bitrate to the desired algorithm.
+*/
+
+/*
+============================
+Includes
+============================
+*/
+#include "bitrate.h"
+
+volatile int max_bitrate = STARTING_BITRATE;
+volatile int max_burst_bitrate = STARTING_BURST_BITRATE;
+#define BAD_BITRATE 10400000
+#define BAD_BURST_BITRATE 31800000
+
+/*
+============================
+Private Functions
+============================
+*/
+Bitrates fallback_bitrate(BitrateStatistics stats);
+Bitrates ewma_bitrate(BitrateStatistics stats);
+
+/*
+============================
+Private Function Implementations
+============================
+*/
+Bitrates fallback_bitrate(BitrateStatistics stats) {
+    /*
+        Switches between two sets of bitrate/burst bitrate: the default of 16mbps/100mbps and a
+       fallback of 10mbps/30mbps. We fall back if we've nacked a lot in the last second.
+
+        Arguments:
+            stats (BitrateStatistics): struct containing the average number of nacks per second
+       since the last time this function was called
+    */
+    static Bitrates bitrates;
+    if (stats.num_nacks_per_second > 6 && max_bitrate != BAD_BITRATE) {
+        bitrates.bitrate = BAD_BITRATE;
+        bitrates.burst_bitrate = BAD_BURST_BITRATE;
+    } else {
+        bitrates.bitrate = max_bitrate;
+        bitrates.burst_bitrate = max_burst_bitrate;
+    }
+    return bitrates;
+}
+
+Bitrates ewma_bitrate(BitrateStatistics stats) {
+    /*
+        Keeps an exponentially weighted moving average of the throughput per second the client is
+       getting, and uses that to predict a good bitrate to ask the server for.
+
+        Arguments:
+            stats (BitrateStatistics): struct containing the throughput per second of the client
+    */
+    static const double alpha = 0.8;
+    // because the max bitrate of the encoder is usually larger than the actual amount of data we
+    // get from the server
+    static const double bitrate_throughput_ratio = 1.25;
+    static int throughput = -1;
+    static Bitrates bitrates;
+    if (throughput == -1) {
+        throughput = (int)(STARTING_BITRATE / bitrate_throughput_ratio);
+    }
+    // sanity check to make sure we're not sending it negative bitrate
+    if (stats.throughput_per_second >= 0) {
+        throughput = (int)(alpha * throughput + (1 - alpha) * stats.throughput_per_second);
+        bitrates.bitrate = (int)(bitrate_throughput_ratio * throughput);
+    }
+    bitrates.burst_bitrate = max_burst_bitrate;
+    return bitrates;
+}
+
+/*
+============================
+Public Function Implementations
+============================
+*/
+
+BitrateCalculator calculate_new_bitrate = fallback_bitrate;

protocol/client/bitrate.h

@@ -0,0 +1,54 @@
+#ifndef CLIENT_BITRATE_H
+#define CLIENT_BITRATE_H
+/**
+ * Copyright Fractal Computers, Inc. 2021
+ * @file bitrate.h
+ * @brief This file contains the client's adaptive bitrate code. Any algorithms we are using for
+predicting bitrate should be stored here.
+============================
+Usage
+============================
+The client should periodically call calculate_new_bitrate with any data rrequested, such as
+throughput or nack data. This will update max_bitrate and max_burst_bitrate if needed.
+*/
+
+/*
+============================
+Includes
+============================
+*/
+
+#include <fractal/core/fractal.h>
+
+/*
+============================
+Custom Types
+============================
+*/
+
+typedef struct BitrateStatistics {
+    int num_nacks_per_second;
+    int throughput_per_second;
+} BitrateStatistics;
+
+typedef struct Bitrates {
+    int bitrate;
+    int burst_bitrate;
+} Bitrates;
+
+typedef Bitrates (*BitrateCalculator)(BitrateStatistics);
+
+/*
+============================
+Public Functions
+============================
+*/
+
+/**
+ * @brief       Update max_bitrate and max_burst_bitrate with the latest client data. This function
+ * does not trigger any client bitrate updates.
+ *
+ * @param stats         A struct containing any information we might need to update bitrate.
+ */
+extern BitrateCalculator calculate_new_bitrate;
+#endif

protocol/client/main.c

@@ -61,8 +61,8 @@ volatile int server_height = -1;
 volatile CodecType server_codec_type = CODEC_TYPE_UNKNOWN;
 
 // maximum mbps
-volatile int max_bitrate = STARTING_BITRATE;
-volatile int max_burst_bitrate = STARTING_BURST_BITRATE;
+extern volatile int max_bitrate;
+extern volatile int max_burst_bitrate;
 volatile bool update_bitrate = false;
 
 // Global state variables

protocol/client/video.c

@@ -32,6 +32,7 @@ Includes
 #include "sdlscreeninfo.h"
 #include "native_window_utils.h"
 #include "network.h"
+#include "bitrate.h"
 
 #define USE_HARDWARE true
 #define NO_NACKS_DURING_IFRAME false
@@ -698,21 +699,25 @@ void calculate_statistics() {
 
     static clock t;
     static bool init_t = false;
+    static BitrateStatistics stats;
+    static Bitrates new_bitrates;
     if (!init_t) {
         start_timer(&t);
         init_t = true;
     }
+    // do some calculation
     // Update mbps every 5 seconds
     if (get_timer(t) > 5.0) {
-        // Note: Uncomment to oscillate between bitrates every 5 seconds
-        if (max_bitrate > STARTING_BITRATE) {
-            max_bitrate = STARTING_BITRATE;
-            max_burst_bitrate = STARTING_BURST_BITRATE;
-        } else {
-            // max_bitrate += 1000000;
-            // max_burst_bitrate += 1000000;
+        stats.num_nacks_per_second = video_ring_buffer->num_nacked / 5;
+        stats.throughput_per_second = -1;
+        new_bitrates = calculate_new_bitrate(stats);
+        if (new_bitrates.bitrate != max_bitrate ||
+            new_bitrates.burst_bitrate != max_burst_bitrate) {
+            max_bitrate = new_bitrates.bitrate;
+            max_burst_bitrate = new_bitrates.burst_bitrate;
+            update_bitrate = true;
         }
-        // update_bitrate = true;
+        video_ring_buffer->num_nacked = 0;
         start_timer(&t);
     }
 }

protocol/fractal/core/fractal.h

@@ -118,8 +118,8 @@ Defines
 #define MINIMUM_BITRATE 2000000
 #define ACK_REFRESH_MS 50
 
-#define STARTING_BITRATE 10400000
-#define STARTING_BURST_BITRATE 31800000
+#define STARTING_BITRATE 15400000
+#define STARTING_BURST_BITRATE 100000000
 
 // 16:10 aspect ratio for minimum screen, which is the Mac aspect ratio
 #define MIN_SCREEN_WIDTH 480
@@ -128,7 +128,8 @@ Defines
 #define MAX_SCREEN_HEIGHT 4096
 
 #define AUDIO_BITRATE 128000
-#define FPS 45
+// We would like to set this higher, but mac displays have a max refresh rate of 60 Hz.
+#define FPS 60
 #define MIN_FPS 10
 #define OUTPUT_WIDTH 1280
 #define OUTPUT_HEIGHT 720

protocol/server/handle_client_message.c

@@ -307,7 +307,7 @@ static int handle_ping_message(FractalClientMessage *fmsg, int client_id, bool i
     fmsg_response.ping_id = fmsg->ping_id;
     int ret = 0;
     if (send_udp_packet(&(clients[client_id].UDP_context), PACKET_MESSAGE,
-                        (uint8_t *)&fmsg_response, sizeof(fmsg_response), 1, STARTING_BURST_BITRATE,
+                        (uint8_t *)&fmsg_response, sizeof(fmsg_response), 1, max_burst_bitrate,
                         NULL, NULL) < 0) {
         LOG_WARNING("Could not send Ping to Client ID: %d", client_id);
         ret = -1;

protocol/server/video.c

@@ -492,7 +492,7 @@ int32_t multithreaded_send_video(void* opaque) {
 
                         // LOG_INFO("Size: %d, MBPS: %f, VS MAX MBPS: %f, Time:
                         // %f, Transmit Time: %f, Delay: %f",
-                        // previous_frame_size, mbps, max_mbps, frame_time,
+                        // previous_frame_size, mbps, max_bitrate, frame_time,
                         // transmit_time, delay);
 
                         if ((current_fps < worst_fps || ideal_bitrate > current_bitrate) &&