feat: Implement Tox network profiler

.gitignore

@@ -36,6 +36,7 @@ testing/data
 
 # Vim
 *.swp
+*.nvimlog
 
 # Object files
 *.o

CMakeLists.txt

@@ -309,6 +309,8 @@ set(toxcore_SOURCES
   toxcore/mono_time.h
   toxcore/net_crypto.c
   toxcore/net_crypto.h
+  toxcore/net_profile.c
+  toxcore/net_profile.h
   toxcore/network.c
   toxcore/network.h
   toxcore/onion_announce.c

auto_tests/CMakeLists.txt

@@ -71,6 +71,7 @@ auto_test(invalid_udp_proxy)
 auto_test(lan_discovery)
 auto_test(lossless_packet)
 auto_test(lossy_packet)
+auto_test(netprof)
 auto_test(network)
 auto_test(onion)
 auto_test(overflow_recvq)

auto_tests/TCP_test.c

@@ -111,12 +111,12 @@ static void test_basic(void)
 
     // Sending the handshake
     ck_assert_msg(net_send(ns, logger, sock, handshake, TCP_CLIENT_HANDSHAKE_SIZE - 1,
-                           &localhost) == TCP_CLIENT_HANDSHAKE_SIZE - 1,
+                           &localhost, nullptr) == TCP_CLIENT_HANDSHAKE_SIZE - 1,
                   "An attempt to send the initial handshake minus last byte failed.");
 
     do_tcp_server_delay(tcp_s, mono_time, 50);
 
-    ck_assert_msg(net_send(ns, logger, sock, handshake + (TCP_CLIENT_HANDSHAKE_SIZE - 1), 1, &localhost) == 1,
+    ck_assert_msg(net_send(ns, logger, sock, handshake + (TCP_CLIENT_HANDSHAKE_SIZE - 1), 1, &localhost, nullptr) == 1,
                   "The attempt to send the last byte of handshake failed.");
 
     free(handshake);
@@ -155,7 +155,7 @@ static void test_basic(void)
             msg_length = sizeof(r_req) - i;
         }
 
-        ck_assert_msg(net_send(ns, logger, sock, r_req + i, msg_length, &localhost) == msg_length,
+        ck_assert_msg(net_send(ns, logger, sock, r_req + i, msg_length, &localhost, nullptr) == msg_length,
                       "Failed to send request after completing the handshake.");
         i += msg_length;
 
@@ -234,12 +234,12 @@ static struct sec_TCP_con *new_tcp_con(const Logger *logger, const Memory *mem,
                   "Failed to encrypt the outgoing handshake.");
 
     ck_assert_msg(net_send(ns, logger, sock, handshake, TCP_CLIENT_HANDSHAKE_SIZE - 1,
-                           &localhost) == TCP_CLIENT_HANDSHAKE_SIZE - 1,
+                           &localhost, nullptr) == TCP_CLIENT_HANDSHAKE_SIZE - 1,
                   "Failed to send the first portion of the handshake to the TCP relay server.");
 
     do_tcp_server_delay(tcp_s, mono_time, 50);
 
-    ck_assert_msg(net_send(ns, logger, sock, handshake + (TCP_CLIENT_HANDSHAKE_SIZE - 1), 1, &localhost) == 1,
+    ck_assert_msg(net_send(ns, logger, sock, handshake + (TCP_CLIENT_HANDSHAKE_SIZE - 1), 1, &localhost, nullptr) == 1,
                   "Failed to send last byte of handshake.");
 
     do_tcp_server_delay(tcp_s, mono_time, 50);
@@ -283,7 +283,7 @@ static int write_packet_tcp_test_connection(const Logger *logger, struct sec_TCP
     localhost.ip = get_loopback();
     localhost.port = 0;
 
-    ck_assert_msg(net_send(con->ns, logger, con->sock, packet, packet_size, &localhost) == packet_size,
+    ck_assert_msg(net_send(con->ns, logger, con->sock, packet, packet_size, &localhost, nullptr) == packet_size,
                   "Failed to send a packet.");
     return 0;
 }
@@ -524,7 +524,7 @@ static void test_client(void)
     ip_port_tcp_s.port = net_htons(ports[random_u32(rng) % NUM_PORTS]);
     ip_port_tcp_s.ip = get_loopback();
 
-    TCP_Client_Connection *conn = new_tcp_connection(logger, mem, mono_time, rng, ns, &ip_port_tcp_s, self_public_key, f_public_key, f_secret_key, nullptr);
+    TCP_Client_Connection *conn = new_tcp_connection(logger, mem, mono_time, rng, ns, &ip_port_tcp_s, self_public_key, f_public_key, f_secret_key, nullptr, nullptr);
     // TCP sockets might need a moment before they can be written to.
     c_sleep(50);
     do_tcp_connection(logger, mono_time, conn, nullptr);
@@ -560,7 +560,7 @@ static void test_client(void)
     crypto_new_keypair(rng, f2_public_key, f2_secret_key);
     ip_port_tcp_s.port = net_htons(ports[random_u32(rng) % NUM_PORTS]);
     TCP_Client_Connection *conn2 = new_tcp_connection(logger, mem, mono_time, rng, ns, &ip_port_tcp_s, self_public_key, f2_public_key,
-                                   f2_secret_key, nullptr);
+                                   f2_secret_key, nullptr, nullptr);
     c_sleep(50);
 
     // The client should call this function (defined earlier) during the routing process.
@@ -657,7 +657,7 @@ static void test_client_invalid(void)
     ip_port_tcp_s.port = net_htons(ports[random_u32(rng) % NUM_PORTS]);
     ip_port_tcp_s.ip = get_loopback();
     TCP_Client_Connection *conn = new_tcp_connection(logger, mem, mono_time, rng, ns, &ip_port_tcp_s,
-                                  self_public_key, f_public_key, f_secret_key, nullptr);
+                                  self_public_key, f_public_key, f_secret_key, nullptr, nullptr);
 
     // Run the client's main loop but not the server.
     mono_time_update(mono_time);

auto_tests/netprof_test.c

@@ -0,0 +1,121 @@
+/** Auto Tests: basic network profile functionality test (UDP only)
+ * TODO(JFreegman): test TCP packets as well
+ */
+
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "../toxcore/tox_private.h"
+#include "../toxcore/util.h"
+
+#include "auto_test_support.h"
+#include "check_compat.h"
+
+#define NUM_TOXES 2
+
+static void test_netprof(AutoTox *autotoxes)
+{
+    // Send some messages to create fake traffic
+    for (size_t i = 0; i < 256; ++i) {
+        for (uint32_t j = 0; j < NUM_TOXES; ++j) {
+            tox_friend_send_message(autotoxes[j].tox, 0, TOX_MESSAGE_TYPE_NORMAL, (const uint8_t *)"test", 4, nullptr);
+        }
+
+        iterate_all_wait(autotoxes, NUM_TOXES, ITERATION_INTERVAL);
+    }
+
+    // idle traffic for a while
+    for (size_t i = 0; i < 100; ++i) {
+        iterate_all_wait(autotoxes, NUM_TOXES, ITERATION_INTERVAL);
+    }
+
+    const Tox *tox1 = autotoxes[0].tox;
+
+    const uint64_t UDP_count_sent1 = tox_netprof_get_packet_total_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP,
+                                     TOX_NETPROF_DIRECTION_SENT);
+    const uint64_t UDP_count_recv1 = tox_netprof_get_packet_total_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP,
+                                     TOX_NETPROF_DIRECTION_RECV);
+    const uint64_t TCP_count_sent1 = tox_netprof_get_packet_total_count(tox1, TOX_NETPROF_PACKET_TYPE_TCP,
+                                     TOX_NETPROF_DIRECTION_SENT);
+    const uint64_t TCP_count_recv1 = tox_netprof_get_packet_total_count(tox1, TOX_NETPROF_PACKET_TYPE_TCP,
+                                     TOX_NETPROF_DIRECTION_RECV);
+
+    const uint64_t UDP_bytes_sent1 = tox_netprof_get_packet_total_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP,
+                                     TOX_NETPROF_DIRECTION_SENT);
+    const uint64_t UDP_bytes_recv1 = tox_netprof_get_packet_total_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP,
+                                     TOX_NETPROF_DIRECTION_RECV);
+    const uint64_t TCP_bytes_sent1 = tox_netprof_get_packet_total_bytes(tox1, TOX_NETPROF_PACKET_TYPE_TCP,
+                                     TOX_NETPROF_DIRECTION_SENT);
+    const uint64_t TCP_bytes_recv1 = tox_netprof_get_packet_total_bytes(tox1, TOX_NETPROF_PACKET_TYPE_TCP,
+                                     TOX_NETPROF_DIRECTION_RECV);
+
+    ck_assert(UDP_count_recv1 > 0 && UDP_count_sent1 > 0);
+    ck_assert(UDP_bytes_recv1 > 0 && UDP_bytes_sent1 > 0);
+
+    (void)TCP_count_sent1;
+    (void)TCP_bytes_sent1;
+    (void)TCP_bytes_recv1;
+    (void)TCP_count_recv1;
+
+    uint64_t total_sent_count = 0;
+    uint64_t total_recv_count = 0;
+    uint64_t total_sent_bytes = 0;
+    uint64_t total_recv_bytes = 0;
+
+    // tox1 makes sure the sum value of all packet ID's is equal to the totals
+    for (size_t i = 0; i < 256; ++i) {
+        // this id isn't valid for UDP packets but we still want to call the
+        // functions and make sure they return some non-zero value
+        if (i == TOX_NETPROF_PACKET_ID_TCP_DATA) {
+            ck_assert(tox_netprof_get_packet_id_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                      TOX_NETPROF_DIRECTION_SENT) > 0);
+            ck_assert(tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                      TOX_NETPROF_DIRECTION_SENT) > 0);
+            ck_assert(tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                      TOX_NETPROF_DIRECTION_SENT) > 0);
+            ck_assert(tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                      TOX_NETPROF_DIRECTION_RECV) > 0);
+            continue;
+        }
+
+        total_sent_count += tox_netprof_get_packet_id_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                            TOX_NETPROF_DIRECTION_SENT);
+        total_recv_count += tox_netprof_get_packet_id_count(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                            TOX_NETPROF_DIRECTION_RECV);
+
+        total_sent_bytes += tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                            TOX_NETPROF_DIRECTION_SENT);
+        total_recv_bytes += tox_netprof_get_packet_id_bytes(tox1, TOX_NETPROF_PACKET_TYPE_UDP, i,
+                            TOX_NETPROF_DIRECTION_RECV);
+    }
+
+    const uint64_t total_packets = total_sent_count + total_recv_count;
+    ck_assert_msg(total_packets == UDP_count_sent1 + UDP_count_recv1,
+                  "%" PRIu64 "does not match %" PRIu64 "\n", total_packets, UDP_count_sent1 + UDP_count_recv1);
+
+    ck_assert_msg(total_sent_count == UDP_count_sent1, "%" PRIu64 " does not match %" PRIu64 "\n", total_sent_count, UDP_count_sent1);
+    ck_assert_msg(total_recv_count == UDP_count_recv1, "%" PRIu64 " does not match %" PRIu64"\n", total_recv_count, UDP_count_recv1);
+
+
+    const uint64_t total_bytes = total_sent_bytes + total_recv_bytes;
+    ck_assert_msg(total_bytes == UDP_bytes_sent1 + UDP_bytes_recv1,
+                  "%" PRIu64 "does not match %" PRIu64 "\n", total_bytes, UDP_bytes_sent1 + UDP_bytes_recv1);
+
+    ck_assert_msg(total_sent_bytes == UDP_bytes_sent1, "%" PRIu64 " does not match %" PRIu64 "\n", total_sent_bytes, UDP_bytes_sent1);
+    ck_assert_msg(total_recv_bytes == UDP_bytes_recv1, "%" PRIu64 " does not match %" PRIu64 "\n", total_recv_bytes, UDP_bytes_recv1);
+}
+
+int main(void)
+{
+    setvbuf(stdout, nullptr, _IONBF, 0);
+
+    Run_Auto_Options autotox_opts = default_run_auto_options();
+    autotox_opts.graph = GRAPH_COMPLETE;
+
+    run_auto_test(nullptr, NUM_TOXES, test_netprof, 0, &autotox_opts);
+
+    return 0;
+}
+
+#undef NUM_TOXES

toxcore/BUILD.bazel

@@ -346,6 +346,18 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "net_profile",
+    srcs = ["net_profile.c"],
+    hdrs = ["net_profile.h"],
+    deps = [
+        ":attributes",
+        ":ccompat",
+        ":logger",
+        ":mem",
+    ],
+)
+
 cc_library(
     name = "network",
     srcs = ["network.c"],
@@ -365,6 +377,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":util",
         "@libsodium",
         "@psocket",
@@ -629,6 +642,7 @@ cc_library(
         ":crypto_core",
         ":logger",
         ":mem",
+        ":net_profile",
         ":network",
     ],
 )
@@ -656,6 +670,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":network",
         ":onion",
         ":util",
@@ -677,6 +692,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":network",
         ":util",
     ],
@@ -699,6 +715,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":network",
         ":onion",
         ":util",
@@ -736,6 +753,7 @@ cc_library(
         ":logger",
         ":mem",
         ":mono_time",
+        ":net_profile",
         ":network",
         ":util",
         "@pthread",
@@ -1060,6 +1078,7 @@ cc_library(
         ":DHT",
         ":Messenger",
         ":TCP_client",
+        ":TCP_server",
         ":attributes",
         ":ccompat",
         ":crypto_core",
@@ -1070,6 +1089,7 @@ cc_library(
         ":mem",
         ":mono_time",
         ":net_crypto",
+        ":net_profile",
         ":network",
         ":onion_client",
         ":state",

toxcore/Makefile.inc

@@ -74,6 +74,8 @@ libtoxcore_la_SOURCES = ../third_party/cmp/cmp.c \
                         ../toxcore/ping_array.c \
                         ../toxcore/net_crypto.h \
                         ../toxcore/net_crypto.c \
+                        ../toxcore/net_profile.c \
+                        ../toxcore/net_profile.h \
                         ../toxcore/friend_requests.h \
                         ../toxcore/friend_requests.c \
                         ../toxcore/LAN_discovery.h \

toxcore/TCP_client.c

@@ -20,6 +20,7 @@
 #include "logger.h"
 #include "mem.h"
 #include "mono_time.h"
+#include "net_profile.h"
 #include "network.h"
 #include "util.h"
 
@@ -582,7 +583,7 @@ void forwarding_handler(TCP_Client_Connection *con, forwarded_response_cb *forwa
 TCP_Client_Connection *new_tcp_connection(
     const Logger *logger, const Memory *mem, const Mono_Time *mono_time, const Random *rng, const Network *ns,
     const IP_Port *ip_port, const uint8_t *public_key, const uint8_t *self_public_key, const uint8_t *self_secret_key,
-    const TCP_Proxy_Info *proxy_info)
+    const TCP_Proxy_Info *proxy_info, Net_Profile *net_profile)
 {
     assert(logger != nullptr);
     assert(mem != nullptr);
@@ -634,6 +635,7 @@ TCP_Client_Connection *new_tcp_connection(
     temp->con.rng = rng;
     temp->con.sock = sock;
     temp->con.ip_port = *ip_port;
+    temp->con.net_profile = net_profile;
     memcpy(temp->public_key, public_key, CRYPTO_PUBLIC_KEY_SIZE);
     memcpy(temp->self_public_key, self_public_key, CRYPTO_PUBLIC_KEY_SIZE);
     encrypt_precompute(temp->public_key, self_secret_key, temp->con.shared_key);
@@ -819,6 +821,8 @@ static int handle_tcp_client_packet(const Logger *logger, TCP_Client_Connection
         return -1;
     }
 
+    netprof_record_packet(conn->con.net_profile, data[0], length, PACKET_DIRECTION_RECV);
+
     switch (data[0]) {
         case TCP_PACKET_ROUTING_RESPONSE:
             return handle_tcp_client_routing_response(conn, data, length);

toxcore/TCP_client.h

@@ -15,6 +15,7 @@
 #include "logger.h"
 #include "mem.h"
 #include "mono_time.h"
+#include "net_profile.h"
 #include "network.h"
 
 #define TCP_CONNECTION_TIMEOUT 10
@@ -60,11 +61,11 @@ non_null()
 void tcp_con_set_custom_uint(TCP_Client_Connection *con, uint32_t value);
 
 /** Create new TCP connection to ip_port/public_key */
-non_null(1, 2, 3, 4, 5, 6, 7, 8, 9) nullable(10)
+non_null(1, 2, 3, 4, 5, 6, 7, 8, 9) nullable(10, 11)
 TCP_Client_Connection *new_tcp_connection(
     const Logger *logger, const Memory *mem, const Mono_Time *mono_time, const Random *rng, const Network *ns,
     const IP_Port *ip_port, const uint8_t *public_key, const uint8_t *self_public_key, const uint8_t *self_secret_key,
-    const TCP_Proxy_Info *proxy_info);
+    const TCP_Proxy_Info *proxy_info, Net_Profile *net_profile);
 
 /** Run the TCP connection */
 non_null(1, 2, 3) nullable(4)