support rtp/rtcp over tcp (ntop#2422)

* support rtp/rtcp over tcp as per rfc4571.

Signed-off-by: mmaatuq <mahmoudmatook.mm@gmail.com>

src/include/ndpi_typedefs.h

@@ -908,12 +908,6 @@ struct ndpi_flow_udp_struct {
   /* NDPI_PROTOCOL_XBOX */
   u_int32_t xbox_stage:1;
 
-  /* NDPI_PROTOCOL_RTP */
-  u_int32_t rtp_stage:2;
-
-  /* NDPI_PROTOCOL_RTCP */
-  u_int32_t rtcp_stage:2;
-
   /* NDPI_PROTOCOL_QUIC */
   u_int32_t quic_0rtt_found:1;
   u_int32_t quic_vn_pair:1;
@@ -928,10 +922,6 @@ struct ndpi_flow_udp_struct {
   /* NDPI_PROTOCOL_RAKNET */
   u_int32_t raknet_custom:1;
 
-  /* NDPI_PROTOCOL_RTP */
-  u_int16_t rtp_seq[2];
-  u_int8_t rtp_seq_set[2];
-
   /* NDPI_PROTOCOL_EAQ */
   u_int8_t eaq_pkt_id;
   u_int32_t eaq_sequence;
@@ -1440,6 +1430,9 @@ struct ndpi_flow_struct {
       u_int8_t url_count;
       char url[4][48];
     } slp;
+
+
+
   } protos;
 
   /*** ALL protocol specific 64 bit variables here ***/
@@ -1498,6 +1491,14 @@ struct ndpi_flow_struct {
 
   /* NDPI_PROTOCOL_TINC */
   u_int8_t tinc_state;
+
+   /* NDPI_PROTOCOL_RTP */
+   u_int32_t rtp_stage:2;
+   u_int8_t rtp_seq_set[2];
+   u_int16_t rtp_seq[2];
+
+   /* NDPI_PROTOCOL_RTCP */ 
+   u_int32_t rtcp_stage:2;
 
   /* Flow payload */
   u_int16_t flow_payload_len;

src/lib/protocols/protobuf.c

@@ -228,7 +228,7 @@ static void ndpi_search_protobuf(struct ndpi_detection_module_struct *ndpi_struc
 #endif
   if ((protobuf_elements >= PROTOBUF_REQUIRED_ELEMENTS && protobuf_len_elements > 0 &&
        /* (On UDP) this packet might be also a RTP/RTCP one. Wait for the next one */
-       (flow->packet_counter > 1 || flow->l4_proto == IPPROTO_TCP || flow->l4.udp.rtp_stage == 0))
+       (flow->packet_counter > 1 || flow->l4_proto == IPPROTO_TCP || flow->rtp_stage == 0))
       || (flow->packet_counter >= PROTOBUF_MIN_PACKETS && protobuf_elements >= PROTOBUF_MIN_ELEMENTS))
   {
 #ifdef DEBUG_PROTOBUF

src/lib/protocols/raknet.c

@@ -295,7 +295,7 @@ static void ndpi_search_raknet(struct ndpi_detection_module_struct *ndpi_struct,
         if (frame_offset == packet->payload_packet_len)
         {
           /* This packet might also be a RTP/RTCP one: give precedence to RTP/RTCP dissector */
-          if(flow->l4.udp.rtp_stage == 0 && flow->l4.udp.rtcp_stage == 0)
+          if(flow->rtp_stage == 0 && flow->rtcp_stage == 0)
             ndpi_int_raknet_add_connection(ndpi_struct, flow);
         } else {
           exclude_proto(ndpi_struct, flow);
@@ -366,7 +366,7 @@ static void ndpi_search_raknet(struct ndpi_detection_module_struct *ndpi_struct,
         if (record_index == record_count && record_offset == packet->payload_packet_len)
         {
           /* This packet might also be a RTP/RTCP one: give precedence to RTP/RTCP dissector */
-          if(flow->l4.udp.rtp_stage == 0 && flow->l4.udp.rtcp_stage == 0)
+          if(flow->rtp_stage == 0 && flow->rtcp_stage == 0)
             ndpi_int_raknet_add_connection(ndpi_struct, flow);
         } else {
           exclude_proto(ndpi_struct, flow);

src/lib/protocols/rtp.c

@@ -90,10 +90,15 @@ int is_rtp_or_rtcp(struct ndpi_detection_module_struct *ndpi_struct, u_int16_t *
   u_int16_t ext_len;
   u_int32_t min_len;
   const u_int8_t *payload = packet->payload;
-  const u_int16_t payload_len = packet->payload_packet_len;
+  u_int16_t payload_len = packet->payload_packet_len;
 
   if(payload_len < 2)
     return NO_RTP_RTCP;
+
+  if(packet->tcp != NULL) {
+    payload_len -= 2;
+    payload += 2; /* Skip the length field */
+  }
 
   if((payload[0] & 0xC0) != 0x80) { /* Version 2 */
     NDPI_LOG_DBG(ndpi_struct, "Not version 2\n");
@@ -142,29 +147,23 @@ int is_rtp_or_rtcp(struct ndpi_detection_module_struct *ndpi_struct, u_int16_t *
 static void ndpi_rtp_search(struct ndpi_detection_module_struct *ndpi_struct,
 			    struct ndpi_flow_struct *flow) {
   u_int8_t is_rtp;
-  u_int16_t d_port = ntohs(ndpi_struct->packet.udp->dest);
   struct ndpi_packet_struct *packet = &ndpi_struct->packet;
   const u_int8_t *payload = packet->payload;
   u_int16_t seq;
 
-  NDPI_LOG_DBG(ndpi_struct, "search RTP (stage %d/%d)\n", flow->l4.udp.rtp_stage, flow->l4.udp.rtcp_stage);
-
-  if(d_port == 5355 || /* LLMNR_PORT */
-     d_port == 5353 || /* MDNS_PORT */
-     d_port == 9600    /* FINS_PORT */) {
-    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
-    NDPI_EXCLUDE_PROTO_EXT(ndpi_struct, flow, NDPI_PROTOCOL_RTCP);
-    return;
+  if(packet->tcp != NULL) {
+      payload += 2; /* Skip the length field */
   }
+  NDPI_LOG_DBG(ndpi_struct, "search RTP (stage %d/%d)\n", flow->rtp_stage, flow->rtcp_stage);
 
   /* * Let some "unknown" packets at the beginning:
      * search for 3/4 consecutive RTP/RTCP packets.
      * Wait a little longer (4 vs 3 pkts) for RTCP to try to tell if there are only
      * RTCP packets in the flow or if RTP/RTCP are multiplexed together */
 
   if(flow->packet_counter > 3 &&
-     flow->l4.udp.rtp_stage == 0 &&
-     flow->l4.udp.rtcp_stage == 0) {
+     flow->rtp_stage == 0 &&
+     flow->rtcp_stage == 0) {
     NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
     NDPI_EXCLUDE_PROTO_EXT(ndpi_struct, flow, NDPI_PROTOCOL_RTCP);
     return;
@@ -173,19 +172,19 @@ static void ndpi_rtp_search(struct ndpi_detection_module_struct *ndpi_struct,
   is_rtp = is_rtp_or_rtcp(ndpi_struct, &seq);
 
   if(is_rtp == IS_RTP) {
-    if(flow->l4.udp.rtp_stage == 2) {
+    if(flow->rtp_stage == 2) {
       if(flow->l4.udp.line_pkts[0] >= 2 && flow->l4.udp.line_pkts[1] >= 2) {
         /* It seems that it is a LINE stuff; let its dissector to evaluate */
       } else if(flow->l4.udp.epicgames_stage > 0) {
         /* It seems that it is a EpicGames stuff; let its dissector to evaluate */
-      } else if(flow->l4.udp.rtp_seq_set[packet->packet_direction] &&
-                flow->l4.udp.rtp_seq[packet->packet_direction] == seq) {
+      } else if(flow->rtp_seq_set[packet->packet_direction] &&
+                flow->rtp_seq[packet->packet_direction] == seq) {
         /* Simple heuristic to avoid false positives. tradeoff between:
 	   * consecutive RTP packets should have different sequence number
 	   * we should handle duplicated traffic */
         NDPI_LOG_DBG(ndpi_struct, "Same seq on consecutive pkts\n");
-        flow->l4.udp.rtp_stage = 0;
-        flow->l4.udp.rtcp_stage = 0;
+        flow->rtp_stage = 0;
+        flow->rtcp_stage = 0;
         NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
         NDPI_EXCLUDE_PROTO_EXT(ndpi_struct, flow, NDPI_PROTOCOL_RTCP);
       } else {
@@ -198,32 +197,32 @@ static void ndpi_rtp_search(struct ndpi_detection_module_struct *ndpi_struct,
       }
       return;
     }
-    if(flow->l4.udp.rtp_stage == 0) {
-      flow->l4.udp.rtp_seq[packet->packet_direction] = seq;
-      flow->l4.udp.rtp_seq_set[packet->packet_direction] = 1;
+    if(flow->rtp_stage == 0) {
+      flow->rtp_seq[packet->packet_direction] = seq;
+      flow->rtp_seq_set[packet->packet_direction] = 1;
     }
-    flow->l4.udp.rtp_stage += 1;
-  } else if(is_rtp == IS_RTCP && flow->l4.udp.rtp_stage > 0) {
+    flow->rtp_stage += 1;
+  } else if(is_rtp == IS_RTCP && flow->rtp_stage > 0) {
     /* RTCP after (some) RTP. Keep looking for RTP */
-  } else if(is_rtp == IS_RTCP && flow->l4.udp.rtp_stage == 0) {
-    if(flow->l4.udp.rtcp_stage == 3) {
+  } else if(is_rtp == IS_RTCP && flow->rtp_stage == 0) {
+    if(flow->rtcp_stage == 3) {
       NDPI_LOG_INFO(ndpi_struct, "Found RTCP\n");
       ndpi_set_detected_protocol(ndpi_struct, flow,
                                  NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_RTCP,
                                  NDPI_CONFIDENCE_DPI);
       return;
     }
-    flow->l4.udp.rtcp_stage += 1;
+    flow->rtcp_stage += 1;
   } else {
-    if(flow->l4.udp.rtp_stage || flow->l4.udp.rtcp_stage) {
+    if(flow->rtp_stage || flow->rtcp_stage) {
       u_int16_t app_proto; /* unused */
       u_int32_t unused;
 
       /* TODO: we should switch to the demultiplexing-code in stun dissector */
       if(is_stun(ndpi_struct, flow, &app_proto) != 0 &&
          !is_dtls(packet->payload, packet->payload_packet_len, &unused)) {
-        flow->l4.udp.rtp_stage = 0;
-        flow->l4.udp.rtcp_stage = 0;
+        flow->rtp_stage = 0;
+        flow->rtcp_stage = 0;
         NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
         NDPI_EXCLUDE_PROTO_EXT(ndpi_struct, flow, NDPI_PROTOCOL_RTCP);
       }
@@ -235,21 +234,68 @@ static void ndpi_rtp_search(struct ndpi_detection_module_struct *ndpi_struct,
 }
 
 /* *************************************************************** */
+/* https://datatracker.ietf.org/doc/html/rfc4571
+ * message format for RTP/RTCP over TCP:
+ *     0                   1                   2                   3
+ *      0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ *      ---------------------------------------------------------------
+ *     |             LENGTH            |  RTP or RTCP packet ...       |
+ *      ---------------------------------------------------------------
+ */
+static void ndpi_search_rtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
+                struct ndpi_flow_struct *flow)
+{
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+  const u_int8_t *payload = packet->payload;
+
+  if(packet->payload_packet_len < 4){ /* (2) len field + (2) min rtp/rtcp*/
+    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+    NDPI_EXCLUDE_PROTO_EXT(ndpi_struct, flow, NDPI_PROTOCOL_RTCP);
+    return;
+  }
 
-static void ndpi_search_rtp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
+  u_int16_t len = ntohs(get_u_int16_t(payload, 0));
+  if(len + sizeof(len) != packet->payload_packet_len) { /*fragmented packets are not handled*/
+    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+    NDPI_EXCLUDE_PROTO_EXT(ndpi_struct, flow, NDPI_PROTOCOL_RTCP);
+  } else {
+    ndpi_rtp_search(ndpi_struct, flow);
+  }
+
+}
+
+/* *************************************************************** */
+static void ndpi_search_rtp_udp(struct ndpi_detection_module_struct *ndpi_struct,
+                struct ndpi_flow_struct *flow)
 {
   struct ndpi_packet_struct *packet = &ndpi_struct->packet;
   u_int16_t source = ntohs(packet->udp->source);
   u_int16_t dest = ntohs(packet->udp->dest);
-
-  if((source != 30303) && (dest != 30303 /* Avoid to mix it with Ethereum that looks alike */)
-     && (dest > 1023)
-     )
-    ndpi_rtp_search(ndpi_struct, flow);
-  else {
+  /* 
+   * XXX: not sure if rtp/rtcp over tcp will also mix with Ethereum
+   * for now, will not add it unitl we have a false positive.
+   */
+  if((source == 30303) || (dest == 30303 /* Avoid to mix it with Ethereum that looks alike */)
+     || (dest == 5355  /* LLMNR_PORT */)
+     || (dest == 5353  /* MDNS_PORT */)
+     || (dest == 9600  /* FINS_PORT */)
+     || (dest <= 1023)){
     NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
     NDPI_EXCLUDE_PROTO_EXT(ndpi_struct, flow, NDPI_PROTOCOL_RTCP);
+    return;
   }
+  ndpi_rtp_search(ndpi_struct, flow);
+}
+
+/* *************************************************************** */
+static void ndpi_search_rtp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
+{
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+  if(packet->tcp != NULL) {
+    ndpi_search_rtp_tcp(ndpi_struct, flow);
+   } else {
+      ndpi_search_rtp_udp(ndpi_struct, flow);
+   }
 }
 
 /* *************************************************************** */
@@ -259,7 +305,7 @@ void init_rtp_dissector(struct ndpi_detection_module_struct *ndpi_struct,
   ndpi_set_bitmask_protocol_detection("RTP", ndpi_struct, *id,
 				      NDPI_PROTOCOL_RTP,
 				      ndpi_search_rtp,
-				      NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_UDP_WITH_PAYLOAD,
+                      NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_OR_UDP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION, 
 				      SAVE_DETECTION_BITMASK_AS_UNKNOWN,
 				      ADD_TO_DETECTION_BITMASK);
 

src/lib/protocols/viber.c

@@ -70,7 +70,7 @@ static void ndpi_search_viber(struct ndpi_detection_module_struct *ndpi_struct,
   if((packet->udp != NULL) && (packet->payload_packet_len > 5)) {
     NDPI_LOG_DBG2(ndpi_struct, "calculating dport over udp\n");
 
-    if((flow->l4.udp.rtp_stage == 0) && (flow->l4.udp.rtcp_stage == 0) /* Avoid collisions with RTP/RTCP */ &&
+    if((flow->rtp_stage == 0) && (flow->rtcp_stage == 0) /* Avoid collisions with RTP/RTCP */ &&
        ((packet->payload[2] == 0x03 && packet->payload[3] == 0x00)
         || (packet->payload_packet_len == 20 && packet->payload[2] == 0x09 && packet->payload[3] == 0x00)
         || (packet->payload[2] == 0x01 && packet->payload[3] == 0x00 && packet->payload[4] == 0x05 && packet->payload[5] == 0x00)

tests/cfgs/caches_cfg/result/ookla.pcap.out

@@ -3,7 +3,7 @@ Guessed flow protos:	1
 DPI Packets (TCP):	40	(6.67 pkts/flow)
 Confidence Match by port    : 1 (flows)
 Confidence DPI              : 5 (flows)
-Num dissector calls: 568 (94.67 diss/flow)
+Num dissector calls: 571 (95.17 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/3/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/caches_cfg/result/teams.pcap.out

@@ -7,7 +7,7 @@ Confidence Unknown          : 1 (flows)
 Confidence Match by port    : 1 (flows)
 Confidence DPI (partial)    : 1 (flows)
 Confidence DPI              : 80 (flows)
-Num dissector calls: 511 (6.16 diss/flow)
+Num dissector calls: 512 (6.17 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/9/0 (insert/search/found)
 LRU cache stun:       30/0/0 (insert/search/found)

tests/cfgs/caches_global/result/bittorrent.pcap.out

@@ -1,6 +1,6 @@
 DPI Packets (TCP):	24	(1.00 pkts/flow)
 Confidence DPI              : 24 (flows)
-Num dissector calls: 1740 (72.50 diss/flow)
+Num dissector calls: 1762 (73.42 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 120/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/caches_global/result/mining.pcapng.out

@@ -1,6 +1,6 @@
 DPI Packets (TCP):	17	(4.25 pkts/flow)
 Confidence DPI              : 4 (flows)
-Num dissector calls: 97 (24.25 diss/flow)
+Num dissector calls: 101 (25.25 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/caches_global/result/ookla.pcap.out

@@ -4,7 +4,7 @@ DPI Packets (TCP):	40	(6.67 pkts/flow)
 Confidence DPI (partial cache): 1 (flows)
 Confidence DPI              : 4 (flows)
 Confidence DPI (aggressive) : 1 (flows)
-Num dissector calls: 568 (94.67 diss/flow)
+Num dissector calls: 571 (95.17 diss/flow)
 LRU cache ookla:      4/2/2 (insert/search/found)
 LRU cache bittorrent: 0/3/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/caches_global/result/teams.pcap.out

@@ -7,7 +7,7 @@ Confidence Unknown          : 1 (flows)
 Confidence Match by port    : 1 (flows)
 Confidence DPI (partial)    : 5 (flows)
 Confidence DPI              : 76 (flows)
-Num dissector calls: 511 (6.16 diss/flow)
+Num dissector calls: 512 (6.17 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/9/0 (insert/search/found)
 LRU cache stun:       30/0/0 (insert/search/found)

tests/cfgs/default/result/1kxun.pcap.out

@@ -5,7 +5,7 @@ DPI Packets (UDP):	120	(1.21 pkts/flow)
 Confidence Unknown          : 14 (flows)
 Confidence Match by port    : 6 (flows)
 Confidence DPI              : 177 (flows)
-Num dissector calls: 4923 (24.99 diss/flow)
+Num dissector calls: 4926 (25.01 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/60/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/default/result/443-chrome.pcap.out

@@ -2,7 +2,7 @@ Guessed flow protos:	1
 
 DPI Packets (TCP):	1	(1.00 pkts/flow)
 Confidence Match by port    : 1 (flows)
-Num dissector calls: 149 (149.00 diss/flow)
+Num dissector calls: 150 (150.00 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/3/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/default/result/443-opvn.pcap.out

@@ -1,6 +1,6 @@
 DPI Packets (TCP):	6	(6.00 pkts/flow)
 Confidence DPI              : 1 (flows)
-Num dissector calls: 150 (150.00 diss/flow)
+Num dissector calls: 151 (151.00 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/default/result/KakaoTalk_chat.pcap.out

@@ -5,7 +5,7 @@ DPI Packets (UDP):	36	(2.00 pkts/flow)
 DPI Packets (other):	1	(1.00 pkts/flow)
 Confidence Match by port    : 5 (flows)
 Confidence DPI              : 33 (flows)
-Num dissector calls: 549 (14.45 diss/flow)
+Num dissector calls: 551 (14.50 diss/flow)
 LRU cache ookla:      0/1/0 (insert/search/found)
 LRU cache bittorrent: 0/15/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/default/result/KakaoTalk_talk.pcap.out

@@ -5,7 +5,7 @@ DPI Packets (UDP):	10	(2.00 pkts/flow)
 Confidence Match by port    : 8 (flows)
 Confidence DPI              : 11 (flows)
 Confidence Match by IP      : 1 (flows)
-Num dissector calls: 1205 (60.25 diss/flow)
+Num dissector calls: 1209 (60.45 diss/flow)
 LRU cache ookla:      0/2/0 (insert/search/found)
 LRU cache bittorrent: 0/27/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

tests/cfgs/default/result/Oscar.pcap.out

@@ -2,7 +2,7 @@ Guessed flow protos:	1
 
 DPI Packets (TCP):	21	(21.00 pkts/flow)
 Confidence Match by port    : 1 (flows)
-Num dissector calls: 255 (255.00 diss/flow)
+Num dissector calls: 256 (256.00 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/3/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)