[citrix_adc] Support addition log message types and ECS mappings (#11781

) Improve Citrix ADC integration log parsing and ECS mappings. Changes are: - Support "Mapped Ip" as value for Nat_Ip in all patterns in the sslvpn pipeline - Add support for additional "Message" subtypes, and add a "DATA" wildcard that will capture all patterns. All valid "Message" patterns are not known, so it's better to capture all without parsing individual fields than to cause an error. - Add addition ECS mappings for event.kind, event.outcome, observer.hostname - Calculate event.duration as the difference from event.start and event.end
elastic · Nov 20, 2024 · 4e7f9de · 4e7f9de
1 parent 22070e4
commit 4e7f9de
Show file tree

Hide file tree

Showing 11 changed files with 508 additions and 111 deletions.
diff --git a/packages/citrix_adc/changelog.yml b/packages/citrix_adc/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.12.0"
+  changes:
+    - description: "Support parsing additional sslvpn log messages"
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11781
 - version: "1.11.0"
   changes:
     - description: "Improve timestamp parsing"

diff --git a/..._adc/data_stream/log/_dev/test/pipeline/test-citrix-native-with-delink.json-expected.json b/..._adc/data_stream/log/_dev/test/pipeline/test-citrix-native-with-delink.json-expected.json
@@ -26,6 +26,7 @@
                     "network"
                 ],
                 "id": "6715345",
+                "kind": "event",
                 "original": "<123> 10/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default TCP CONN_TERMINATE 6715345 0 :  Source 127.1.2.1:80 - Destination 127.1.1.2:20714 - Start Time 10/08/2024:09:37:54  - End Time 10/08/2024:09:38:41  - Total_bytes_send 1 - Total_bytes_recv 1 \n",
                 "severity": 0,
                 "timezone": "UTC",
@@ -35,6 +36,7 @@
                 ]
             },
             "observer": {
+                "hostname": "SYSLOGHOST",
                 "product": "Netscaler",
                 "type": "firewall",
                 "vendor": "Citrix"
@@ -95,6 +97,7 @@
                 ],
                 "end": "2024-08-10T09:38:41.000Z",
                 "id": "6715345",
+                "kind": "event",
                 "original": "<131> 10/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default TCP CONN_DELINK 6715345 0 :  Source 127.1.2.1:80 - Vserver 81.2.69.144:80 - NatIP 192.168.10.10:52187 - Destination 81.2.69.144:80 - Delink Time 10/08/2024:09:38:41  - Total_bytes_send 0 - Total_bytes_recv 3118\n",
                 "severity": 0,
                 "timezone": "UTC",
@@ -104,6 +107,7 @@
                 ]
             },
             "observer": {
+                "hostname": "SYSLOGHOST",
                 "product": "Netscaler",
                 "type": "firewall",
                 "vendor": "Citrix"
@@ -159,6 +163,7 @@
                     "network"
                 ],
                 "id": "6715345",
+                "kind": "event",
                 "original": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default TCP CONN_TERMINATE 6715345 0 :  Source 127.1.2.1:80 - Destination 127.1.1.2:20714 - Start Time 21/08/2024:09:37:54  - End Time 21/08/2024:09:38:41  - Total_bytes_send 1 - Total_bytes_recv 1 \n",
                 "severity": 0,
                 "timezone": "UTC",
@@ -168,6 +173,7 @@
                 ]
             },
             "observer": {
+                "hostname": "SYSLOGHOST",
                 "product": "Netscaler",
                 "type": "firewall",
                 "vendor": "Citrix"
@@ -228,6 +234,7 @@
                 ],
                 "end": "2024-08-21T09:38:41.000Z",
                 "id": "6715345",
+                "kind": "event",
                 "original": "<131> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default TCP CONN_DELINK 6715345 0 :  Source 127.1.2.1:80 - Vserver 81.2.69.144:80 - NatIP 192.168.10.10:52187 - Destination 81.2.69.144:80 - Delink Time 21/08/2024:09:38:41  - Total_bytes_send 0 - Total_bytes_recv 3118\n",
                 "severity": 0,
                 "timezone": "UTC",
@@ -237,6 +244,7 @@
                 ]
             },
             "observer": {
+                "hostname": "SYSLOGHOST",
                 "product": "Netscaler",
                 "type": "firewall",
                 "vendor": "Citrix"
@@ -267,4 +275,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json
@@ -12,73 +12,73 @@
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default TCP CONN_TERMINATE 6715345 0 :  Source 127.1.2.1:80 - Destination 127.1.1.2:20714 - Start Time 21/08/2024:09:37:54  - End Time 21/08/2024:09:38:41  - Total_bytes_send 1 - Total_bytes_recv 1 \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN Message 600000 0 : Logout handler : starting 30sec timer after sending saml logout req to IdP, for user user_name \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN TCPCONNSTAT 600000 0 : Context user_name@domain.com@0.0.0.0 - SessionId: 343368 - User user_name - Client_ip 0.0.0.0 - Nat_ip 0.0.0.0 - Vserver 0.0.0.0:443 - Source 0.0.0.0:51607 - Destination 0.0.0.0:443 - Start_time \"11/06/2024:08:32:59\" - End_time \"11/06/2024:08:33:03\" - Duration 00:00:04  - Total_bytes_send 0 - Total_bytes_recv 378 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - Access Allowed - Group(s) \"N/A\" \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN HTTPREQUEST 600000 0 : Context user_name@domain.com@0.0.0.0 - SessionId: 342014 - subdomain.domain.com User user_name : Group(s) N/A : Vserver 0.0.0.0:443 - 11/06/2024:08:33:03 : SSO is ON : POST /Citrix/EXTFASWeb/Resources/GetLaunchStatus/QDQ2mj0ij09NPOAKJPOJl-- - - \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN ICASTART 600000 0 : [TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 0.0.0.0:62480 - Destination 0.0.0.0:2598 - customername  - username:domainname username:domain - applicationName Developer Europe $P14189 - startTime \"11/06/2024:08:32:58\" - connectionId 16879892 \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN ICAEND_CONNSTAT 600000 0 : [TCP] [CGP][ICAUUID=000bb24f-efd3-172a-9678-000d3ac7ec06] Source 0.0.0.0:51547 - Destination 0.0.0.0:2598 - customername  - username:domainname username:domain - startTime \"11/06/2024:04:25:54\" - endTime \"11/06/2024:08:33:02\" - Duration 04:07:08  - Total_bytes_send 109566281 - Total_bytes_recv 32996419 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 1057494 \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN LOGIN 600000 0 : Logout handler : Context user_name@domain.com@0.0.0.0 - SessionId: 343368 - User user_name - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Browser_type \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36\" - SSLVPN_client_type ICA - Group(s) \"N/A\" \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN LOGOUT 600000 0 : Logout handler : Context user_name@domain.com@0.0.0.0 - SessionId: 17790 - User user_name - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/06/2024:07:49:17\" - End_time \"11/06/2024:08:32:57\" - Duration 00:43:40  - Http_resources_accessed 0 - NonHttp_services_accessed 0 - Total_TCP_connections 176 - Total_UDP_flows 0 - Total_policies_allowed 175 - Total_policies_denied 0 - Total_bytes_send 804 - Total_bytes_recv 3079180 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"TimedOut\" - Group(s) \"N/A\" \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN REMOVE_SESSION_DEBUG 600000 0 : Sessionid 13707 - User user_name - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver_ip 0.0.0.0 - Errmsg \"\" \n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN Message 600000 0 : Logout handler : starting 30sec timer after sending saml logout req to IdP, for user <user_name@domain.com>\n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN TCPCONNSTAT 600000 0 : Context user_name@acme.com@0.0.0.0 - SessionId: 346153 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip 0.0.0.0 - Vserver 0.0.0.0:443 - Source 0.0.0.0:4595 - Destination 0.0.0.0:443 - Start_time \"11/08/2024:15:01:39\" - End_time \"11/08/2024:15:01:39\" - Duration 00:00:00  - Total_bytes_send 0 - Total_bytes_recv 417 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - Access Allowed - Group(s) \"N/A\"\n"
         },
-    		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN LOGIN 600000 0 : Context user_name@acme.com@0.0.0.0 - SessionId: 346153 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Browser_type \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/0.0.0.0 Safari/537.36\" - SSLVPN_client_type ICA - Group(s) \"N/A\"\n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN LOGOUT 600000 0 : Context user_name@acme.com@0.0.0.0 - SessionId: 352037 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/08/2024:11:14:19\" - End_time \"11/08/2024:11:46:32\" - Duration 00:32:13  - Http_resources_accessed 0 - NonHttp_services_accessed 0 - Total_TCP_connections 67 - Total_UDP_flows 0 - Total_policies_allowed 67 - Total_policies_denied 0 - Total_bytes_send 0 - Total_bytes_recv 1529833 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"TimedOut\" - Group(s) \"N/A\"\n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
             "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN LOGOUT 600000 0 : Context user_name@acme.com@0.0.0.0 - SessionId: 351869 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/08/2024:10:37:13\" - End_time \"11/08/2024:10:43:39\" - Duration 00:06:26  - Http_resources_accessed 7 - NonHttp_services_accessed 0 - Total_TCP_connections 14 - Total_UDP_flows 0 - Total_policies_allowed 14 - Total_policies_denied 0 - Total_bytes_send 0 - Total_bytes_recv 86130 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"Explicit\" - Group(s) \"N/A\"\n"
         },
-		{
+        {
             "@timestamp": "2024-08-21T09:38:41.000Z",
-            "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN CHANGEME 600000 0 : \n"
+            "message": "<134> 21/08/2024:13:25:41  SYSLOGHOST 0-PPE-1 : default SSLVPN ICAEND_CONNSTAT 600000 0 : [TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 67.43.156.1:50385 - Destination 10.0.10.75:2598 - customername  - username:domainname user_name:domain_name - startTime \"11/08/2024:10:37:13 \" - endTime \"11/08/2024:10:43:39 \" - Duration 00:06:26 - Total_bytes_send 8379078 - Total_bytes_recv 2761789 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 20459456 \n"
         },
-		{
-            "@timestamp": "2024-08-21T09:38:41.000Z",
-            "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN CHANGEME 600000 0 : \n"
+        {
+            "@timestamp": "2024-11-18T12:18:56.000Z",
+            "message": "<134> 11/18/2024:12:18:56 GMT CITRIX-HOST 0-PPE-0 : default AAATM Message 7488570 0 : \"SAML: successfully verified digest and signature on saml:Response\"\n"
         },
-		{
-            "@timestamp": "2024-08-21T09:38:41.000Z",
-            "message": "<123> 21/08/2024:09:38:41  SYSLOGHOST 0-PPE-1 : default SSLVPN CHANGEME 600000 0 : \n"
+        {
+            "@timestamp": "2024-11-18T12:18:56.000Z",
+            "message": "<134> 11/18/2024:12:18:56 GMT CITRIX-HOST 0-PPE-0 : default AAATM Message 7488573 0 : \"aaatm_handler successfully parsed assertion client ip is fbe2s, username is UserName@domain.com\"\n"
         },
         {
-          "@timestamp": "2024-08-21T09:38:41.000Z",
-          "message": "<134> 21/08/2024:13:25:41  SYSLOGHOST 0-PPE-1 : default SSLVPN ICAEND_CONNSTAT 600000 0 : [TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 67.43.156.1:50385 - Destination 10.0.10.75:2598 - customername  - username:domainname user_name:domain_name - startTime \"11/08/2024:10:37:13 \" - endTime \"11/08/2024:10:43:39 \" - Duration 00:06:26 - Total_bytes_send 8379078 - Total_bytes_recv 2761789 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 20459456 \n"
+            "@timestamp": "2024-11-18T10:59:53.000Z",
+            "message": "<134> 11/18/2024:10:59:53 GMT HOSTNAME 0-PPE-2 : default SSLVPN LOGOUT 6918043 0 : User UserName@domain.com - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/18/2024:10:59:53 GMT\" - End_time \"11/18/2024:10:59:53 GMT\" - Duration 00:00:00 - Http_resources_accessed 0 - NonHttp_services_accessed 0 - Total_TCP_connections 0 - Total_UDP_flows 0 - Total_policies_allowed 0 - Total_policies_denied 0 - Total_bytes_send 0 - Total_bytes_recv 0 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"InternalError\" - Group(s) \"N/A\"\n"
         }
-	]
+    ]
 }