From 56e7b5b84d271b3dea31e6e1e36e8610208e508f Mon Sep 17 00:00:00 2001 From: Ramachandran A G Date: Tue, 2 Apr 2024 19:07:13 +0530 Subject: [PATCH] *Changes to use CEF config --- docker-e2e/Logstash-Docker | 1 + docker-e2e/fortigate.pattern | 1 + docker-e2e/fortinet-fortigate.log | 8 + .../logstash-fortigate-paloalto.conf.template | 142 +++++++++++------- lib/logstash/outputs/kusto/ingestor.rb | 1 + 5 files changed, 99 insertions(+), 54 deletions(-) create mode 100644 docker-e2e/fortigate.pattern create mode 100644 docker-e2e/fortinet-fortigate.log diff --git a/docker-e2e/Logstash-Docker b/docker-e2e/Logstash-Docker index dcd75a8..d899087 100644 --- a/docker-e2e/Logstash-Docker +++ b/docker-e2e/Logstash-Docker @@ -4,5 +4,6 @@ RUN rm -f /usr/share/logstash/pipeline/logstash.conf && \ bin/logstash-plugin install /tmp/logstash-output-kusto-2.0.5-java.gem #-e2eCOPY logstash-nsg-logs.conf /usr/share/logstash/pipeline/logstash.conf COPY logstash-fortigate-paloalto.conf.template /usr/share/logstash/pipeline/logstash.conf +COPY fortigate.pattern /etc/logstash/patterns.d/fortigate.pattern COPY *.log /tmp/ COPY logstash.yml /usr/share/logstash/config/logstash.yml diff --git a/docker-e2e/fortigate.pattern b/docker-e2e/fortigate.pattern new file mode 100644 index 0000000..e8912ca --- /dev/null +++ b/docker-e2e/fortigate.pattern @@ -0,0 +1 @@ +FORTIDATE %{YEAR:year}\-%{MONTHNUM:month}\-%{MONTHDAY:day} \ No newline at end of file diff --git a/docker-e2e/fortinet-fortigate.log b/docker-e2e/fortinet-fortigate.log new file mode 100644 index 0000000..cffec16 --- /dev/null +++ b/docker-e2e/fortinet-fortigate.log @@ -0,0 +1,8 @@ +<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0 +<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0 +<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0 +<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0 +<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0 +<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0 +<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0 +<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0 \ No newline at end of file diff --git a/docker-e2e/logstash-fortigate-paloalto.conf.template b/docker-e2e/logstash-fortigate-paloalto.conf.template index c255778..56e1218 100644 --- a/docker-e2e/logstash-fortigate-paloalto.conf.template +++ b/docker-e2e/logstash-fortigate-paloalto.conf.template @@ -1,70 +1,104 @@ input { stdin {} - file { + #file { # Took the file that you provided as the sample and sent that data into ADX - add_field => { "[@metadata][source_type]" => "file" } - path => "/tmp/palo-alto-2.log" + # add_field => { "[@metadata][source_type]" => "file" } + # path => "/tmp/palo-alto-2.log" + # start_position => "beginning" + # tags => "paloalto" + #} + file { + #Took the file that you provided as the sample and sent that data into ADX + add_field => { "[@metadata][source_type]" => "cef-data" } + path => "/tmp/2023-12-15-12-fw-d-hub01.log" start_position => "beginning" - } + codec => cef { + ecs_compatibility => v1 + } + tags => "cef-data" + } } filter { - csv { - source => "message" - columns => [ - "FUTURE_USE_1","RECEIVE_TIME","SERIAL_NUMBER","TYPE","THREAT_CONTENT_TYPE","FUTURE_USE_2","GENERATED_TIME","SOURCE_ADDRESS","DESTINATION_ADDRESS","NAT_SOURCE_IP","NAT_DESTINATION_IP","RULE_NAME"," - SOURCE_USER","DESTINATION_USER","APPLICATION","VIRTUAL_SYSTEM","SOURCE_ZONE","DESTINATION_ZONE","INBOUND_INTERFACE","OUTBOUND_INTERFACE","LOG_ACTION","FUTURE_USE_3","SESSION_ID","REPEAT_COUNT"," - SOURCE_PORT","DESTINATION_PORT","NAT_SOURCE_PORT","NAT_DESTINATION_PORT","FLAGS","PROTOCOL","ACTION","BYTES","BYTES_SENT","BYTES_RECEIVED","PACKETS","START_TIME","ELAPSED_TIME","CATEGORY","FUTURE_USE_4", - "SEQUENCE_NUMBER","ACTION_FLAGS","SOURCE_COUNTRY","DESTINATION_COUNTRY","FUTURE_USE_5","PACKETS_SENT","PACKETS_RECEIVED","SESSION_END_REASON","DEVICE_GROUP_HIERARCHY_LEVEL_1"," - DEVICE_GROUP_HIERARCHY_LEVEL_2","DEVICE_GROUP_HIERARCHY_LEVEL_3","DEVICE_GROUP_HIERARCHY_LEVEL_4","VIRTUAL_SYSTEM_NAME","DEVICE_NAME","ACTION_SOURCE","SOURCE_VM_UUID"," - DESTINATION_VM_UUID","TUNNEL_ID_IMSI","MONITOR_TAG_IMEI","PARENT_SESSION_ID","PARENT_START_TIME","TUNNEL_TYPE","SCTP_ASSOCIATION_ID","SCTP_CHUNKS","SCTP_CHUNKS_SENT"," - SCTP_CHUNKS_RECEIVED","RULE_UUID","HTTP_2_CONNECTION","APP_FLAP_COUNT","POLICY_ID","LINK_SWITCHES","SD_WAN_CLUSTER","SD_WAN_DEVICE_TYPE","SD_WAN_CLUSTER_TYPE","SD_WAN_SITE"," - DYNAMIC_USER_GROUP_NAME","XFF_ADDRESS","SOURCE_DEVICE_CATEGORY","SOURCE_DEVICE_PROFILE","SOURCE_DEVICE_MODEL","SOURCE_DEVICE_VENDOR","SOURCE_DEVICE_OS_FAMILY"," - SOURCE_DEVICE_OS_VERSION","SOURCE_HOSTNAME","SOURCE_MAC_ADDRESS","DESTINATION_DEVICE_CATEGORY","DESTINATION_DEVICE_PROFILE","DESTINATION_DEVICE_MODEL"," - DESTINATION_DEVICE_VENDOR","DESTINATION_DEVICE_OS_FAMILY","DESTINATION_DEVICE_OS_VERSION","DESTINATION_HOSTNAME","DESTINATION_MAC_ADDRESS","CONTAINER_ID"," - POD_NAMESPACE","POD_NAME","SOURCE_EXTERNAL_DYNAMIC_LIST","DESTINATION_EXTERNAL_DYNAMIC_LIST","HOST_ID","USER_SERIAL_NUMBER","SOURCE_DYNAMIC_ADDRESS_GROUP"," - DESTINATION_DYNAMIC_ADDRESS_GROUP","SESSION_OWNER","HIGH_RESOLUTION_TIMESTAMP","A_SLICE_SERVICE_TYPE","A_SLICE_DIFFERENTIATOR","APPLICATION_SUBCATEGORY","APPLICATION_CATEGORY"," - APPLICATION_TECHNOLOGY","APPLICATION_RISK","APPLICATION_CHARACTERISTIC","APPLICATION_CONTAINER","TUNNELED_APPLICATION","APPLICATION_SAAS","APPLICATION_SANCTIONED_STATE","OFFLOADED" - ] - } - - - date { - timezone => "GMT" - match => [ "ReceiveTime", "YYYY_MM_dd HH:mm:ss" ] - } + if "paloalto" in [tags] { + csv { + source => "message" + columns => [ + "FUTURE_USE_1","RECEIVE_TIME","SERIAL_NUMBER","TYPE","THREAT_CONTENT_TYPE","FUTURE_USE_2","GENERATED_TIME","SOURCE_ADDRESS","DESTINATION_ADDRESS","NAT_SOURCE_IP","NAT_DESTINATION_IP","RULE_NAME", + "SOURCE_USER","DESTINATION_USER","APPLICATION","VIRTUAL_SYSTEM","SOURCE_ZONE","DESTINATION_ZONE","INBOUND_INTERFACE","OUTBOUND_INTERFACE","LOG_ACTION","FUTURE_USE_3","SESSION_ID","REPEAT_COUNT", + "SOURCE_PORT","DESTINATION_PORT","NAT_SOURCE_PORT","NAT_DESTINATION_PORT","FLAGS","PROTOCOL","ACTION","BYTES","BYTES_SENT","BYTES_RECEIVED","PACKETS","START_TIME","ELAPSED_TIME","CATEGORY","FUTURE_USE_4", + "SEQUENCE_NUMBER","ACTION_FLAGS","SOURCE_COUNTRY","DESTINATION_COUNTRY","FUTURE_USE_5","PACKETS_SENT","PACKETS_RECEIVED","SESSION_END_REASON","DEVICE_GROUP_HIERARCHY_LEVEL_1", + "DEVICE_GROUP_HIERARCHY_LEVEL_2","DEVICE_GROUP_HIERARCHY_LEVEL_3","DEVICE_GROUP_HIERARCHY_LEVEL_4","VIRTUAL_SYSTEM_NAME","DEVICE_NAME","ACTION_SOURCE","SOURCE_VM_UUID", + "DESTINATION_VM_UUID","TUNNEL_ID_IMSI","MONITOR_TAG_IMEI","PARENT_SESSION_ID","PARENT_START_TIME","TUNNEL_TYPE","SCTP_ASSOCIATION_ID","SCTP_CHUNKS","SCTP_CHUNKS_SENT", + "SCTP_CHUNKS_RECEIVED","RULE_UUID","HTTP_2_CONNECTION","APP_FLAP_COUNT","POLICY_ID","LINK_SWITCHES","SD_WAN_CLUSTER","SD_WAN_DEVICE_TYPE","SD_WAN_CLUSTER_TYPE","SD_WAN_SITE", + "DYNAMIC_USER_GROUP_NAME","XFF_ADDRESS","SOURCE_DEVICE_CATEGORY","SOURCE_DEVICE_PROFILE","SOURCE_DEVICE_MODEL","SOURCE_DEVICE_VENDOR","SOURCE_DEVICE_OS_FAMILY", + "SOURCE_DEVICE_OS_VERSION","SOURCE_HOSTNAME","SOURCE_MAC_ADDRESS","DESTINATION_DEVICE_CATEGORY","DESTINATION_DEVICE_PROFILE","DESTINATION_DEVICE_MODEL", + "DESTINATION_DEVICE_VENDOR","DESTINATION_DEVICE_OS_FAMILY","DESTINATION_DEVICE_OS_VERSION","DESTINATION_HOSTNAME","DESTINATION_MAC_ADDRESS","CONTAINER_ID", + "POD_NAMESPACE","POD_NAME","SOURCE_EXTERNAL_DYNAMIC_LIST","DESTINATION_EXTERNAL_DYNAMIC_LIST","HOST_ID","USER_SERIAL_NUMBER","SOURCE_DYNAMIC_ADDRESS_GROUP", + "DESTINATION_DYNAMIC_ADDRESS_GROUP","SESSION_OWNER","HIGH_RESOLUTION_TIMESTAMP","A_SLICE_SERVICE_TYPE","A_SLICE_DIFFERENTIATOR","APPLICATION_SUBCATEGORY","APPLICATION_CATEGORY", + "APPLICATION_TECHNOLOGY","APPLICATION_RISK","APPLICATION_CHARACTERISTIC","APPLICATION_CONTAINER","TUNNELED_APPLICATION","APPLICATION_SAAS","APPLICATION_SANCTIONED_STATE","OFFLOADED" + ] + } - mutate { - convert => [ "NAT_DESTINATION_PORT", "integer" ] - convert => [ "NAT_SOURCE_PORT", "integer" ] - convert => [ "DESTINATION_PORT", "integer" ] - convert => [ "SOURCE_PORT", "integer" ] - convert => [ "SEQUENCE_NUMBER", "integer" ] - remove_field => [ "message", "host", "path", "original","event"] - } + date { + timezone => "GMT" + match => [ "ReceiveTime", "YYYY_MM_dd HH:mm:ss" ] + } - ruby { - code => " - hash = event.to_hash - hash.each do |field,value| - if value == nil - event.remove(field) + mutate { + convert => [ "NAT_DESTINATION_PORT", "integer" ] + convert => [ "NAT_SOURCE_PORT", "integer" ] + convert => [ "DESTINATION_PORT", "integer" ] + convert => [ "SOURCE_PORT", "integer" ] + convert => [ "SEQUENCE_NUMBER", "integer" ] + remove_field => [ "message", "host", "path", "original","event"] + } + ruby { + code => " + hash = event.to_hash + hash.each do |field,value| + if value == nil + event.remove(field) + end end - end - " + " + } + } + + if "cef-data" in [tags] { + json { + source => "message" + } } } output { - #stdout {} - kusto { - path => "/tmp/kusto/paloalto-traffic/%{+YYYY-MM-dd-HH-mm}.txt" - ingest_url => "${INGEST_CLUSTER_URL}" - app_id => "${APP_ID}" - app_key => "${APP_KEY}" - app_tenant => "${APP_TENANT}" - database => "${DATABASE}" - table => "PaloAltoTrafficLogs" # fw as defined above - } + if "cef-data" in [tags] { + stdout {codec => json_lines} + } + if "cef-data" in [tags] { + kusto { + codec => json_lines + path => "/tmp/kusto/paloalto-traffic/%{+YYYY-MM-dd-HH-mm}.txt" + ingest_url => "${INGEST_CLUSTER_URL}" + app_id => "${APP_ID}" + app_key => "${APP_KEY}" + app_tenant => "${APP_TENANT}" + database => "${DATABASE}" + table => "RawPaloAltoTrafficLogs" # fw as defined above + } + } + if "paloalto" in [tags] { + kusto { + codec => json_lines + path => "/tmp/kusto/paloalto-traffic/%{+YYYY-MM-dd-HH-mm}.txt" + ingest_url => "${INGEST_CLUSTER_URL}" + app_id => "${APP_ID}" + app_key => "${APP_KEY}" + app_tenant => "${APP_TENANT}" + database => "${DATABASE}" + table => "PaloAltoTrafficLogs" # fw as defined above + } + } } \ No newline at end of file diff --git a/lib/logstash/outputs/kusto/ingestor.rb b/lib/logstash/outputs/kusto/ingestor.rb index 3a0b4a5..e590281 100755 --- a/lib/logstash/outputs/kusto/ingestor.rb +++ b/lib/logstash/outputs/kusto/ingestor.rb @@ -74,6 +74,7 @@ def initialize(ingest_url, app_id, app_key, app_tenant, managed_identity_id, dat @ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::JSON) @ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON) else + @ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON) @logger.debug('No mapping reference provided. Columns will be mapped by names in the logstash output') end @delete_local = delete_local