Skip to content

Commit

Permalink
*Changes to use CEF config
Browse files Browse the repository at this point in the history
  • Loading branch information
@ag-ramachandran
ag-ramachandran committed Apr 2, 2024
1 parent 9f52fe3 commit 56e7b5b
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 54 deletions.
1 change: 1 addition & 0 deletions docker-e2e/Logstash-Docker
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ RUN rm -f /usr/share/logstash/pipeline/logstash.conf && \
bin/logstash-plugin install /tmp/logstash-output-kusto-2.0.5-java.gem
#-e2eCOPY logstash-nsg-logs.conf /usr/share/logstash/pipeline/logstash.conf
COPY logstash-fortigate-paloalto.conf.template /usr/share/logstash/pipeline/logstash.conf
COPY fortigate.pattern /etc/logstash/patterns.d/fortigate.pattern
COPY *.log /tmp/
COPY logstash.yml /usr/share/logstash/config/logstash.yml
1 change: 1 addition & 0 deletions docker-e2e/fortigate.pattern
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
FORTIDATE %{YEAR:year}\-%{MONTHNUM:month}\-%{MONTHDAY:day}
8 changes: 8 additions & 0 deletions docker-e2e/fortinet-fortigate.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0
<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0
<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0
<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0
<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0
<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0
<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0
<189>logver=702071577 timestamp=1711530039 devname="IT-FWNAME" devid="FG100XXX00000000" vd="root" date=2024-03-27 time=12:00:39 eventtime=1711530039741065160 tz="+0300" logid="0000000013" type="traffic" subtype="forward" level="notice" srcip=10.10.10.10 srcport=51893 srcintf="INF01" srcintfrole="undefined" dstip=10.10.10.11 dstport=53 dstintf="lan" dstintfrole="lan" srccountry="Reserved" dstcountry="Reserved" sessionid=142857785 proto=17 action="accept" policyid=84 policytype="policy" poluuid="ac859df6-aeb8-51ee-77ae-fcd568320f57" policyname="mypolicyname" service="DNS" trandisp="noop" duration=180 sentbyte=64 rcvdbyte=80 sentpkt=1 rcvdpkt=1 vpntype="ipsecvpn" appcat="unscanned" dsthwvendor="Cisco" dstdevtype="Router" masterdstmac="7c:21:0d:1d:f8:52" dstmac="7c:21:0e:1d:c0:51" dstserver=0
142 changes: 88 additions & 54 deletions docker-e2e/logstash-fortigate-paloalto.conf.template
View file
Original file line number Diff line number Diff line change
@@ -1,70 +1,104 @@
input {
stdin {}
file {
#file {
# Took the file that you provided as the sample and sent that data into ADX
add_field => { "[@metadata][source_type]" => "file" }
path => "/tmp/palo-alto-2.log"
# add_field => { "[@metadata][source_type]" => "file" }
# path => "/tmp/palo-alto-2.log"
# start_position => "beginning"
# tags => "paloalto"
#}
file {
#Took the file that you provided as the sample and sent that data into ADX
add_field => { "[@metadata][source_type]" => "cef-data" }
path => "/tmp/2023-12-15-12-fw-d-hub01.log"
start_position => "beginning"
}
codec => cef {
ecs_compatibility => v1
}
tags => "cef-data"
}
}

filter {
csv {
source => "message"
columns => [
"FUTURE_USE_1","RECEIVE_TIME","SERIAL_NUMBER","TYPE","THREAT_CONTENT_TYPE","FUTURE_USE_2","GENERATED_TIME","SOURCE_ADDRESS","DESTINATION_ADDRESS","NAT_SOURCE_IP","NAT_DESTINATION_IP","RULE_NAME","
SOURCE_USER","DESTINATION_USER","APPLICATION","VIRTUAL_SYSTEM","SOURCE_ZONE","DESTINATION_ZONE","INBOUND_INTERFACE","OUTBOUND_INTERFACE","LOG_ACTION","FUTURE_USE_3","SESSION_ID","REPEAT_COUNT","
SOURCE_PORT","DESTINATION_PORT","NAT_SOURCE_PORT","NAT_DESTINATION_PORT","FLAGS","PROTOCOL","ACTION","BYTES","BYTES_SENT","BYTES_RECEIVED","PACKETS","START_TIME","ELAPSED_TIME","CATEGORY","FUTURE_USE_4",
"SEQUENCE_NUMBER","ACTION_FLAGS","SOURCE_COUNTRY","DESTINATION_COUNTRY","FUTURE_USE_5","PACKETS_SENT","PACKETS_RECEIVED","SESSION_END_REASON","DEVICE_GROUP_HIERARCHY_LEVEL_1","
DEVICE_GROUP_HIERARCHY_LEVEL_2","DEVICE_GROUP_HIERARCHY_LEVEL_3","DEVICE_GROUP_HIERARCHY_LEVEL_4","VIRTUAL_SYSTEM_NAME","DEVICE_NAME","ACTION_SOURCE","SOURCE_VM_UUID","
DESTINATION_VM_UUID","TUNNEL_ID_IMSI","MONITOR_TAG_IMEI","PARENT_SESSION_ID","PARENT_START_TIME","TUNNEL_TYPE","SCTP_ASSOCIATION_ID","SCTP_CHUNKS","SCTP_CHUNKS_SENT","
SCTP_CHUNKS_RECEIVED","RULE_UUID","HTTP_2_CONNECTION","APP_FLAP_COUNT","POLICY_ID","LINK_SWITCHES","SD_WAN_CLUSTER","SD_WAN_DEVICE_TYPE","SD_WAN_CLUSTER_TYPE","SD_WAN_SITE","
DYNAMIC_USER_GROUP_NAME","XFF_ADDRESS","SOURCE_DEVICE_CATEGORY","SOURCE_DEVICE_PROFILE","SOURCE_DEVICE_MODEL","SOURCE_DEVICE_VENDOR","SOURCE_DEVICE_OS_FAMILY","
SOURCE_DEVICE_OS_VERSION","SOURCE_HOSTNAME","SOURCE_MAC_ADDRESS","DESTINATION_DEVICE_CATEGORY","DESTINATION_DEVICE_PROFILE","DESTINATION_DEVICE_MODEL","
DESTINATION_DEVICE_VENDOR","DESTINATION_DEVICE_OS_FAMILY","DESTINATION_DEVICE_OS_VERSION","DESTINATION_HOSTNAME","DESTINATION_MAC_ADDRESS","CONTAINER_ID","
POD_NAMESPACE","POD_NAME","SOURCE_EXTERNAL_DYNAMIC_LIST","DESTINATION_EXTERNAL_DYNAMIC_LIST","HOST_ID","USER_SERIAL_NUMBER","SOURCE_DYNAMIC_ADDRESS_GROUP","
DESTINATION_DYNAMIC_ADDRESS_GROUP","SESSION_OWNER","HIGH_RESOLUTION_TIMESTAMP","A_SLICE_SERVICE_TYPE","A_SLICE_DIFFERENTIATOR","APPLICATION_SUBCATEGORY","APPLICATION_CATEGORY","
APPLICATION_TECHNOLOGY","APPLICATION_RISK","APPLICATION_CHARACTERISTIC","APPLICATION_CONTAINER","TUNNELED_APPLICATION","APPLICATION_SAAS","APPLICATION_SANCTIONED_STATE","OFFLOADED"
]
}


date {
timezone => "GMT"
match => [ "ReceiveTime", "YYYY_MM_dd HH:mm:ss" ]
}
if "paloalto" in [tags] {
csv {
source => "message"
columns => [
"FUTURE_USE_1","RECEIVE_TIME","SERIAL_NUMBER","TYPE","THREAT_CONTENT_TYPE","FUTURE_USE_2","GENERATED_TIME","SOURCE_ADDRESS","DESTINATION_ADDRESS","NAT_SOURCE_IP","NAT_DESTINATION_IP","RULE_NAME",
"SOURCE_USER","DESTINATION_USER","APPLICATION","VIRTUAL_SYSTEM","SOURCE_ZONE","DESTINATION_ZONE","INBOUND_INTERFACE","OUTBOUND_INTERFACE","LOG_ACTION","FUTURE_USE_3","SESSION_ID","REPEAT_COUNT",
"SOURCE_PORT","DESTINATION_PORT","NAT_SOURCE_PORT","NAT_DESTINATION_PORT","FLAGS","PROTOCOL","ACTION","BYTES","BYTES_SENT","BYTES_RECEIVED","PACKETS","START_TIME","ELAPSED_TIME","CATEGORY","FUTURE_USE_4",
"SEQUENCE_NUMBER","ACTION_FLAGS","SOURCE_COUNTRY","DESTINATION_COUNTRY","FUTURE_USE_5","PACKETS_SENT","PACKETS_RECEIVED","SESSION_END_REASON","DEVICE_GROUP_HIERARCHY_LEVEL_1",
"DEVICE_GROUP_HIERARCHY_LEVEL_2","DEVICE_GROUP_HIERARCHY_LEVEL_3","DEVICE_GROUP_HIERARCHY_LEVEL_4","VIRTUAL_SYSTEM_NAME","DEVICE_NAME","ACTION_SOURCE","SOURCE_VM_UUID",
"DESTINATION_VM_UUID","TUNNEL_ID_IMSI","MONITOR_TAG_IMEI","PARENT_SESSION_ID","PARENT_START_TIME","TUNNEL_TYPE","SCTP_ASSOCIATION_ID","SCTP_CHUNKS","SCTP_CHUNKS_SENT",
"SCTP_CHUNKS_RECEIVED","RULE_UUID","HTTP_2_CONNECTION","APP_FLAP_COUNT","POLICY_ID","LINK_SWITCHES","SD_WAN_CLUSTER","SD_WAN_DEVICE_TYPE","SD_WAN_CLUSTER_TYPE","SD_WAN_SITE",
"DYNAMIC_USER_GROUP_NAME","XFF_ADDRESS","SOURCE_DEVICE_CATEGORY","SOURCE_DEVICE_PROFILE","SOURCE_DEVICE_MODEL","SOURCE_DEVICE_VENDOR","SOURCE_DEVICE_OS_FAMILY",
"SOURCE_DEVICE_OS_VERSION","SOURCE_HOSTNAME","SOURCE_MAC_ADDRESS","DESTINATION_DEVICE_CATEGORY","DESTINATION_DEVICE_PROFILE","DESTINATION_DEVICE_MODEL",
"DESTINATION_DEVICE_VENDOR","DESTINATION_DEVICE_OS_FAMILY","DESTINATION_DEVICE_OS_VERSION","DESTINATION_HOSTNAME","DESTINATION_MAC_ADDRESS","CONTAINER_ID",
"POD_NAMESPACE","POD_NAME","SOURCE_EXTERNAL_DYNAMIC_LIST","DESTINATION_EXTERNAL_DYNAMIC_LIST","HOST_ID","USER_SERIAL_NUMBER","SOURCE_DYNAMIC_ADDRESS_GROUP",
"DESTINATION_DYNAMIC_ADDRESS_GROUP","SESSION_OWNER","HIGH_RESOLUTION_TIMESTAMP","A_SLICE_SERVICE_TYPE","A_SLICE_DIFFERENTIATOR","APPLICATION_SUBCATEGORY","APPLICATION_CATEGORY",
"APPLICATION_TECHNOLOGY","APPLICATION_RISK","APPLICATION_CHARACTERISTIC","APPLICATION_CONTAINER","TUNNELED_APPLICATION","APPLICATION_SAAS","APPLICATION_SANCTIONED_STATE","OFFLOADED"
]
}

mutate {
convert => [ "NAT_DESTINATION_PORT", "integer" ]
convert => [ "NAT_SOURCE_PORT", "integer" ]
convert => [ "DESTINATION_PORT", "integer" ]
convert => [ "SOURCE_PORT", "integer" ]
convert => [ "SEQUENCE_NUMBER", "integer" ]
remove_field => [ "message", "host", "path", "original","event"]

}
date {
timezone => "GMT"
match => [ "ReceiveTime", "YYYY_MM_dd HH:mm:ss" ]
}

ruby {
code => "
hash = event.to_hash
hash.each do |field,value|
if value == nil
event.remove(field)
mutate {
convert => [ "NAT_DESTINATION_PORT", "integer" ]
convert => [ "NAT_SOURCE_PORT", "integer" ]
convert => [ "DESTINATION_PORT", "integer" ]
convert => [ "SOURCE_PORT", "integer" ]
convert => [ "SEQUENCE_NUMBER", "integer" ]
remove_field => [ "message", "host", "path", "original","event"]
}
ruby {
code => "
hash = event.to_hash
hash.each do |field,value|
if value == nil
event.remove(field)
end
end
end
"
"
}
}

if "cef-data" in [tags] {
json {
source => "message"
}
}
}

output {
#stdout {}
kusto {
path => "/tmp/kusto/paloalto-traffic/%{+YYYY-MM-dd-HH-mm}.txt"
ingest_url => "${INGEST_CLUSTER_URL}"
app_id => "${APP_ID}"
app_key => "${APP_KEY}"
app_tenant => "${APP_TENANT}"
database => "${DATABASE}"
table => "PaloAltoTrafficLogs" # fw as defined above
}
if "cef-data" in [tags] {
stdout {codec => json_lines}
}
if "cef-data" in [tags] {
kusto {
codec => json_lines
path => "/tmp/kusto/paloalto-traffic/%{+YYYY-MM-dd-HH-mm}.txt"
ingest_url => "${INGEST_CLUSTER_URL}"
app_id => "${APP_ID}"
app_key => "${APP_KEY}"
app_tenant => "${APP_TENANT}"
database => "${DATABASE}"
table => "RawPaloAltoTrafficLogs" # fw as defined above
}
}
if "paloalto" in [tags] {
kusto {
codec => json_lines
path => "/tmp/kusto/paloalto-traffic/%{+YYYY-MM-dd-HH-mm}.txt"
ingest_url => "${INGEST_CLUSTER_URL}"
app_id => "${APP_ID}"
app_key => "${APP_KEY}"
app_tenant => "${APP_TENANT}"
database => "${DATABASE}"
table => "PaloAltoTrafficLogs" # fw as defined above
}
}
}
1 change: 1 addition & 0 deletions lib/logstash/outputs/kusto/ingestor.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ def initialize(ingest_url, app_id, app_key, app_tenant, managed_identity_id, dat
@ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::JSON)
@ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON)
else
@ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON)
@logger.debug('No mapping reference provided. Columns will be mapped by names in the logstash output')
end
@delete_local = delete_local
Expand Down

0 comments on commit 56e7b5b

Please sign in to comment.