L7 Visibility support in Antrea

Signed-off-by: Tushar Tathgur <tathgurt@tathgurtFLVDL.vmware.com>

build/charts/antrea/conf/antrea-agent.conf

@@ -73,6 +73,9 @@ featureGates:
 # into account application context.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "L7NetworkPolicy" "default" false) }}
 
+# Enable L7Visibility on Pods and Namespace.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "L7Visibility" "default" false) }}
+
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 ovsBridge: {{ .Values.ovs.bridgeName | quote }}

build/charts/antrea/templates/agent/clusterrole.yaml

@@ -173,7 +173,6 @@ rules:
     resources:
       - externalippools
       - ippools
-      - trafficcontrols
     verbs:
       - get
       - watch
@@ -219,3 +218,15 @@ rules:
     - get
     - list
     - watch
+  - apiGroups:
+      - crd.antrea.io
+    resources:
+      - trafficcontrols
+    verbs:
+      - get
+      - watch
+      - list
+      - update
+      - patch
+      - create
+      - delete

build/charts/antrea/values.yaml

@@ -306,7 +306,7 @@ controller:
 
 flowExporter:
   # -- Enable the flow exporter feature.
-  enable: false
+  enable: true
   # -- IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
   # If the collector is running in-cluster as a Service, set <HOST> to
   # <Service namespace>/<Service name>.

build/charts/flow-aggregator/values.yaml

@@ -44,7 +44,7 @@ flowCollector:
 # clickHouse contains ClickHouse related configuration options.
 clickHouse:
   # -- Determine whether to enable exporting flow records to ClickHouse.
-  enable: false
+  enable: true
   # -- DatabaseURL is the url to the database. TCP protocol is required.
   databaseURL: "tcp://clickhouse-clickhouse.flow-visibility.svc:9000"
   # -- Debug enables debug logs from ClickHouse sql driver.

build/yamls/antrea-aks.yml

@@ -3282,7 +3282,7 @@ data:
       # IPFIX flow records from each agent to a configured collector. To enable this
       # feature, you need to set "enable" to true, and ensure that the FlowExporter
       # feature gate is also enabled.
-      enable: false
+      enable: true
       # Provide the IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
       # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If
       # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be

build/yamls/antrea-eks.yml

@@ -3282,7 +3282,7 @@ data:
       # IPFIX flow records from each agent to a configured collector. To enable this
       # feature, you need to set "enable" to true, and ensure that the FlowExporter
       # feature gate is also enabled.
-      enable: false
+      enable: true
       # Provide the IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
       # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If
       # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be

build/yamls/antrea-gke.yml

@@ -3282,7 +3282,7 @@ data:
       # IPFIX flow records from each agent to a configured collector. To enable this
       # feature, you need to set "enable" to true, and ensure that the FlowExporter
       # feature gate is also enabled.
-      enable: false
+      enable: true
       # Provide the IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
       # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If
       # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be

build/yamls/antrea-ipsec.yml

@@ -3295,7 +3295,7 @@ data:
       # IPFIX flow records from each agent to a configured collector. To enable this
       # feature, you need to set "enable" to true, and ensure that the FlowExporter
       # feature gate is also enabled.
-      enable: false
+      enable: true
       # Provide the IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
       # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If
       # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be

build/yamls/antrea.yml

@@ -3282,7 +3282,7 @@ data:
       # IPFIX flow records from each agent to a configured collector. To enable this
       # feature, you need to set "enable" to true, and ensure that the FlowExporter
       # feature gate is also enabled.
-      enable: false
+      enable: true
       # Provide the IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
       # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If
       # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be

build/yamls/flow-aggregator.yml

@@ -219,7 +219,7 @@ data:
     # clickHouse contains ClickHouse related configuration options.
     clickHouse:
       # Enable is the switch to enable exporting flow records to ClickHouse.
-      enable: false
+      enable: true
 
       # Database is the name of database where Antrea "flows" table is created.
       database: "default"

cmd/antrea-agent/agent.go

@@ -39,6 +39,7 @@ import (
 	"antrea.io/antrea/pkg/agent/controller/egress"
 	"antrea.io/antrea/pkg/agent/controller/ipseccertificate"
 	"antrea.io/antrea/pkg/agent/controller/networkpolicy"
+	"antrea.io/antrea/pkg/agent/controller/networkpolicy/l7engine"
 	"antrea.io/antrea/pkg/agent/controller/noderoute"
 	"antrea.io/antrea/pkg/agent/controller/serviceexternalip"
 	"antrea.io/antrea/pkg/agent/controller/traceflow"
@@ -690,6 +691,7 @@ func run(o *Options) error {
 
 	if features.DefaultFeatureGate.Enabled(features.TrafficControl) {
 		tcController := trafficcontrol.NewTrafficControlController(ofClient,
+			crdClient,
 			ifaceStore,
 			ovsBridgeClient,
 			ovsCtlClient,
@@ -698,6 +700,9 @@ func run(o *Options) error {
 			namespaceInformer,
 			podUpdateChannel)
 		go tcController.Run(stopCh)
+		if features.DefaultFeatureGate.Enabled(features.L7Visibility) {
+			go l7engine.Run(tcController)
+		}
 	}
 
 	//  Start the localPodInformer

go.mod

@@ -233,3 +233,5 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
+
+replace github.com/vmware/go-ipfix v0.6.1 => github.com/tushartathgur/go-ipfix v0.0.0-20230714214305-c3c7a13353f8

go.sum

@@ -1083,6 +1083,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
+github.com/tushartathgur/go-ipfix v0.0.0-20230714214305-c3c7a13353f8 h1:5f0hm5NzcSY4tKiKInkCGLnps+DDWEVK5qV9wYnE9/g=
+github.com/tushartathgur/go-ipfix v0.0.0-20230714214305-c3c7a13353f8/go.mod h1:dGCppoeqknr9o3yz9BD74mP/FPHgefb6v34xdUKxDPI=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=

pkg/agent/controller/networkpolicy/l7engine/reconciler.go

@@ -228,6 +228,7 @@ func (r *Reconciler) AddRule(ruleID, policyName string, vlanID uint32, l7Protoco
 	// Add a Suricata tenant.
 	if err := r.addBindingSuricataTenant(vlanID, rulesPath); err != nil {
 		return fmt.Errorf("failed to add Suricata tenant for L7 rule %s of %s: %w", ruleID, policyName, err)
+
 	}
 
 	return nil
@@ -386,7 +387,7 @@ func (r *Reconciler) unregisterSuricataTenantHandler(tenantID, vlanID uint32) (*
 	return r.suricataScFn(scCmd)
 }
 
-func (r *Reconciler) startSuricata() {
+func (r *Reconciler)  startSuricata() {
 	// Create the config file /etc/suricata/antrea.yaml for Antrea which will be included in the default Suricata config file
 	// /etc/suricata/suricata.yaml.
 	suricataAntreaConfigData := fmt.Sprintf(`%%YAML 1.1
@@ -405,7 +406,28 @@ outputs:
       types:
         - alert:
             tagged-packets: yes
+  - eve-log:
+      enabled: yes
+      filetype: unix_stream
+      filename: /var/log/antrea/networkpolicy/suricata_eve.socket
+      rotate-interval: day
+      pcap-file: false
+      community-id: false
+      community-id-seed: 0
+      xff:
+        enabled: no
+      types:
+        - http:
+            extended: yes
 af-packet:
+  - interface: eth0
+    threads: auto
+    cluster-id: 93
+    cluster-type: cluster_flow
+    defrag: no
+    use-mmap: yes
+    tpacket-v2: yes
+    checksum-checks: no
   - interface: %[1]s
     threads: auto
     cluster-id: 80

pkg/agent/controller/trafficcontrol/controller.go

@@ -42,6 +42,7 @@ import (
 	"antrea.io/antrea/pkg/agent/types"
 	"antrea.io/antrea/pkg/agent/util"
 	"antrea.io/antrea/pkg/apis/crd/v1alpha2"
+	clientsetversioned "antrea.io/antrea/pkg/client/clientset/versioned"
 	crdinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha2"
 	crdlisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha2"
 	"antrea.io/antrea/pkg/ovs/ovsconfig"
@@ -112,7 +113,8 @@ type portToTCBinding struct {
 }
 
 type Controller struct {
-	ofClient openflow.Client
+	ofClient  openflow.Client
+	CrdClient clientsetversioned.Interface
 
 	portToTCBindings   map[string]*portToTCBinding
 	ovsBridgeClient    ovsconfig.OVSBridgeClient
@@ -142,6 +144,7 @@ type Controller struct {
 }
 
 func NewTrafficControlController(ofClient openflow.Client,
+	crdClient clientsetversioned.Interface,
 	interfaceStore interfacestore.InterfaceStore,
 	ovsBridgeClient ovsconfig.OVSBridgeClient,
 	ovsCtlClient ovsctl.OVSCtlClient,
@@ -151,6 +154,7 @@ func NewTrafficControlController(ofClient openflow.Client,
 	podUpdateSubscriber channel.Subscriber) *Controller {
 	c := &Controller{
 		ofClient:                   ofClient,
+		CrdClient:                  crdClient,
 		ovsBridgeClient:            ovsBridgeClient,
 		ovsCtlClient:               ovsCtlClient,
 		interfaceStore:             interfaceStore,

pkg/agent/flowexporter/connections/conntrack_ovs.go

@@ -32,13 +32,6 @@ import (
 
 // Following map is for converting protocol name (string) to protocol identifier
 var (
-	protocols = map[string]uint8{
-		"icmp":      1,
-		"igmp":      2,
-		"tcp":       6,
-		"udp":       17,
-		"ipv6-icmp": 58,
-	}
 	// Mapping is defined at https://github.com/torvalds/linux/blob/v5.9/include/uapi/linux/netfilter/nf_conntrack_common.h#L42
 	conntrackStatusMap = map[string]uint32{
 		"EXPECTED":      uint32(1),
@@ -140,7 +133,7 @@ func flowStringToAntreaConnection(flow string, zoneFilter uint16) (*flowexporter
 		switch {
 		case hasAnyProto(fs):
 			// Proto identifier
-			proto, err := lookupProtocolMap(fs)
+			proto, err := flowexporter.LookupProtocolMap(fs)
 			if err != nil {
 				return nil, err
 			}
@@ -287,25 +280,14 @@ func flowStringToAntreaConnection(flow string, zoneFilter uint16) (*flowexporter
 }
 
 func hasAnyProto(text string) bool {
-	for proto := range protocols {
+	for proto := range flowexporter.Protocols {
 		if strings.Contains(strings.ToLower(text), proto) {
 			return true
 		}
 	}
 	return false
 }
 
-// lookupProtocolMap returns protocol identifier given protocol name
-func lookupProtocolMap(name string) (uint8, error) {
-	name = strings.TrimSpace(name)
-	lowerCaseStr := strings.ToLower(name)
-	proto, found := protocols[lowerCaseStr]
-	if !found {
-		return 0, fmt.Errorf("unknown IP protocol specified: %s", name)
-	}
-	return proto, nil
-}
-
 func (ct *connTrackOvsCtl) GetMaxConnections() (int, error) {
 	cmdOutput, execErr := ct.ovsctlClient.RunAppctlCmd("dpctl/ct-get-maxconns", false)
 	if execErr != nil {