Add support for Service Traffic Distribution in Antrea Proxy

`ServiceTrafficDistribution` enables Traffic Distribution for Services in Antrea Proxy. This feature allows for more flexible and intelligent routing decisions by considering both topology and non-topology factors. For more details, refer to this [link](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/4444-service-traffic-distribution). Note: To activate this feature in Antrea Proxy, Kubernetes must be version 1.30 or higher, with the `ServiceTrafficDistribution` feature gate (a Kubernetes-specific feature gate) enabled to add Endpoint zone hints in the Kubernetes EndpointSlice controller. Without these prerequisites, the feature may not function as intended in Antrea Proxy. Signed-off-by: Hongliang Liu <lhongliang@vmware.com>
antrea-io · Aug 14, 2024 · b6f6fd0 · b6f6fd0
1 parent 7d8d4b1
commit b6f6fd0
Show file tree

Hide file tree

Showing 12 changed files with 166 additions and 79 deletions.
diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -16,6 +16,10 @@ featureGates:
 # enabled, otherwise this flag will not take effect.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "TopologyAwareHints" "default" true) }}
 
+# Enable ServiceTrafficDistribution in AntreaProxy. This requires AntreaProxy and EndpointSlice to be
+# enabled, otherwise this flag will not take effect.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "ServiceTrafficDistribution" "default" false) }}
+
 # Enable support for cleaning up stale UDP Service conntrack connections in AntreaProxy. This requires AntreaProxy to
 # be enabled, otherwise this flag will not take effect.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "CleanupStaleUDPSvcConntrack" "default" true) }}

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3732,6 +3732,10 @@ data:
     # enabled, otherwise this flag will not take effect.
     #  TopologyAwareHints: true
 
+    # Enable ServiceTrafficDistribution in AntreaProxy. This requires AntreaProxy and EndpointSlice to be
+    # enabled, otherwise this flag will not take effect.
+    #  ServiceTrafficDistribution: false
+
     # Enable support for cleaning up stale UDP Service conntrack connections in AntreaProxy. This requires AntreaProxy to
     # be enabled, otherwise this flag will not take effect.
     #  CleanupStaleUDPSvcConntrack: true
@@ -5126,7 +5130,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f950d38c3e5f05b4e6290aae92fc46eeda9126a68a0ed6b88eee7f5c4c6fb491
+        checksum/config: 6438535a09fb284988314ef035af70f38c72064cc48ad8988d479b670fb1b775
       labels:
         app: antrea
         component: antrea-agent
@@ -5364,7 +5368,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f950d38c3e5f05b4e6290aae92fc46eeda9126a68a0ed6b88eee7f5c4c6fb491
+        checksum/config: 6438535a09fb284988314ef035af70f38c72064cc48ad8988d479b670fb1b775
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3732,6 +3732,10 @@ data:
     # enabled, otherwise this flag will not take effect.
     #  TopologyAwareHints: true
 
+    # Enable ServiceTrafficDistribution in AntreaProxy. This requires AntreaProxy and EndpointSlice to be
+    # enabled, otherwise this flag will not take effect.
+    #  ServiceTrafficDistribution: false
+
     # Enable support for cleaning up stale UDP Service conntrack connections in AntreaProxy. This requires AntreaProxy to
     # be enabled, otherwise this flag will not take effect.
     #  CleanupStaleUDPSvcConntrack: true
@@ -5126,7 +5130,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f950d38c3e5f05b4e6290aae92fc46eeda9126a68a0ed6b88eee7f5c4c6fb491
+        checksum/config: 6438535a09fb284988314ef035af70f38c72064cc48ad8988d479b670fb1b775
       labels:
         app: antrea
         component: antrea-agent
@@ -5365,7 +5369,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f950d38c3e5f05b4e6290aae92fc46eeda9126a68a0ed6b88eee7f5c4c6fb491
+        checksum/config: 6438535a09fb284988314ef035af70f38c72064cc48ad8988d479b670fb1b775
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3732,6 +3732,10 @@ data:
     # enabled, otherwise this flag will not take effect.
     #  TopologyAwareHints: true
 
+    # Enable ServiceTrafficDistribution in AntreaProxy. This requires AntreaProxy and EndpointSlice to be
+    # enabled, otherwise this flag will not take effect.
+    #  ServiceTrafficDistribution: false
+
     # Enable support for cleaning up stale UDP Service conntrack connections in AntreaProxy. This requires AntreaProxy to
     # be enabled, otherwise this flag will not take effect.
     #  CleanupStaleUDPSvcConntrack: true
@@ -5126,7 +5130,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5bab13c466e83f8a14191bfb9aad49229945c442808ea135f80cafe5e21be5f3
+        checksum/config: 53e03d66843747e7a7537314e0550a5f7652b5a535c4796dadb5ec0d49d0b10c
       labels:
         app: antrea
         component: antrea-agent
@@ -5362,7 +5366,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5bab13c466e83f8a14191bfb9aad49229945c442808ea135f80cafe5e21be5f3
+        checksum/config: 53e03d66843747e7a7537314e0550a5f7652b5a535c4796dadb5ec0d49d0b10c
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3745,6 +3745,10 @@ data:
     # enabled, otherwise this flag will not take effect.
     #  TopologyAwareHints: true
 
+    # Enable ServiceTrafficDistribution in AntreaProxy. This requires AntreaProxy and EndpointSlice to be
+    # enabled, otherwise this flag will not take effect.
+    #  ServiceTrafficDistribution: false
+
     # Enable support for cleaning up stale UDP Service conntrack connections in AntreaProxy. This requires AntreaProxy to
     # be enabled, otherwise this flag will not take effect.
     #  CleanupStaleUDPSvcConntrack: true
@@ -5139,7 +5143,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7212fbcdde8fe4be00f31ebbbcd7b03a7335666b4af245eed7dac1ba9e99118c
+        checksum/config: af6fec74f411f3c839afd2b26fc0075fa4da84bbdde2d5008790899e978c5b78
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -5421,7 +5425,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7212fbcdde8fe4be00f31ebbbcd7b03a7335666b4af245eed7dac1ba9e99118c
+        checksum/config: af6fec74f411f3c839afd2b26fc0075fa4da84bbdde2d5008790899e978c5b78
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3732,6 +3732,10 @@ data:
     # enabled, otherwise this flag will not take effect.
     #  TopologyAwareHints: true
 
+    # Enable ServiceTrafficDistribution in AntreaProxy. This requires AntreaProxy and EndpointSlice to be
+    # enabled, otherwise this flag will not take effect.
+    #  ServiceTrafficDistribution: false
+
     # Enable support for cleaning up stale UDP Service conntrack connections in AntreaProxy. This requires AntreaProxy to
     # be enabled, otherwise this flag will not take effect.
     #  CleanupStaleUDPSvcConntrack: true
@@ -5126,7 +5130,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 616b79b8deedba740ff992ca870b346c64c1dde5e3381436dc2cb24c0bd98ead
+        checksum/config: 859c4b3dbc63c548c67271a7485cf4d09b90c89e11dfaecb1dfc9c3bbb435ad9
       labels:
         app: antrea
         component: antrea-agent
@@ -5362,7 +5366,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 616b79b8deedba740ff992ca870b346c64c1dde5e3381436dc2cb24c0bd98ead
+        checksum/config: 859c4b3dbc63c548c67271a7485cf4d09b90c89e11dfaecb1dfc9c3bbb435ad9
       labels:
         app: antrea
         component: antrea-controller

diff --git a/docs/feature-gates.md b/docs/feature-gates.md
@@ -35,6 +35,7 @@ edit the Agent configuration in the
 | `AntreaProxy`                 | Agent              | `true`  | GA    | v0.8          | v0.11        | v1.14      | Yes                | Must be enabled for Windows.                  |
 | `EndpointSlice`               | Agent              | `true`  | GA    | v0.13.0       | v1.11        | v1.14      | Yes                |                                               |
 | `TopologyAwareHints`          | Agent              | `true`  | Beta  | v1.8          | v1.12        | N/A        | Yes                |                                               |
+| `ServiceTrafficDistribution`  | Agent              | `true`  | Alpha | v2.2          | N/A          | N/A        | Yes                |                                               |
 | `CleanupStaleUDPSvcConntrack` | Agent              | `true`  | Beta  | v1.13         | v2.1         | N/A        | Yes                |                                               |
 | `LoadBalancerModeDSR`         | Agent              | `false` | Alpha | v1.13         | N/A          | N/A        | Yes                |                                               |
 | `AntreaPolicy`                | Agent + Controller | `true`  | Beta  | v0.8          | v1.0         | N/A        | No                 | Agent side config required from v0.9.0+.      |
@@ -104,6 +105,19 @@ Refer to this [link](https://kubernetes.io/docs/concepts/services-networking/top
 - Option `antreaProxy.enable` is set to true.
 - EndpointSlice API version v1 is available in Kubernetes.
 
+### ServiceTrafficDistribution
+
+`ServiceTrafficDistribution` enables Traffic Distribution for Services in Antrea Proxy. This feature allows for more
+flexible and intelligent routing decisions by considering both topology and non-topology factors. For more details,
+refer to this [link](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/4444-service-traffic-distribution).
+
+#### Requirements for this Feature
+
+- Option `antreaProxy.enable` is set to true.
+- EndpointSlice API version v1 is available in Kubernetes.
+- Kubernetes must be version 1.30 or higher, with the `ServiceTrafficDistribution` feature gate (a Kubernetes-specific
+  feature gate) enabled.
+
 ### LoadBalancerModeDSR
 
 `LoadBalancerModeDSR` allows users to specify the load balancer mode as DSR (Direct Server Return). The load balancer

diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go
@@ -124,19 +124,20 @@ type proxier struct {
 	syncedOnce      bool
 	syncedOnceMutex sync.RWMutex
 
-	runner                      *k8sproxy.BoundedFrequencyRunner
-	stopChan                    <-chan struct{}
-	ofClient                    openflow.Client
-	routeClient                 route.Interface
-	nodePortAddresses           []net.IP
-	hostname                    string
-	isIPv6                      bool
-	proxyAll                    bool
-	endpointSliceEnabled        bool
-	proxyLoadBalancerIPs        bool
-	topologyAwareHintsEnabled   bool
-	supportNestedService        bool
-	cleanupStaleUDPSvcConntrack bool
+	runner                            *k8sproxy.BoundedFrequencyRunner
+	stopChan                          <-chan struct{}
+	ofClient                          openflow.Client
+	routeClient                       route.Interface
+	nodePortAddresses                 []net.IP
+	hostname                          string
+	isIPv6                            bool
+	proxyAll                          bool
+	endpointSliceEnabled              bool
+	proxyLoadBalancerIPs              bool
+	topologyAwareHintsEnabled         bool
+	serviceTrafficDistributionEnabled bool
+	supportNestedService              bool
+	cleanupStaleUDPSvcConntrack       bool
 
 	// When a Service's LoadBalancerMode is DSR, the following changes will be applied to the OpenFlow flows and groups:
 	// 1. ClusterGroup will be used by traffic working in DSR mode on ingress Node.
@@ -1221,7 +1222,7 @@ func (p *proxier) Run(stopCh <-chan struct{}) {
 		go p.serviceConfig.Run(stopCh)
 		if p.endpointSliceEnabled {
 			go p.endpointSliceConfig.Run(stopCh)
-			if p.topologyAwareHintsEnabled {
+			if p.topologyAwareHintsEnabled || p.serviceTrafficDistributionEnabled {
 				go p.nodeConfig.Run(stopCh)
 			}
 		} else {
@@ -1372,6 +1373,7 @@ func newProxier(
 		}
 	}
 	topologyAwareHintsEnabled := endpointSliceEnabled && features.DefaultFeatureGate.Enabled(features.TopologyAwareHints)
+	serviceTrafficDistributionEnabled := endpointSliceEnabled && features.DefaultFeatureGate.Enabled(features.ServiceTrafficDistribution)
 	ipFamily := corev1.IPv4Protocol
 	if isIPv6 {
 		ipFamily = corev1.IPv6Protocol
@@ -1401,40 +1403,41 @@ func newProxier(
 	serviceLabelSelector = serviceLabelSelector.Add(*serviceProxyNameSelector, *nonHeadlessServiceSelector)
 
 	p := &proxier{
-		nodeIPChecker:               nodeIPChecker,
-		serviceConfig:               config.NewServiceConfig(serviceInformer, resyncPeriod),
-		endpointsChanges:            newEndpointsChangesTracker(hostname, endpointSliceEnabled, isIPv6),
-		serviceChanges:              newServiceChangesTracker(recorder, ipFamily, serviceLabelSelector, skipServices),
-		serviceMap:                  k8sproxy.ServiceMap{},
-		serviceInstalledMap:         k8sproxy.ServiceMap{},
-		endpointsInstalledMap:       types.EndpointsMap{},
-		endpointsMap:                types.EndpointsMap{},
-		endpointReferenceCounter:    map[string]int{},
-		nodeLabels:                  map[string]string{},
-		serviceStringMap:            map[string]k8sproxy.ServicePortName{},
-		groupCounter:                groupCounter,
-		ofClient:                    ofClient,
-		routeClient:                 routeClient,
-		nodePortAddresses:           nodePortAddresses,
-		isIPv6:                      isIPv6,
-		proxyAll:                    proxyAllEnabled,
-		endpointSliceEnabled:        endpointSliceEnabled,
-		topologyAwareHintsEnabled:   topologyAwareHintsEnabled,
-		cleanupStaleUDPSvcConntrack: features.DefaultFeatureGate.Enabled(features.CleanupStaleUDPSvcConntrack),
-		proxyLoadBalancerIPs:        proxyLoadBalancerIPs,
-		hostname:                    hostname,
-		serviceHealthServer:         serviceHealthServer,
-		numLocalEndpoints:           map[apimachinerytypes.NamespacedName]int{},
-		supportNestedService:        supportNestedService,
-		defaultLoadBalancerMode:     defaultLoadBalancerMode,
+		nodeIPChecker:                     nodeIPChecker,
+		serviceConfig:                     config.NewServiceConfig(serviceInformer, resyncPeriod),
+		endpointsChanges:                  newEndpointsChangesTracker(hostname, endpointSliceEnabled, isIPv6),
+		serviceChanges:                    newServiceChangesTracker(recorder, ipFamily, serviceLabelSelector, skipServices),
+		serviceMap:                        k8sproxy.ServiceMap{},
+		serviceInstalledMap:               k8sproxy.ServiceMap{},
+		endpointsInstalledMap:             types.EndpointsMap{},
+		endpointsMap:                      types.EndpointsMap{},
+		endpointReferenceCounter:          map[string]int{},
+		nodeLabels:                        map[string]string{},
+		serviceStringMap:                  map[string]k8sproxy.ServicePortName{},
+		groupCounter:                      groupCounter,
+		ofClient:                          ofClient,
+		routeClient:                       routeClient,
+		nodePortAddresses:                 nodePortAddresses,
+		isIPv6:                            isIPv6,
+		proxyAll:                          proxyAllEnabled,
+		endpointSliceEnabled:              endpointSliceEnabled,
+		topologyAwareHintsEnabled:         topologyAwareHintsEnabled,
+		serviceTrafficDistributionEnabled: serviceTrafficDistributionEnabled,
+		cleanupStaleUDPSvcConntrack:       features.DefaultFeatureGate.Enabled(features.CleanupStaleUDPSvcConntrack),
+		proxyLoadBalancerIPs:              proxyLoadBalancerIPs,
+		hostname:                          hostname,
+		serviceHealthServer:               serviceHealthServer,
+		numLocalEndpoints:                 map[apimachinerytypes.NamespacedName]int{},
+		supportNestedService:              supportNestedService,
+		defaultLoadBalancerMode:           defaultLoadBalancerMode,
 	}
 
 	p.serviceConfig.RegisterEventHandler(p)
 	p.runner = k8sproxy.NewBoundedFrequencyRunner(componentName, p.syncProxyRules, time.Second, 30*time.Second, 2)
 	if endpointSliceEnabled {
 		p.endpointSliceConfig = config.NewEndpointSliceConfig(endpointSliceInformer, resyncPeriod)
 		p.endpointSliceConfig.RegisterEventHandler(p)
-		if p.topologyAwareHintsEnabled {
+		if p.topologyAwareHintsEnabled || p.serviceTrafficDistributionEnabled {
 			p.nodeConfig = config.NewNodeConfig(nodeInformer, resyncPeriod)
 			p.nodeConfig.RegisterEventHandler(p)
 		}

diff --git a/pkg/agent/proxy/topology.go b/pkg/agent/proxy/topology.go
@@ -115,19 +115,22 @@ func (p *proxier) categorizeEndpoints(endpoints map[string]k8sproxy.Endpoint, sv
 
 // canUseTopology returns true if topology aware routing is enabled and properly configured in this cluster. That is,
 // it checks that:
-// - The TopologyAwareHints feature is enabled.
-// - The "service.kubernetes.io/topology-aware-hints" annotation on this Service is set to "Auto".
-// - The node's labels include "topology.kubernetes.io/zone".
-// - All of the Endpoints for this Service have a topology hint.
-// - At least one Endpoint for this Service is hinted for this Node's zone.
+//   - The TopologyAwareHints or ServiceTrafficDistribution feature is enabled.
+//   - If ServiceTrafficDistribution feature is not enabled, then the "service.kubernetes.io/topology-aware-hints"
+//     annotation on this Service should be set to "Auto" or "auto".
+//   - The node's labels include "topology.kubernetes.io/zone".
+//   - All of the Endpoints for this Service have a topology hint.
+//   - At least one Endpoint for this Service is hinted for this Node's zone.
 func (p *proxier) canUseTopology(endpoints map[string]k8sproxy.Endpoint, svcInfo k8sproxy.ServicePort) bool {
-	if !p.topologyAwareHintsEnabled {
+	if !p.topologyAwareHintsEnabled && !p.serviceTrafficDistributionEnabled {
 		return false
 	}
-	// Any non-empty and non-disabled values for the hints annotation are acceptable.
-	hintsAnnotation := svcInfo.HintsAnnotation()
-	if hintsAnnotation == "" || hintsAnnotation == "disabled" || hintsAnnotation == "Disabled" {
-		return false
+	if !p.serviceTrafficDistributionEnabled {
+		// Any non-empty and non-disabled values for the hints annotation are acceptable.
+		hintsAnnotation := svcInfo.HintsAnnotation()
+		if hintsAnnotation == "" || hintsAnnotation == "disabled" || hintsAnnotation == "Disabled" {
+			return false
+		}
 	}
 
 	zone, ok := p.nodeLabels[v1.LabelTopologyZone]