Feature: nlb zone affinity via r53 (#694)

* feature: set nlb zone affinity for clients This enables you to set dns_record.client_routing_policy configuration, such that you can decide for NLB via R53 feature that clients running in the same zone will hit also the same zone of your NLB. This is interesting for cross cluster traffic or traffic that stays in the same cluster but passes the NLB. Valid configurations are: - availability_zone_affinity - partial_availability_zone_affinity - any_availability_zone (default) see also https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html\#load-balancer-attributes and https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html\#zonal-dns-affinity Signed-off-by: Sandor Szücs <sandor.szuecs@zalando.de> * log: log default cross zone and zone affinity config Signed-off-by: Sandor Szücs <sandor.szuecs@zalando.de> * fix: comments Signed-off-by: Sandor Szücs <sandor.szuecs@zalando.de> --------- Signed-off-by: Sandor Szücs <sandor.szuecs@zalando.de>
zalando-incubator · Apr 30, 2024 · d93850d · d93850d
1 parent 0363b3f
commit d93850d
Show file tree

Hide file tree

Showing 11 changed files with 95 additions and 3 deletions.
diff --git a/aws/adapter.go b/aws/adapter.go
@@ -66,6 +66,7 @@ type Adapter struct {
 	ipAddressType               string
 	albLogsS3Bucket             string
 	albLogsS3Prefix             string
+	nlbZoneAffinity             string
 	httpRedirectToHTTPS         bool
 	nlbCrossZone                bool
 	nlbHTTPEnabled              bool
@@ -124,8 +125,13 @@ const (
 	DefaultAlbS3LogsPrefix = ""
 
 	DefaultCustomFilter = ""
+
+	// DefaultZoneAffinity specifies dns_record.client_routing_policy, see also https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#load-balancer-attributes
+	DefaultZoneAffinity = "any_availability_zone"
+
 	// DefaultNLBCrossZone specifies the default configuration for cross
 	// zone load balancing: https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#load-balancer-attributes
+	// It it is safe to change as per https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticloadbalancingv2-loadbalancer-loadbalancerattribute.html#aws-properties-elasticloadbalancingv2-loadbalancer-loadbalancerattribute-properties
 	DefaultNLBCrossZone   = false
 	DefaultNLBHTTPEnabled = false
 
@@ -246,6 +252,7 @@ func NewAdapter(clusterID, newControllerID, vpcID string, debug, disableInstrume
 		albLogsS3Bucket:            DefaultAlbS3LogsBucket,
 		albLogsS3Prefix:            DefaultAlbS3LogsPrefix,
 		nlbCrossZone:               DefaultNLBCrossZone,
+		nlbZoneAffinity:            DefaultZoneAffinity,
 		nlbHTTPEnabled:             DefaultNLBHTTPEnabled,
 		customFilter:               DefaultCustomFilter,
 		TargetCNI: &TargetCNIconfig{
@@ -446,6 +453,13 @@ func (a *Adapter) WithNLBCrossZone(nlbCrossZone bool) *Adapter {
 	return a
 }
 
+// WithNLBZoneAffinity returns the receiver adapter after setting the
+// nlbZoneAffinity config.
+func (a *Adapter) WithNLBZoneAffinity(nlbZoneAffinity string) *Adapter {
+	a.nlbZoneAffinity = nlbZoneAffinity
+	return a
+}
+
 // WithNLBHTTPEnabled returns the receiver adapter after setting the
 // nlbHTTPEnabled config.
 func (a *Adapter) WithNLBHTTPEnabled(nlbHTTPEnabled bool) *Adapter {
@@ -753,6 +767,7 @@ func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, o
 		cwAlarms:                          cwAlarms,
 		httpRedirectToHTTPS:               a.httpRedirectToHTTPS,
 		nlbCrossZone:                      a.nlbCrossZone,
+		nlbZoneAffinity:                   a.nlbZoneAffinity,
 		http2:                             http2,
 		tags:                              a.stackTags,
 		internalDomains:                   a.internalDomains,
@@ -809,6 +824,7 @@ func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time.
 		cwAlarms:                          cwAlarms,
 		httpRedirectToHTTPS:               a.httpRedirectToHTTPS,
 		nlbCrossZone:                      a.nlbCrossZone,
+		nlbZoneAffinity:                   a.nlbZoneAffinity,
 		http2:                             http2,
 		tags:                              a.stackTags,
 		internalDomains:                   a.internalDomains,

diff --git a/aws/cf.go b/aws/cf.go
@@ -179,6 +179,7 @@ type stackSpec struct {
 	albLogsS3Bucket                   string
 	albLogsS3Prefix                   string
 	wafWebAclId                       string
+	nlbZoneAffinity                   string
 	cwAlarms                          CloudWatchAlarmList
 	httpRedirectToHTTPS               bool
 	nlbCrossZone                      bool

diff --git a/aws/cf_template.go b/aws/cf_template.go
@@ -282,7 +282,7 @@ func generateTemplate(spec *stackSpec) (string, error) {
 	}
 
 	// Build up the LoadBalancerAttributes list, as there is no way to make attributes conditional in the template
-	lbAttrList := make(cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttributeList, 0, 4)
+	lbAttrList := make(cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttributeList, 0, 5)
 
 	if spec.loadbalancerType == LoadBalancerTypeApplication {
 		lbAttrList = append(lbAttrList,
@@ -307,6 +307,18 @@ func generateTemplate(spec *stackSpec) (string, error) {
 				Value: cloudformation.String(fmt.Sprintf("%t", spec.nlbCrossZone)),
 			},
 		)
+
+		lbAttrList = append(lbAttrList,
+			cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute{
+				// https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#load-balancer-attributes
+				// https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#zonal-dns-affinity
+				Key: cloudformation.String("dns_record.client_routing_policy"),
+				// availability_zone_affinity 100%
+				// partial_availability_zone_affinity 85%
+				// any_availability_zone 0%
+				Value: cloudformation.String(spec.nlbZoneAffinity),
+			},
+		)
 	}
 
 	if spec.albLogsS3Bucket != "" {

diff --git a/aws/cf_template_test.go b/aws/cf_template_test.go
@@ -291,6 +291,48 @@ func TestGenerateTemplate(t *testing.T) {
 				require.Equal(t, attributes[0].Value.Literal, "false")
 			},
 		},
+		{
+			name: "nlb zone affinity Route53 can be configured to any_availability_zone for Network load balancers",
+			spec: &stackSpec{
+				loadbalancerType: LoadBalancerTypeNetwork,
+				nlbZoneAffinity:  "any_availability_zone",
+			},
+			validate: func(t *testing.T, template *cloudformation.Template) {
+				require.NotNil(t, template.Resources["LB"])
+				properties := template.Resources["LB"].Properties.(*cloudformation.ElasticLoadBalancingV2LoadBalancer)
+				attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes)
+				require.Equal(t, attributes[1].Key.Literal, "dns_record.client_routing_policy")
+				require.Equal(t, attributes[1].Value.Literal, "any_availability_zone")
+			},
+		},
+		{
+			name: "nlb zone affinity Route53 can be configured to availability_zone_affinity for Network load balancers",
+			spec: &stackSpec{
+				loadbalancerType: LoadBalancerTypeNetwork,
+				nlbZoneAffinity:  "availability_zone_affinity",
+			},
+			validate: func(t *testing.T, template *cloudformation.Template) {
+				require.NotNil(t, template.Resources["LB"])
+				properties := template.Resources["LB"].Properties.(*cloudformation.ElasticLoadBalancingV2LoadBalancer)
+				attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes)
+				require.Equal(t, attributes[1].Key.Literal, "dns_record.client_routing_policy")
+				require.Equal(t, attributes[1].Value.Literal, "availability_zone_affinity")
+			},
+		},
+		{
+			name: "nlb zone affinity Route53 can be configured to partial_availability_zone_affinity for Network load balancers",
+			spec: &stackSpec{
+				loadbalancerType: LoadBalancerTypeNetwork,
+				nlbZoneAffinity:  "partial_availability_zone_affinity",
+			},
+			validate: func(t *testing.T, template *cloudformation.Template) {
+				require.NotNil(t, template.Resources["LB"])
+				properties := template.Resources["LB"].Properties.(*cloudformation.ElasticLoadBalancingV2LoadBalancer)
+				attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes)
+				require.Equal(t, attributes[1].Key.Literal, "dns_record.client_routing_policy")
+				require.Equal(t, attributes[1].Value.Literal, "partial_availability_zone_affinity")
+			},
+		},
 		{
 			name: "nlb HTTP listener should not be enabled when HTTP is disabled",
 			spec: &stackSpec{

diff --git a/controller.go b/controller.go
@@ -76,6 +76,7 @@ var (
 	cwAlarmConfigMap              string
 	cwAlarmConfigMapLocation      *kubernetes.ResourceLocation
 	loadBalancerType              string
+	nlbZoneAffinity               string
 	nlbCrossZone                  bool
 	nlbHTTPEnabled                bool
 	ingressAPIVersion             string
@@ -175,6 +176,8 @@ func loadSettings() error {
 		Default(defaultHTTPRedirectToHTTPS).BoolVar(&httpRedirectToHTTPS)
 	kingpin.Flag("load-balancer-type", "Sets default Load Balancer type (application or network).").
 		Default(aws.LoadBalancerTypeApplication).EnumVar(&loadBalancerType, aws.LoadBalancerTypeApplication, aws.LoadBalancerTypeNetwork)
+	kingpin.Flag("nlb-zone-affinity", "Specify whether Route53 should return zone aware Network Load Balancers IPs. It configures dns_record.client_routing_policy NLB configuration. This setting only apply to 'network' Load Balancers.").
+		Default(aws.DefaultZoneAffinity).StringVar(&nlbZoneAffinity)
 	kingpin.Flag("nlb-cross-zone", "Specify whether Network Load Balancers should balance cross availablity zones. This setting only apply to 'network' Load Balancers.").
 		Default("false").BoolVar(&nlbCrossZone)
 	kingpin.Flag("nlb-http-enabled", "Enable HTTP (port 80) for Network Load Balancers. By default this is disabled as NLB can't provide HTTP -> HTTPS redirect.").
@@ -339,6 +342,7 @@ func main() {
 		WithAlbLogsS3Prefix(albLogsS3Prefix).
 		WithHTTPRedirectToHTTPS(httpRedirectToHTTPS).
 		WithNLBCrossZone(nlbCrossZone).
+		WithNLBZoneAffinity(nlbZoneAffinity).
 		WithNLBHTTPEnabled(nlbHTTPEnabled).
 		WithCustomFilter(customFilter).
 		WithStackTags(additionalStackTags).
@@ -409,6 +413,8 @@ func main() {
 	log.Infof("CloudWatch Alarm ConfigMap: %s", cwAlarmConfigMapLocation)
 	log.Infof("Default LoadBalancer type: %s", loadBalancerType)
 	log.Infof("Target access mode: %s", targetAccessMode)
+	log.Infof("NLB Cross Zone: %t", nlbCrossZone)
+	log.Infof("NLB Zone Affinity: %s", nlbZoneAffinity)
 
 	go handleTerminationSignals(cancel, syscall.SIGTERM, syscall.SIGQUIT)
 	go serveMetrics(metricsAddress)

diff --git a/controller_test.go b/controller_test.go
@@ -67,6 +67,7 @@ func TestDefaultLoadSettings(t *testing.T) {
 	require.Equal(t, "", cwAlarmConfigMap)
 	require.Equal(t, (*kubernetes.ResourceLocation)(nil), cwAlarmConfigMapLocation)
 	require.Equal(t, "application", loadBalancerType)
+	require.Equal(t, "any_availability_zone", nlbZoneAffinity)
 	require.Equal(t, false, nlbCrossZone)
 	require.Equal(t, false, nlbHTTPEnabled)
 	require.Equal(t, "networking.k8s.io/v1", ingressAPIVersion)

diff --git a/testdata/ingress_nlb/input/k8s/ing.yaml b/testdata/ingress_nlb/input/k8s/ing.yaml
@@ -3,7 +3,7 @@ kind: Ingress
 metadata:
   name: myingress
   annotations:
-    zalando.org/aws-load-balancer-type: nlb	
+    zalando.org/aws-load-balancer-type: nlb
 spec:
   rules:
   - host: foo.bar.org

diff --git a/testdata/ingress_nlb/output/templates/ing.cf b/testdata/ingress_nlb/output/templates/ing.cf
@@ -116,6 +116,10 @@
                         "Key": "load_balancing.cross_zone.enabled",
                         "Value": "false"
                     },
+                    {
+                        "Key": "dns_record.client_routing_policy",
+                        "Value": "any_availability_zone"
+                    },
                     {
                         "Key": "access_logs.s3.enabled",
                         "Value": "false"

diff --git a/testdata/ingress_rg_shared_nlb/output/templates/shared.cf b/testdata/ingress_rg_shared_nlb/output/templates/shared.cf
@@ -116,6 +116,10 @@
                         "Key": "load_balancing.cross_zone.enabled",
                         "Value": "false"
                     },
+                    {
+                        "Key": "dns_record.client_routing_policy",
+                        "Value": "any_availability_zone"
+                    },
                     {
                         "Key": "access_logs.s3.enabled",
                         "Value": "false"

diff --git a/testdata/rg_nlb/output/templates/rg.cf b/testdata/rg_nlb/output/templates/rg.cf
@@ -116,6 +116,10 @@
                         "Key": "load_balancing.cross_zone.enabled",
                         "Value": "false"
                     },
+                    {
+                        "Key": "dns_record.client_routing_policy",
+                        "Value": "any_availability_zone"
+                    },
                     {
                         "Key": "access_logs.s3.enabled",
                         "Value": "false"

diff --git a/worker_test.go b/worker_test.go
@@ -18,6 +18,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/zalando-incubator/kube-ingress-aws-controller/aws"
 	awsAdapter "github.com/zalando-incubator/kube-ingress-aws-controller/aws"
 	"github.com/zalando-incubator/kube-ingress-aws-controller/certs"
 	"github.com/zalando-incubator/kube-ingress-aws-controller/kubernetes"
@@ -471,7 +472,8 @@ func TestResourceConversionOneToOne(tt *testing.T) {
 			a = a.WithCustomAutoScalingClient(clientASG).
 				WithCustomEc2Client(clientEC2).
 				WithCustomElbv2Client(clientELBv2).
-				WithCustomCloudFormationClient(clientCF)
+				WithCustomCloudFormationClient(clientCF).
+				WithNLBZoneAffinity(aws.DefaultZoneAffinity)
 
 			a, err = a.UpdateManifest(clusterID, vpcID)
 			if err != nil {