Support k8gb behind a reverse proxy (#1710)

* Add reverse proxy support

Problem
K8GB reads IP addresses from `Ingress.Status.LoadBalancer.Ingress` or from `Service.Status.LoadBalancer.Ingress` for ingress configured with Kubernetes Ingress and Istio Virtual Service, respectively.
The IP addresses exposed by these resources are the IP addresses exposed by the Kubernetes Cluster. However, in some setups the clients do not route their traffic to these IP addresses because the cluster is behind a reverse proxy.

Solution
To support this setup, K8GB should expose DNS records with the IP address of the reverse proxy. Since the address is unknown to the cluster the K8GB administrator must provide it via configuration. This PR adds to K8GB the capability to read IP address from an annotation `k8gb.io/external-ips` on the GSLB resource.

Example
```
apiVersion: k8gb.absa.oss/v1beta1
kind: Gslb
metadata:
  labels:
    app: ingress
  annotations:
    k8gb.io/external-ips: "185.199.110.153"
```

Fixes #1275

Signed-off-by: Andre Baptista Aguas <andre.aguas@protonmail.com>

* Validate k8gb.io/exposed-ip-addresses annotation

Signed-off-by: Andre Aguas <andre.aguas@protonmail.com>

* set the admission policy to false by default

Signed-off-by: Andre Aguas <andre.aguas@protonmail.com>

---------

Signed-off-by: Andre Baptista Aguas <andre.aguas@protonmail.com>
Signed-off-by: Andre Aguas <andre.aguas@protonmail.com>

chart/k8gb/templates/validatingadmissionpolicy.yaml

@@ -0,0 +1,32 @@
+{{ if .Values.k8gb.validatingAdmissionPolicy.enabled -}}
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: k8gb-exposed-ip-annotation
+spec:
+  policyName: k8gb-exposed-ip-annotation
+  matchResources:
+    namespaceSelector: {}
+  validationActions:
+    - Deny
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: k8gb-exposed-ip-annotation
+spec:
+  validations:
+    - expression: object.metadata.annotations['k8gb.io/exposed-ip-addresses'].matches('^((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])$')
+      message: The annotation 'k8gb.io/exposed-ip-addresses' must contain a valid IPv4 address
+  matchConditions:
+    - name: hasExposedIPAddressesAnnotation
+      expression: "has(object.metadata.annotations) && 'k8gb.io/exposed-ip-addresses' in object.metadata.annotations"
+  matchConstraints:
+    resourceRules:
+      - apiGroups: ["k8gb.absa.oss"]
+        apiVersions: ["v1beta1"]
+        operations: ["CREATE", "UPDATE"]
+        resources: ["gslbs"]
+  failurePolicy: Fail
+{{ end -}}

chart/k8gb/values.schema.json

@@ -324,6 +324,9 @@
                 "serviceMonitor": {
                     "$ref": "#/definitions/k8gbServiceMonitor"
                 },
+                "validatingAdmissionPolicy": {
+                    "$ref": "#/definitions/k8gbValidatingAdmissionPolicy"
+                },
                 "podAnnotations": {
                     "type": "object"
                 },
@@ -409,6 +412,15 @@
                 }
             }
         },
+        "k8gbValidatingAdmissionPolicy": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                }
+            }
+        },
         "Ns1": {
             "type": "object",
             "additionalProperties": false,

chart/k8gb/values.yaml

@@ -56,6 +56,9 @@ k8gb:
   # -- enable ServiceMonitor
   serviceMonitor:
     enabled: false
+  # -- enable validating admission policies
+  validatingAdmissionPolicy:
+    enabled: false
   # -- pod annotations
   podAnnotations: {}
   # -- pod labels

controllers/depresolver/testdata/invalid_omitempty_empty.yaml

@@ -37,5 +37,3 @@ spec:
     type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too
     splitBrainThresholdSeconds:
     weight:
-
-

controllers/gslb_controller_reconciliation.go

@@ -178,7 +178,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 	}
 	gslb.Status.Servers = servers
 
-	loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Config.EdgeDNSServers)
+	loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(gslb.Annotations, r.Config.EdgeDNSServers)
 	if err != nil {
 		m.IncrementError(gslb)
 		return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err))

controllers/refresolver/ingress/ingress.go

@@ -35,6 +35,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	ingress *netv1.Ingress
 }
@@ -158,9 +163,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 }
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
+		return utils.ParseIPAddresses(ingressIPsFromAnnotation)
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.ingress.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

controllers/refresolver/ingress/ingress_test.go

@@ -96,40 +96,80 @@ func TestGetServers(t *testing.T) {
 
 func TestGetGslbExposedIPs(t *testing.T) {
 	var tests = []struct {
-		name        string
-		ingressYaml string
-		expectedIPs []string
+		name          string
+		annotations   map[string]string
+		ingressYaml   string
+		expectedIPs   []string
+		expectedError bool
 	}{
 		{
-			name:        "no exposed IPs",
-			ingressYaml: "./testdata/ingress_no_ips.yaml",
-			expectedIPs: []string{},
+			name:          "no exposed IPs",
+			annotations:   map[string]string{},
+			ingressYaml:   "./testdata/ingress_no_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: false,
 		},
 		{
-			name:        "single exposed IP",
-			ingressYaml: "../testdata/ingress_referenced.yaml",
-			expectedIPs: []string{"10.0.0.1"},
+			name:          "single exposed IP",
+			annotations:   map[string]string{},
+			ingressYaml:   "../testdata/ingress_referenced.yaml",
+			expectedIPs:   []string{"10.0.0.1"},
+			expectedError: false,
 		},
 		{
-			name:        "multiple exposed IPs",
-			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
-			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
+			name:          "multiple exposed IPs",
+			annotations:   map[string]string{},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{"10.0.0.1", "10.0.0.2"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with no exposed IPs",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": ""},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: true,
+		},
+		{
+			name:          "annotation with single exposed IP",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{"185.199.110.153"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with multiple exposed IPs",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{"185.199.110.153", "185.199.109.153"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with invalid IP",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "192.169.0.test"},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: true,
 		},
 	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
 			// arrange
-			ingress := utils.FileToIngress(test.ingressYaml)
+			ingress := utils.FileToIngress(tt.ingressYaml)
 			resolver := ReferenceResolver{
 				ingress: ingress,
 			}
 
 			// act
-			IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
-			assert.NoError(t, err)
+			IPs, err := resolver.GetGslbExposedIPs(tt.annotations, []utils.DNSServer{})
+			if tt.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
 
 			// assert
-			assert.Equal(t, test.expectedIPs, IPs)
+			assert.Equal(t, tt.expectedIPs, IPs)
 		})
 	}
 }

controllers/refresolver/istiovirtualservice/istiovirtualservice.go

@@ -37,6 +37,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	virtualService *istio.VirtualService
 	lbService      *corev1.Service
@@ -191,9 +196,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 }
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
+		return utils.ParseIPAddresses(ingressIPsFromAnnotation)
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.lbService.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go

@@ -111,40 +111,73 @@ func TestGetServers(t *testing.T) {
 
 func TestGetGslbExposedIPs(t *testing.T) {
 	var tests = []struct {
-		name        string
-		serviceYaml string
-		expectedIPs []string
+		name          string
+		annotations   map[string]string
+		serviceYaml   string
+		expectedIPs   []string
+		expectedError bool
 	}{
 		{
-			name:        "no exposed IPs",
-			serviceYaml: "./testdata/istio_service_no_ips.yaml",
-			expectedIPs: []string{},
+			name:          "no exposed IPs",
+			serviceYaml:   "./testdata/istio_service_no_ips.yaml",
+			annotations:   map[string]string{},
+			expectedIPs:   []string{},
+			expectedError: false,
 		},
 		{
-			name:        "single exposed IP",
-			serviceYaml: "../testdata/istio_service.yaml",
-			expectedIPs: []string{"10.0.0.1"},
+			name:          "single exposed IP",
+			annotations:   map[string]string{},
+			serviceYaml:   "../testdata/istio_service.yaml",
+			expectedIPs:   []string{"10.0.0.1"},
+			expectedError: false,
 		},
 		{
-			name:        "multiple exposed IPs",
-			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
-			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
+			name:          "multiple exposed IPs",
+			annotations:   map[string]string{},
+			serviceYaml:   "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs:   []string{"10.0.0.1", "10.0.0.2"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with no exposed IPs",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": ""},
+			serviceYaml:   "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: true,
+		},
+		{
+			name:          "annotation with single exposed IP",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
+			serviceYaml:   "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs:   []string{"185.199.110.153"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with invalid IP",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "192.169.0.test"},
+			serviceYaml:   "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: true,
 		},
 	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
 			// arrange
-			svc := utils.FileToService(test.serviceYaml)
+			svc := utils.FileToService(tt.serviceYaml)
 			resolver := ReferenceResolver{
 				lbService: svc,
 			}
 
 			// act
-			IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
-			assert.NoError(t, err)
+			IPs, err := resolver.GetGslbExposedIPs(tt.annotations, []utils.DNSServer{})
+			if tt.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
 
 			// assert
-			assert.Equal(t, test.expectedIPs, IPs)
+			assert.Equal(t, tt.expectedIPs, IPs)
 		})
 	}
 }

controllers/refresolver/refresolver.go

@@ -34,7 +34,7 @@ type GslbReferenceResolver interface {
 	// GetServers retrieves GSLB the server configuration
 	GetServers() ([]*k8gbv1beta1.Server, error)
 	// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-	GetGslbExposedIPs(utils.DNSList) ([]string, error)
+	GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error)
 }
 
 // New creates a new GSLBReferenceResolver