feat(envoy): closing off the admin interface (#5936)

* closing off the admin interface * update hacks * remove port 9003 from envoy svc * fixing the service monitor
SeldonIO · Oct 22, 2024 · 5801de6 · 5801de6
1 parent be379f8
commit 5801de6
Show file tree

Hide file tree

Showing 14 changed files with 193 additions and 31 deletions.
diff --git a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml
@@ -1369,12 +1369,12 @@ spec:
         - containerPort: 9000
           name: http
         - containerPort: 9003
-          name: envoy-admin
+          name: envoy-stats
         readinessProbe:
           failureThreshold: 3
           httpGet:
             path: /ready
-            port: envoy-admin
+            port: envoy-stats
           initialDelaySeconds: 10
           periodSeconds: 5
         resources:

diff --git a/k8s/yaml/components.yaml b/k8s/yaml/components.yaml
@@ -1000,12 +1000,12 @@ spec:
         - containerPort: 9000
           name: http
         - containerPort: 9003
-          name: envoy-admin
+          name: envoy-stats
         readinessProbe:
           failureThreshold: 3
           httpGet:
             path: /ready
-            port: envoy-admin
+            port: envoy-stats
           initialDelaySeconds: 10
           periodSeconds: 5
         resources:

diff --git a/operator/config/seldonconfigs/default.yaml b/operator/config/seldonconfigs/default.yaml
@@ -58,7 +58,7 @@ spec:
         - containerPort: 9000
           name: http
         - containerPort: 9003
-          name: envoy-admin
+          name: envoy-stats
         resources:
           limits:
             memory: 128Mi
@@ -68,7 +68,7 @@ spec:
         readinessProbe:
           httpGet:
             path: /ready
-            port: envoy-admin
+            port: envoy-stats
           initialDelaySeconds: 10
           periodSeconds: 5
           failureThreshold: 3

diff --git a/operator/controllers/reconcilers/seldon/service_reconciler.go b/operator/controllers/reconcilers/seldon/service_reconciler.go
@@ -119,8 +119,8 @@ func getSeldonMeshService(meta metav1.ObjectMeta, serviceConfig mlopsv1alpha1.Se
 				},
 				{
 					Port:       9003,
-					TargetPort: intstr.FromString("envoy-admin"),
-					Name:       "admin",
+					TargetPort: intstr.FromString("envoy-stats"),
+					Name:       "stats",
 					Protocol:   v1.ProtocolTCP,
 				},
 			},

diff --git a/prometheus/monitors/envoy-servicemonitor.yaml b/prometheus/monitors/envoy-servicemonitor.yaml
@@ -10,6 +10,6 @@ spec:
     matchNames: []
     any: false
   endpoints:
-  - port: admin
+  - port: stats
     interval: 15s
     path: /stats/prometheus
diff --git a/scheduler/config/envoy-compose.yaml b/scheduler/config/envoy-compose.yaml
@@ -1,4 +1,3 @@
-# Base config for a split xDS management server on 9002, admin port on 9003
 static_resources:
   clusters:
     - connect_timeout: 1s
@@ -14,6 +13,49 @@ static_resources:
                       port_value: 9002
       http2_protocol_options: {}
       name: xds_cluster
+    - connect_timeout: 0.250s
+      type: LOGICAL_DNS
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: admin_interface_cluster
+        endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 9901
+      name: admin_interface_cluster
+  listeners:
+    - name: util_endpoint_listener
+      address:
+        socket_address:
+          address: 0.0.0.0
+          port_value: 9003
+      filter_chains:
+      - filters:
+        - name: envoy.http_connection_manager
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+            stat_prefix: util_endpoint_http
+            http_filters:
+            - name: envoy.filters.http.router
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+            route_config:
+              name: local_admin_interface_route
+              virtual_hosts:
+              - name: admin_interface
+                domains: ["*"]
+                routes:
+                - match:
+                    prefix: /stats
+                  route:
+                    cluster: admin_interface_cluster
+                - match:
+                    prefix: /ready
+                  route:
+                    cluster: admin_interface_cluster
 dynamic_resources:
   cds_config:
     resource_api_version: V3
@@ -53,5 +95,5 @@ admin:
   access_log_path: /dev/null
   address:
     socket_address:
-      address: 0.0.0.0
-      port_value: 9003
+      address: 127.0.0.1
+      port_value: 9901
diff --git a/scheduler/config/envoy-local.yaml b/scheduler/config/envoy-local.yaml
@@ -1,4 +1,3 @@
-# Base config for a split xDS management server on 9002, admin port on 9003
 static_resources:
   clusters:
     - connect_timeout: 1s
@@ -14,6 +13,49 @@ static_resources:
                       port_value: 9002
       http2_protocol_options: {}
       name: xds_cluster
+    - connect_timeout: 0.250s
+      type: LOGICAL_DNS
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: admin_interface_cluster
+        endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 9901
+      name: admin_interface_cluster
+  listeners:
+    - name: util_endpoint_listener
+      address:
+        socket_address:
+          address: 0.0.0.0
+          port_value: 9003
+      filter_chains:
+      - filters:
+        - name: envoy.http_connection_manager
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+            stat_prefix: util_endpoint_http
+            http_filters:
+            - name: envoy.filters.http.router
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+            route_config:
+              name: local_admin_interface_route
+              virtual_hosts:
+              - name: admin_interface
+                domains: ["*"]
+                routes:
+                - match:
+                    prefix: /stats
+                  route:
+                    cluster: admin_interface_cluster
+                - match:
+                    prefix: /ready
+                  route:
+                    cluster: admin_interface_cluster
 dynamic_resources:
   cds_config:
     resource_api_version: V3
@@ -53,5 +95,5 @@ admin:
   access_log_path: /dev/null
   address:
     socket_address:
-      address: 0.0.0.0
-      port_value: 9003
+      address: 127.0.0.1
+      port_value: 9901
diff --git a/scheduler/config/envoy-tls.yaml b/scheduler/config/envoy-tls.yaml
@@ -1,4 +1,3 @@
-# Base config for a split xDS management server on 9002, admin port on 9003
 static_resources:
   clusters:
     - connect_timeout: 1s
@@ -27,6 +26,49 @@ static_resources:
                name: validation_context_sds
                sds_config:
                  path: /etc/validation_context_sds_secret.yaml
+    - connect_timeout: 0.250s
+      type: LOGICAL_DNS
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: admin_interface_cluster
+        endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 9901
+      name: admin_interface_cluster
+  listeners:
+    - name: util_endpoint_listener
+      address:
+        socket_address:
+          address: 0.0.0.0
+          port_value: 9003
+      filter_chains:
+      - filters:
+        - name: envoy.http_connection_manager
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+            stat_prefix: util_endpoint_http
+            http_filters:
+            - name: envoy.filters.http.router
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+            route_config:
+              name: local_admin_interface_route
+              virtual_hosts:
+              - name: admin_interface
+                domains: ["*"]
+                routes:
+                - match:
+                    prefix: /stats
+                  route:
+                    cluster: admin_interface_cluster
+                - match:
+                    prefix: /ready
+                  route:
+                    cluster: admin_interface_cluster
 dynamic_resources:
   cds_config:
     resource_api_version: V3
@@ -66,5 +108,5 @@ admin:
   access_log_path: /dev/null
   address:
     socket_address:
-      address: 0.0.0.0
-      port_value: 9003
+      address: 127.0.0.1
+      port_value: 9901
diff --git a/scheduler/config/envoy.yaml b/scheduler/config/envoy.yaml
@@ -1,4 +1,3 @@
-# Base config for a split xDS management server on 9002, admin port on 9003
 static_resources:
   clusters:
     - connect_timeout: 1s
@@ -14,6 +13,49 @@ static_resources:
                       port_value: 9002
       http2_protocol_options: {}
       name: xds_cluster
+    - connect_timeout: 0.250s
+      type: LOGICAL_DNS
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: admin_interface_cluster
+        endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 9901
+      name: admin_interface_cluster
+  listeners:
+    - name: util_endpoint_listener
+      address:
+        socket_address:
+          address: 0.0.0.0
+          port_value: 9003
+      filter_chains:
+      - filters:
+        - name: envoy.http_connection_manager
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+            stat_prefix: util_endpoint_http
+            http_filters:
+            - name: envoy.filters.http.router
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+            route_config:
+              name: local_admin_interface_route
+              virtual_hosts:
+              - name: admin_interface
+                domains: ["*"]
+                routes:
+                - match:
+                    prefix: /stats
+                  route:
+                    cluster: admin_interface_cluster
+                - match:
+                    prefix: /ready
+                  route:
+                    cluster: admin_interface_cluster
 dynamic_resources:
   cds_config:
     resource_api_version: V3
@@ -53,5 +95,5 @@ admin:
   access_log_path: /dev/null
   address:
     socket_address:
-      address: 0.0.0.0
-      port_value: 9003
+      address: 127.0.0.1
+      port_value: 9901
diff --git a/scheduler/hack/bootstrap.yaml b/scheduler/hack/bootstrap.yaml
@@ -1,4 +1,3 @@
-# Base config for a split xDS management server on 9002, admin port on 9003
 static_resources:
   clusters:
     - connect_timeout: 1s
@@ -53,4 +52,4 @@ admin:
   address:
     socket_address:
       address: 127.0.0.1
-      port_value: 9003
+      port_value: 9901
diff --git a/scheduler/hack/bootstrap_delta.yaml b/scheduler/hack/bootstrap_delta.yaml
@@ -1,4 +1,3 @@
-# Base config for a split xDS management server on 9002, admin port on 9003
 static_resources:
   clusters:
     - connect_timeout: 1s
@@ -53,4 +52,4 @@ admin:
   address:
     socket_address:
       address: 127.0.0.1
-      port_value: 9003
+      port_value: 9901
diff --git a/scheduler/k8s/envoy/envoy.yaml b/scheduler/k8s/envoy/envoy.yaml
@@ -45,6 +45,6 @@ spec:
         ports:
         - name: http
           containerPort: 9000
-        - name: envoy-admin
+        - name: envoy-stats
           containerPort: 9003
       terminationGracePeriodSeconds: 5
diff --git a/scheduler/k8s/envoy/svc.yaml b/scheduler/k8s/envoy/svc.yaml
@@ -11,9 +11,5 @@ spec:
     port: 80
     targetPort: http
     protocol: TCP
-  - name: admin
-    port: 9003
-    targetPort: envoy-admin
-    protocol: TCP
   selector:
     app: seldon-envoy
diff --git a/scheduler/pkg/envoy/resources/resource.go b/scheduler/pkg/envoy/resources/resource.go
@@ -844,7 +844,7 @@ func MakeHTTPListener(listenerName, address string,
 	// HTTP filter configuration
 	manager := &hcm.HttpConnectionManager{
 		CodecType:                    hcm.HttpConnectionManager_AUTO,
-		StatPrefix:                   "http",
+		StatPrefix:                   listenerName,
 		AlwaysSetRequestIdInResponse: false,
 		GenerateRequestId:            &wrappers.BoolValue{Value: false},
 		RouteSpecifier: &hcm.HttpConnectionManager_Rds{