From 5801de69918bcd62af9ca56e416fad55b2847acc Mon Sep 17 00:00:00 2001 From: Niall D <4562759+driev@users.noreply.github.com> Date: Tue, 22 Oct 2024 11:24:21 +0100 Subject: [PATCH] feat(envoy): closing off the admin interface (#5936) * closing off the admin interface * update hacks * remove port 9003 from envoy svc * fixing the service monitor --- .../templates/seldon-v2-components.yaml | 4 +- k8s/yaml/components.yaml | 4 +- operator/config/seldonconfigs/default.yaml | 4 +- .../reconcilers/seldon/service_reconciler.go | 4 +- prometheus/monitors/envoy-servicemonitor.yaml | 2 +- scheduler/config/envoy-compose.yaml | 48 +++++++++++++++++-- scheduler/config/envoy-local.yaml | 48 +++++++++++++++++-- scheduler/config/envoy-tls.yaml | 48 +++++++++++++++++-- scheduler/config/envoy.yaml | 48 +++++++++++++++++-- scheduler/hack/bootstrap.yaml | 3 +- scheduler/hack/bootstrap_delta.yaml | 3 +- scheduler/k8s/envoy/envoy.yaml | 2 +- scheduler/k8s/envoy/svc.yaml | 4 -- scheduler/pkg/envoy/resources/resource.go | 2 +- 14 files changed, 193 insertions(+), 31 deletions(-) diff --git a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml index a704782abe..5fbf251709 100644 --- a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml +++ b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml @@ -1369,12 +1369,12 @@ spec: - containerPort: 9000 name: http - containerPort: 9003 - name: envoy-admin + name: envoy-stats readinessProbe: failureThreshold: 3 httpGet: path: /ready - port: envoy-admin + port: envoy-stats initialDelaySeconds: 10 periodSeconds: 5 resources: diff --git a/k8s/yaml/components.yaml b/k8s/yaml/components.yaml index c64a51b82c..a2d5b9dc41 100644 --- a/k8s/yaml/components.yaml +++ b/k8s/yaml/components.yaml @@ -1000,12 +1000,12 @@ spec: - containerPort: 9000 name: http - containerPort: 9003 - name: envoy-admin + name: envoy-stats readinessProbe: failureThreshold: 3 httpGet: path: /ready - port: envoy-admin + port: envoy-stats initialDelaySeconds: 10 periodSeconds: 5 resources: diff --git a/operator/config/seldonconfigs/default.yaml b/operator/config/seldonconfigs/default.yaml index 7fc4c64721..6fa892911d 100644 --- a/operator/config/seldonconfigs/default.yaml +++ b/operator/config/seldonconfigs/default.yaml @@ -58,7 +58,7 @@ spec: - containerPort: 9000 name: http - containerPort: 9003 - name: envoy-admin + name: envoy-stats resources: limits: memory: 128Mi @@ -68,7 +68,7 @@ spec: readinessProbe: httpGet: path: /ready - port: envoy-admin + port: envoy-stats initialDelaySeconds: 10 periodSeconds: 5 failureThreshold: 3 diff --git a/operator/controllers/reconcilers/seldon/service_reconciler.go b/operator/controllers/reconcilers/seldon/service_reconciler.go index 84874cba9a..b00b57cfa9 100644 --- a/operator/controllers/reconcilers/seldon/service_reconciler.go +++ b/operator/controllers/reconcilers/seldon/service_reconciler.go @@ -119,8 +119,8 @@ func getSeldonMeshService(meta metav1.ObjectMeta, serviceConfig mlopsv1alpha1.Se }, { Port: 9003, - TargetPort: intstr.FromString("envoy-admin"), - Name: "admin", + TargetPort: intstr.FromString("envoy-stats"), + Name: "stats", Protocol: v1.ProtocolTCP, }, }, diff --git a/prometheus/monitors/envoy-servicemonitor.yaml b/prometheus/monitors/envoy-servicemonitor.yaml index e2b3e31b32..ee82850069 100644 --- a/prometheus/monitors/envoy-servicemonitor.yaml +++ b/prometheus/monitors/envoy-servicemonitor.yaml @@ -10,6 +10,6 @@ spec: matchNames: [] any: false endpoints: - - port: admin + - port: stats interval: 15s path: /stats/prometheus diff --git a/scheduler/config/envoy-compose.yaml b/scheduler/config/envoy-compose.yaml index 7362005818..d45b70979c 100644 --- a/scheduler/config/envoy-compose.yaml +++ b/scheduler/config/envoy-compose.yaml @@ -1,4 +1,3 @@ -# Base config for a split xDS management server on 9002, admin port on 9003 static_resources: clusters: - connect_timeout: 1s @@ -14,6 +13,49 @@ static_resources: port_value: 9002 http2_protocol_options: {} name: xds_cluster + - connect_timeout: 0.250s + type: LOGICAL_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: admin_interface_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 + name: admin_interface_cluster + listeners: + - name: util_endpoint_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 9003 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: util_endpoint_http + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_admin_interface_route + virtual_hosts: + - name: admin_interface + domains: ["*"] + routes: + - match: + prefix: /stats + route: + cluster: admin_interface_cluster + - match: + prefix: /ready + route: + cluster: admin_interface_cluster dynamic_resources: cds_config: resource_api_version: V3 @@ -53,5 +95,5 @@ admin: access_log_path: /dev/null address: socket_address: - address: 0.0.0.0 - port_value: 9003 + address: 127.0.0.1 + port_value: 9901 diff --git a/scheduler/config/envoy-local.yaml b/scheduler/config/envoy-local.yaml index 2550b835d7..579327175f 100644 --- a/scheduler/config/envoy-local.yaml +++ b/scheduler/config/envoy-local.yaml @@ -1,4 +1,3 @@ -# Base config for a split xDS management server on 9002, admin port on 9003 static_resources: clusters: - connect_timeout: 1s @@ -14,6 +13,49 @@ static_resources: port_value: 9002 http2_protocol_options: {} name: xds_cluster + - connect_timeout: 0.250s + type: LOGICAL_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: admin_interface_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 + name: admin_interface_cluster + listeners: + - name: util_endpoint_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 9003 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: util_endpoint_http + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_admin_interface_route + virtual_hosts: + - name: admin_interface + domains: ["*"] + routes: + - match: + prefix: /stats + route: + cluster: admin_interface_cluster + - match: + prefix: /ready + route: + cluster: admin_interface_cluster dynamic_resources: cds_config: resource_api_version: V3 @@ -53,5 +95,5 @@ admin: access_log_path: /dev/null address: socket_address: - address: 0.0.0.0 - port_value: 9003 + address: 127.0.0.1 + port_value: 9901 diff --git a/scheduler/config/envoy-tls.yaml b/scheduler/config/envoy-tls.yaml index 8a0514e185..948bfc7bd9 100644 --- a/scheduler/config/envoy-tls.yaml +++ b/scheduler/config/envoy-tls.yaml @@ -1,4 +1,3 @@ -# Base config for a split xDS management server on 9002, admin port on 9003 static_resources: clusters: - connect_timeout: 1s @@ -27,6 +26,49 @@ static_resources: name: validation_context_sds sds_config: path: /etc/validation_context_sds_secret.yaml + - connect_timeout: 0.250s + type: LOGICAL_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: admin_interface_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 + name: admin_interface_cluster + listeners: + - name: util_endpoint_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 9003 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: util_endpoint_http + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_admin_interface_route + virtual_hosts: + - name: admin_interface + domains: ["*"] + routes: + - match: + prefix: /stats + route: + cluster: admin_interface_cluster + - match: + prefix: /ready + route: + cluster: admin_interface_cluster dynamic_resources: cds_config: resource_api_version: V3 @@ -66,5 +108,5 @@ admin: access_log_path: /dev/null address: socket_address: - address: 0.0.0.0 - port_value: 9003 + address: 127.0.0.1 + port_value: 9901 diff --git a/scheduler/config/envoy.yaml b/scheduler/config/envoy.yaml index a5a647e219..5275cc8b15 100644 --- a/scheduler/config/envoy.yaml +++ b/scheduler/config/envoy.yaml @@ -1,4 +1,3 @@ -# Base config for a split xDS management server on 9002, admin port on 9003 static_resources: clusters: - connect_timeout: 1s @@ -14,6 +13,49 @@ static_resources: port_value: 9002 http2_protocol_options: {} name: xds_cluster + - connect_timeout: 0.250s + type: LOGICAL_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: admin_interface_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 + name: admin_interface_cluster + listeners: + - name: util_endpoint_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 9003 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: util_endpoint_http + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_admin_interface_route + virtual_hosts: + - name: admin_interface + domains: ["*"] + routes: + - match: + prefix: /stats + route: + cluster: admin_interface_cluster + - match: + prefix: /ready + route: + cluster: admin_interface_cluster dynamic_resources: cds_config: resource_api_version: V3 @@ -53,5 +95,5 @@ admin: access_log_path: /dev/null address: socket_address: - address: 0.0.0.0 - port_value: 9003 + address: 127.0.0.1 + port_value: 9901 diff --git a/scheduler/hack/bootstrap.yaml b/scheduler/hack/bootstrap.yaml index ed735199ad..df0b9bb0d3 100644 --- a/scheduler/hack/bootstrap.yaml +++ b/scheduler/hack/bootstrap.yaml @@ -1,4 +1,3 @@ -# Base config for a split xDS management server on 9002, admin port on 9003 static_resources: clusters: - connect_timeout: 1s @@ -53,4 +52,4 @@ admin: address: socket_address: address: 127.0.0.1 - port_value: 9003 + port_value: 9901 diff --git a/scheduler/hack/bootstrap_delta.yaml b/scheduler/hack/bootstrap_delta.yaml index 2d073395f3..3e6aff03d4 100644 --- a/scheduler/hack/bootstrap_delta.yaml +++ b/scheduler/hack/bootstrap_delta.yaml @@ -1,4 +1,3 @@ -# Base config for a split xDS management server on 9002, admin port on 9003 static_resources: clusters: - connect_timeout: 1s @@ -53,4 +52,4 @@ admin: address: socket_address: address: 127.0.0.1 - port_value: 9003 + port_value: 9901 diff --git a/scheduler/k8s/envoy/envoy.yaml b/scheduler/k8s/envoy/envoy.yaml index 27b8e4bda1..2d8d431add 100644 --- a/scheduler/k8s/envoy/envoy.yaml +++ b/scheduler/k8s/envoy/envoy.yaml @@ -45,6 +45,6 @@ spec: ports: - name: http containerPort: 9000 - - name: envoy-admin + - name: envoy-stats containerPort: 9003 terminationGracePeriodSeconds: 5 diff --git a/scheduler/k8s/envoy/svc.yaml b/scheduler/k8s/envoy/svc.yaml index 6cba090e9f..3829ddbd79 100644 --- a/scheduler/k8s/envoy/svc.yaml +++ b/scheduler/k8s/envoy/svc.yaml @@ -11,9 +11,5 @@ spec: port: 80 targetPort: http protocol: TCP - - name: admin - port: 9003 - targetPort: envoy-admin - protocol: TCP selector: app: seldon-envoy diff --git a/scheduler/pkg/envoy/resources/resource.go b/scheduler/pkg/envoy/resources/resource.go index d2285565af..51c0816e2b 100644 --- a/scheduler/pkg/envoy/resources/resource.go +++ b/scheduler/pkg/envoy/resources/resource.go @@ -844,7 +844,7 @@ func MakeHTTPListener(listenerName, address string, // HTTP filter configuration manager := &hcm.HttpConnectionManager{ CodecType: hcm.HttpConnectionManager_AUTO, - StatPrefix: "http", + StatPrefix: listenerName, AlwaysSetRequestIdInResponse: false, GenerateRequestId: &wrappers.BoolValue{Value: false}, RouteSpecifier: &hcm.HttpConnectionManager_Rds{