From 37cfdc9f10365354691604fcf16dacaf47135c54 Mon Sep 17 00:00:00 2001 From: Jay Jijie Chen <1180092+jijiechen@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:32:27 +0800 Subject: [PATCH] feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107) * feat(kuma-dp): add a readiness handler * feat(kuma-dp): support TERMINATING state * use 9902 to expose dp readiness * support port 0 to disable the dp readiness and use Envoy * support ipv6 address: reuse AdminAddress and make readiness reporter listen on it * don't send 503 when receiving SIGUSR2 --------- Signed-off-by: Jay Chen <1180092+jijiechen@users.noreply.github.com> --- .github/workflows/_e2e.yaml | 11 ++ app/kuma-dp/cmd/run.go | 29 +++++ app/kuma-dp/pkg/dataplane/envoy/envoy.go | 12 ++ .../pkg/dataplane/envoy/remote_bootstrap.go | 10 ++ .../pkg/dataplane/readiness/component.go | 111 +++++++++++++++++ mk/k3d.mk | 1 + pkg/config/app/kuma-dp/config.go | 15 ++- .../testdata/default-config.golden.yaml | 1 + pkg/core/xds/metadata.go | 25 ++++ pkg/core/xds/metadata_test.go | 16 +++ pkg/xds/bootstrap/generator.go | 13 ++ pkg/xds/bootstrap/parameters.go | 28 +++++ pkg/xds/bootstrap/template_v3.go | 5 + pkg/xds/bootstrap/types/bootstrap_request.go | 13 ++ pkg/xds/envoy/names/resource_names.go | 4 + pkg/xds/generator/admin_proxy_generator.go | 58 +++++++-- .../generator/admin_proxy_generator_test.go | 23 +++- .../admin/01.envoy-config.golden.yaml | 5 + .../admin/02.envoy-config.golden.yaml | 5 + .../admin/03.envoy-config.golden.yaml | 5 + .../admin/04.envoy-config.golden.yaml | 5 + .../admin/05.envoy-config.golden.yaml | 5 + .../admin/06.envoy-config.golden.yaml | 115 ++++++++++++++++++ .../admin/07.envoy-config.golden.yaml | 115 ++++++++++++++++++ .../profile-source/1-envoy-config.golden.yaml | 5 + .../profile-source/2-envoy-config.golden.yaml | 5 + .../profile-source/3-envoy-config.golden.yaml | 5 + .../profile-source/4-envoy-config.golden.yaml | 5 + 28 files changed, 635 insertions(+), 15 deletions(-) create mode 100644 app/kuma-dp/pkg/dataplane/readiness/component.go create mode 100644 pkg/xds/generator/testdata/admin/06.envoy-config.golden.yaml create mode 100644 pkg/xds/generator/testdata/admin/07.envoy-config.golden.yaml diff --git a/.github/workflows/_e2e.yaml b/.github/workflows/_e2e.yaml index d1779ce9f5e4..23c10988a16b 100644 --- a/.github/workflows/_e2e.yaml +++ b/.github/workflows/_e2e.yaml @@ -110,3 +110,14 @@ jobs: target="test/e2e" fi make ${MAKE_PARAMETERS} CI=true "${target}" +<<<<<<< HEAD +======= + - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + if: always() + with: + name: e2e-debug-${{ env.E2E_PARAM_TARGET }} + if-no-files-found: ignore + path: | + /tmp/e2e-debug/ + retention-days: ${{ github.event_name == 'pull_request' && 1 || 30 }} +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) diff --git a/app/kuma-dp/cmd/run.go b/app/kuma-dp/cmd/run.go index db401a931205..84f1b25cfe73 100644 --- a/app/kuma-dp/cmd/run.go +++ b/app/kuma-dp/cmd/run.go @@ -16,6 +16,11 @@ import ( "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/meshmetrics" "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/metrics" +<<<<<<< HEAD +======= + "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/probes" + "github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/readiness" +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) kuma_cmd "github.com/kumahq/kuma/pkg/cmd" "github.com/kumahq/kuma/pkg/config" kumadp "github.com/kumahq/kuma/pkg/config/app/kuma-dp" @@ -176,6 +181,7 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { runLog.Info("generating bootstrap configuration") bootstrap, kumaSidecarConfiguration, err := rootCtx.BootstrapGenerator(gracefulCtx, opts.Config.ControlPlane.URL, opts.Config, envoy.BootstrapParams{ +<<<<<<< HEAD Dataplane: opts.Dataplane, DNSPort: cfg.DNS.EnvoyDNSPort, EmptyDNSPort: cfg.DNS.CoreDNSEmptyPort, @@ -186,6 +192,17 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { DynamicMetadata: rootCtx.BootstrapDynamicMetadata, MetricsCertPath: cfg.DataplaneRuntime.Metrics.CertPath, MetricsKeyPath: cfg.DataplaneRuntime.Metrics.KeyPath, +======= + Dataplane: opts.Dataplane, + DNSPort: cfg.DNS.EnvoyDNSPort, + ReadinessPort: cfg.Dataplane.ReadinessPort, + EnvoyVersion: *envoyVersion, + Workdir: cfg.DataplaneRuntime.SocketDir, + DynamicMetadata: rootCtx.BootstrapDynamicMetadata, + MetricsCertPath: cfg.DataplaneRuntime.Metrics.CertPath, + MetricsKeyPath: cfg.DataplaneRuntime.Metrics.KeyPath, + SystemCaPath: cfg.DataplaneRuntime.SystemCaPath, +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) }) if err != nil { return errors.Errorf("Failed to generate Envoy bootstrap config. %v", err) @@ -233,6 +250,15 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { observabilityComponents := setupObservability(kumaSidecarConfiguration, bootstrap, cfg) components = append(components, observabilityComponents...) + + var readinessReporter *readiness.Reporter + if cfg.Dataplane.ReadinessPort > 0 { + readinessReporter = readiness.NewReporter( + bootstrap.GetAdmin().GetAddress().GetSocketAddress().GetAddress(), + cfg.Dataplane.ReadinessPort) + components = append(components, readinessReporter) + } + if err := rootCtx.ComponentManager.Add(components...); err != nil { return err } @@ -262,6 +288,9 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command { if draining { runLog.Info("already drained, exit immediately") } else { + if readinessReporter != nil { + readinessReporter.Terminating() + } runLog.Info("draining Envoy connections") if err := envoyComponent.FailHealthchecks(); err != nil { runLog.Error(err, "could not drain connections") diff --git a/app/kuma-dp/pkg/dataplane/envoy/envoy.go b/app/kuma-dp/pkg/dataplane/envoy/envoy.go index e6b8e488047d..6bcbf93d1da2 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/envoy.go +++ b/app/kuma-dp/pkg/dataplane/envoy/envoy.go @@ -28,6 +28,7 @@ import ( var runLog = core.Log.WithName("kuma-dp").WithName("run").WithName("envoy") type BootstrapParams struct { +<<<<<<< HEAD Dataplane rest.Resource DNSPort uint32 EmptyDNSPort uint32 @@ -38,6 +39,17 @@ type BootstrapParams struct { AccessLogSocketPath string MetricsCertPath string MetricsKeyPath string +======= + Dataplane rest.Resource + DNSPort uint32 + ReadinessPort uint32 + EnvoyVersion EnvoyVersion + DynamicMetadata map[string]string + Workdir string + MetricsCertPath string + MetricsKeyPath string + SystemCaPath string +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) } type BootstrapConfigFactoryFunc func(ctx context.Context, url string, cfg kuma_dp.Config, params BootstrapParams) (*envoy_bootstrap_v3.Bootstrap, *types.KumaSidecarConfiguration, error) diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go index 9387a8951199..3574b366d31e 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go @@ -177,6 +177,7 @@ func (b *remoteBootstrap) requestForBootstrap(ctx context.Context, client *http. KumaDpCompatible: params.EnvoyVersion.KumaDpCompatible, }, }, +<<<<<<< HEAD DynamicMetadata: params.DynamicMetadata, DNSPort: params.DNSPort, EmptyDNSPort: params.EmptyDNSPort, @@ -185,6 +186,15 @@ func (b *remoteBootstrap) requestForBootstrap(ctx context.Context, client *http. Resources: resources, Workdir: params.Workdir, AccessLogSocketPath: params.AccessLogSocketPath, +======= + DynamicMetadata: params.DynamicMetadata, + DNSPort: params.DNSPort, + ReadinessPort: params.ReadinessPort, + OperatingSystem: b.operatingSystem, + Features: b.features, + Resources: resources, + Workdir: params.Workdir, +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) MetricsResources: types.MetricsResources{ SocketPath: params.MetricsSocketPath, CertPath: params.MetricsCertPath, diff --git a/app/kuma-dp/pkg/dataplane/readiness/component.go b/app/kuma-dp/pkg/dataplane/readiness/component.go new file mode 100644 index 000000000000..4544fb31b0c7 --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/readiness/component.go @@ -0,0 +1,111 @@ +package readiness + +import ( + "context" + "fmt" + "net" + "net/http" + "sync/atomic" + "time" + + "github.com/asaskevich/govalidator" + "github.com/bakito/go-log-logr-adapter/adapter" + + "github.com/kumahq/kuma/pkg/core" + "github.com/kumahq/kuma/pkg/core/runtime/component" +) + +const ( + pathPrefixReady = "/ready" + stateReady = "READY" + stateTerminating = "TERMINATING" +) + +// Reporter reports the health status of this Kuma Dataplane Proxy +type Reporter struct { + localListenAddr string + localListenPort uint32 + isTerminating atomic.Bool +} + +var logger = core.Log.WithName("readiness") + +func NewReporter(localIPAddr string, localListenPort uint32) *Reporter { + return &Reporter{ + localListenPort: localListenPort, + localListenAddr: localIPAddr, + } +} + +func (r *Reporter) Start(stop <-chan struct{}) error { + protocol := "tcp" + addr := r.localListenAddr + if govalidator.IsIPv6(addr) { + protocol = "tcp6" + addr = fmt.Sprintf("[%s]", addr) + } + lis, err := net.Listen(protocol, fmt.Sprintf("%s:%d", addr, r.localListenPort)) + if err != nil { + return err + } + + defer func() { + _ = lis.Close() + }() + + logger.Info("starting readiness reporter", "addr", lis.Addr().String()) + + mux := http.NewServeMux() + mux.HandleFunc(pathPrefixReady, r.handleReadiness) + server := &http.Server{ + ReadHeaderTimeout: time.Second, + Handler: mux, + ErrorLog: adapter.ToStd(logger), + } + + errCh := make(chan error) + go func() { + if err := server.Serve(lis); err != nil { + errCh <- err + } + }() + + select { + case err := <-errCh: + return err + case <-stop: + logger.Info("stopping readiness reporter") + return server.Shutdown(context.Background()) + } +} + +func (r *Reporter) Terminating() { + r.isTerminating.Store(true) +} + +func (r *Reporter) handleReadiness(writer http.ResponseWriter, req *http.Request) { + state := stateReady + stateHTTPStatus := http.StatusOK + if r.isTerminating.Load() { + state = stateTerminating + stateHTTPStatus = http.StatusServiceUnavailable + } + + stateBytes := []byte(state) + writer.Header().Set("content-type", "text/plain") + writer.Header().Set("content-length", fmt.Sprintf("%d", len(stateBytes))) + writer.Header().Set("cache-control", "no-cache, max-age=0") + writer.Header().Set("x-powered-by", "kuma-dp") + writer.WriteHeader(stateHTTPStatus) + _, err := writer.Write(stateBytes) + logger.V(1).Info("responding readiness state", "state", state, "client", req.RemoteAddr) + if err != nil { + logger.Info("[WARNING] could not write response", "err", err) + } +} + +func (r *Reporter) NeedLeaderElection() bool { + return false +} + +var _ component.Component = &Reporter{} diff --git a/mk/k3d.mk b/mk/k3d.mk index 7da4ff60887c..4ac9194cda2d 100644 --- a/mk/k3d.mk +++ b/mk/k3d.mk @@ -42,6 +42,7 @@ PORT_PREFIX := $$(($(patsubst 300-%,300+%-1,$(KIND_CLUSTER_NAME:kuma%=300%)))) K3D_NETWORK_CNI ?= flannel K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \ --k3s-arg '--disable=traefik@server:0' \ + --k3s-arg '--kubelet-arg=image-gc-high-threshold=100@server:0' \ --k3s-arg '--disable=servicelb@server:0' \ --volume '$(subst @,\@,$(TOP)/$(KUMA_DIR))/test/framework/deployments:/tmp/deployments@server:0' \ --network kind \ diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index 7c0a1113c7aa..00c70efc21cf 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -25,10 +25,11 @@ var DefaultConfig = func() Config { }, }, Dataplane: Dataplane{ - Mesh: "", - Name: "", // Dataplane name must be set explicitly - DrainTime: config_types.Duration{Duration: 30 * time.Second}, - ProxyType: "dataplane", + Mesh: "", + Name: "", // Dataplane name must be set explicitly + DrainTime: config_types.Duration{Duration: 30 * time.Second}, + ProxyType: "dataplane", + ReadinessPort: 9902, }, DataplaneRuntime: DataplaneRuntime{ BinaryPath: "envoy", @@ -133,6 +134,8 @@ type Dataplane struct { ProxyType string `json:"proxyType,omitempty" envconfig:"kuma_dataplane_proxy_type"` // Drain time for listeners. DrainTime config_types.Duration `json:"drainTime,omitempty" envconfig:"kuma_dataplane_drain_time"` + // Port that exposes kuma-dp readiness status on localhost, set this value to 0 to provide readiness by "/ready" endpoint from Envoy adminAPI + ReadinessPort uint32 `json:"readinessPort,omitempty" envconfig:"kuma_readiness_port"` } func (d *Dataplane) PostProcess() error { @@ -304,6 +307,10 @@ func (d *Dataplane) Validate() error { errs = multierr.Append(errs, errors.Errorf(".DrainTime must be positive")) } + if d.ReadinessPort > 65353 { + return errors.New(".ReadinessPort has to be in [0, 65353] range") + } + return errs } diff --git a/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml b/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml index 22e45f56ae3d..437298233d06 100644 --- a/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml +++ b/pkg/config/app/kuma-dp/testdata/default-config.golden.yaml @@ -8,6 +8,7 @@ controlPlane: dataplane: drainTime: 30s proxyType: dataplane + readinessPort: 9902 dataplaneRuntime: binaryPath: envoy dynamicConfiguration: diff --git a/pkg/core/xds/metadata.go b/pkg/core/xds/metadata.go index e6600f49d41a..ee2423706c43 100644 --- a/pkg/core/xds/metadata.go +++ b/pkg/core/xds/metadata.go @@ -21,6 +21,7 @@ const ( // Supported Envoy node metadata fields. FieldDataplaneAdminPort = "dataplane.admin.port" FieldDataplaneAdminAddress = "dataplane.admin.address" + FieldDataplaneReadinessPort = "dataplane.readinessReporter.port" FieldDataplaneDNSPort = "dataplane.dns.port" FieldDataplaneDNSEmptyPort = "dataplane.dns.empty.port" FieldDataplaneDataplaneResource = "dataplane.resource" @@ -51,6 +52,7 @@ const ( // This way, xDS server will be able to use Envoy node metadata // to generate xDS resources that depend on environment-specific configuration. type DataplaneMetadata struct { +<<<<<<< HEAD Resource model.Resource AdminPort uint32 AdminAddress string @@ -65,6 +67,21 @@ type DataplaneMetadata struct { MetricsSocketPath string MetricsCertPath string MetricsKeyPath string +======= + Resource model.Resource + AdminPort uint32 + AdminAddress string + ReadinessPort uint32 + DNSPort uint32 + DynamicMetadata map[string]string + ProxyType mesh_proto.ProxyType + Version *mesh_proto.Version + Features Features + WorkDir string + MetricsCertPath string + MetricsKeyPath string + SystemCaPath string +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) } // GetDataplaneResource returns the underlying DataplaneResource, if present. @@ -117,6 +134,13 @@ func (m *DataplaneMetadata) GetAdminPort() uint32 { return m.AdminPort } +func (m *DataplaneMetadata) GetReadinessPort() uint32 { + if m == nil { + return 0 + } + return m.ReadinessPort +} + func (m *DataplaneMetadata) GetAdminAddress() string { if m == nil { return "" @@ -165,6 +189,7 @@ func DataplaneMetadataFromXdsMetadata(xdsMetadata *structpb.Struct, tmpDir strin } metadata.AdminPort = uint32Metadata(xdsMetadata, FieldDataplaneAdminPort) metadata.AdminAddress = xdsMetadata.Fields[FieldDataplaneAdminAddress].GetStringValue() + metadata.ReadinessPort = uint32Metadata(xdsMetadata, FieldDataplaneReadinessPort) metadata.DNSPort = uint32Metadata(xdsMetadata, FieldDataplaneDNSPort) metadata.EmptyDNSPort = uint32Metadata(xdsMetadata, FieldDataplaneDNSEmptyPort) if value := xdsMetadata.Fields[FieldDataplaneDataplaneResource]; value != nil { diff --git a/pkg/core/xds/metadata_test.go b/pkg/core/xds/metadata_test.go index 075b456cc289..62e7c64cb4c1 100644 --- a/pkg/core/xds/metadata_test.go +++ b/pkg/core/xds/metadata_test.go @@ -54,7 +54,16 @@ var _ = Describe("DataplaneMetadataFromXdsMetadata", func() { StringValue: "8000", }, }, +<<<<<<< HEAD "dataplane.dns.empty.port": { +======= + "dataplane.readinessReporter.port": { + Kind: &structpb.Value_StringValue{ + StringValue: "9300", + }, + }, + "systemCaPath": { +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) Kind: &structpb.Value_StringValue{ StringValue: "8001", }, @@ -72,11 +81,18 @@ var _ = Describe("DataplaneMetadataFromXdsMetadata", func() { }, }, expected: xds.DataplaneMetadata{ +<<<<<<< HEAD AdminPort: 1234, DNSPort: 8000, EmptyDNSPort: 8001, AccessLogSocketPath: "/tmp/logs", MetricsSocketPath: "/tmp/metrics", +======= + AdminPort: 1234, + DNSPort: 8000, + SystemCaPath: "/etc/certs/cert.pem", + ReadinessPort: 9300, +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) }, }), Entry("should ignore dependencies version provided through metadata if version is not set at all", testCase{ diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 82402df88141..8db9ea91fc2a 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -114,6 +114,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr KumaDpCompatible: request.Version.Envoy.KumaDpCompatible, }, }, +<<<<<<< HEAD DynamicMetadata: request.DynamicMetadata, DNSPort: request.DNSPort, EmptyDNSPort: request.EmptyDNSPort, @@ -125,6 +126,18 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr MetricsSocketPath: metricsSocketPath, MetricsCertPath: request.MetricsResources.CertPath, MetricsKeyPath: request.MetricsResources.KeyPath, +======= + DynamicMetadata: request.DynamicMetadata, + DNSPort: request.DNSPort, + ReadinessPort: request.ReadinessPort, + ProxyType: request.ProxyType, + Features: request.Features, + Resources: request.Resources, + Workdir: request.Workdir, + MetricsCertPath: request.MetricsResources.CertPath, + MetricsKeyPath: request.MetricsResources.KeyPath, + SystemCaPath: request.SystemCaPath, +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) } setAdminPort := func(adminPortFromResource uint32) { diff --git a/pkg/xds/bootstrap/parameters.go b/pkg/xds/bootstrap/parameters.go index 78256d07652e..495bad7088c5 100644 --- a/pkg/xds/bootstrap/parameters.go +++ b/pkg/xds/bootstrap/parameters.go @@ -26,6 +26,7 @@ type AggregateMetricsConfig struct { } type configParameters struct { +<<<<<<< HEAD Id string Service string AdminAddress string @@ -52,4 +53,31 @@ type configParameters struct { Features []string IsGatewayDataplane bool Resources types.ProxyResources +======= + Id string + Service string + AdminAddress string + AdminPort uint32 + ReadinessPort uint32 + AdminAccessLogPath string + XdsHost string + XdsPort uint32 + XdsConnectTimeout time.Duration + Workdir string + MetricsCertPath string + MetricsKeyPath string + DataplaneToken string + DataplaneTokenPath string + DataplaneResource string + CertBytes []byte + Version *mesh_proto.Version + HdsEnabled bool + DynamicMetadata map[string]string + DNSPort uint32 + ProxyType string + Features []string + IsGatewayDataplane bool + Resources types.ProxyResources + SystemCaPath string +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) } diff --git a/pkg/xds/bootstrap/template_v3.go b/pkg/xds/bootstrap/template_v3.go index f2da11bf3e27..d943dea32561 100644 --- a/pkg/xds/bootstrap/template_v3.go +++ b/pkg/xds/bootstrap/template_v3.go @@ -338,8 +338,13 @@ func genConfig(parameters configParameters, proxyConfig xds.Proxy, enableReloada if parameters.DNSPort != 0 { res.Node.Metadata.Fields[core_xds.FieldDataplaneDNSPort] = util_proto.MustNewValueForStruct(strconv.Itoa(int(parameters.DNSPort))) } +<<<<<<< HEAD if parameters.EmptyDNSPort != 0 { res.Node.Metadata.Fields[core_xds.FieldDataplaneDNSEmptyPort] = util_proto.MustNewValueForStruct(strconv.Itoa(int(parameters.EmptyDNSPort))) +======= + if parameters.ReadinessPort != 0 { + res.Node.Metadata.Fields[core_xds.FieldDataplaneReadinessPort] = util_proto.MustNewValueForStruct(strconv.Itoa(int(parameters.ReadinessPort))) +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) } if parameters.ProxyType != "" { res.Node.Metadata.Fields[core_xds.FieldDataplaneProxyType] = util_proto.MustNewValueForStruct(parameters.ProxyType) diff --git a/pkg/xds/bootstrap/types/bootstrap_request.go b/pkg/xds/bootstrap/types/bootstrap_request.go index 7ad742e5bd63..d73695170195 100644 --- a/pkg/xds/bootstrap/types/bootstrap_request.go +++ b/pkg/xds/bootstrap/types/bootstrap_request.go @@ -10,6 +10,7 @@ type BootstrapRequest struct { Host string `json:"-"` Version Version `json:"version"` // CaCert is a PEM-encoded CA cert that DP uses to verify CP +<<<<<<< HEAD CaCert string `json:"caCert"` DynamicMetadata map[string]string `json:"dynamicMetadata"` DNSPort uint32 `json:"dnsPort,omitempty"` @@ -20,6 +21,18 @@ type BootstrapRequest struct { Workdir string `json:"workdir"` AccessLogSocketPath string `json:"accessLogSocketPath"` MetricsResources MetricsResources `json:"metricsResources"` +======= + CaCert string `json:"caCert"` + DynamicMetadata map[string]string `json:"dynamicMetadata"` + DNSPort uint32 `json:"dnsPort,omitempty"` + ReadinessPort uint32 `json:"readinessPort,omitempty"` + OperatingSystem string `json:"operatingSystem"` + Features []string `json:"features"` + Resources ProxyResources `json:"resources"` + Workdir string `json:"workdir"` + MetricsResources MetricsResources `json:"metricsResources"` + SystemCaPath string `json:"systemCaPath"` +>>>>>>> 20208eb60 (feat(kuma-dp): add a separate component to handle kuma-sidecar readiness probes (#11107)) } type Version struct { diff --git a/pkg/xds/envoy/names/resource_names.go b/pkg/xds/envoy/names/resource_names.go index 214c03cb5fd2..30e6dd0da7fa 100644 --- a/pkg/xds/envoy/names/resource_names.go +++ b/pkg/xds/envoy/names/resource_names.go @@ -65,6 +65,10 @@ func GetMetricsHijackerClusterName() string { return Join("kuma", "metrics", "hijacker") } +func GetDPPReadinessClusterName() string { + return Join("kuma", "readiness") +} + func GetInternalClusterNamePrefix() string { return "_" } diff --git a/pkg/xds/generator/admin_proxy_generator.go b/pkg/xds/generator/admin_proxy_generator.go index aec6c33e4dd6..3a5fb5d1ea10 100644 --- a/pkg/xds/generator/admin_proxy_generator.go +++ b/pkg/xds/generator/admin_proxy_generator.go @@ -28,6 +28,10 @@ var staticEndpointPaths = []*envoy_common.StaticEndpointPath{ } var staticTlsEndpointPaths = []*envoy_common.StaticEndpointPath{ + { + Path: "/ready", + RewritePath: "/ready", + }, { Path: "/", RewritePath: "/", @@ -53,6 +57,7 @@ func (g AdminProxyGenerator) Generate(ctx context.Context, _ *core_xds.ResourceS } adminPort := proxy.Metadata.GetAdminPort() + readinessPort := proxy.Metadata.GetReadinessPort() // We assume that Admin API must be available on a loopback interface (while users // can override the default value `127.0.0.1` in the Bootstrap Server section of `kuma-cp` config, // the only reasonable alternatives are `::1`, `0.0.0.0` or `::`). @@ -60,6 +65,7 @@ func (g AdminProxyGenerator) Generate(ctx context.Context, _ *core_xds.ResourceS // since it would allow a malicious user to manipulate that value and use Prometheus endpoint // as a gateway to another host. envoyAdminClusterName := envoy_names.GetEnvoyAdminClusterName() + dppReadinessClusterName := envoy_names.GetDPPReadinessClusterName() adminAddress := proxy.Metadata.GetAdminAddress() if _, ok := adminAddressAllowedValues[adminAddress]; !ok { var allowedAddresses []string @@ -74,7 +80,8 @@ func (g AdminProxyGenerator) Generate(ctx context.Context, _ *core_xds.ResourceS case "::": adminAddress = "::1" } - cluster, err := envoy_clusters.NewClusterBuilder(proxy.APIVersion, envoyAdminClusterName). + + envoyAdminCluster, err := envoy_clusters.NewClusterBuilder(proxy.APIVersion, envoyAdminClusterName). Configure(envoy_clusters.ProvidedEndpointCluster( govalidator.IsIPv6(adminAddress), core_xds.Endpoint{Target: adminAddress, Port: adminPort})). @@ -84,12 +91,30 @@ func (g AdminProxyGenerator) Generate(ctx context.Context, _ *core_xds.ResourceS return nil, err } - resources := core_xds.NewResourceSet() + assignReadinessPort := func(se *envoy_common.StaticEndpointPath) { + if readinessPort > 0 { + // we only have /ready for now, so assign it to the readiness cluster directly + se.ClusterName = dppReadinessClusterName + } else { + // we keep the previous behavior if readinessPort is not set + // this can happen when an existing DPP is connecting to this CP, it does not have this metadata + se.ClusterName = envoyAdminClusterName + } + } for _, se := range staticEndpointPaths { - se.ClusterName = envoyAdminClusterName + assignReadinessPort(se) + } + for _, se := range staticTlsEndpointPaths { + switch se.Path { + case "/ready": + assignReadinessPort(se) + default: + se.ClusterName = envoyAdminClusterName + } } + resources := core_xds.NewResourceSet() // We bind admin to 127.0.0.1 by default, creating another listener with same address and port will result in error. if g.getAddress(proxy) != adminAddress { filterChains := []envoy_listeners.ListenerBuilderOpt{ @@ -97,9 +122,6 @@ func (g AdminProxyGenerator) Generate(ctx context.Context, _ *core_xds.ResourceS Configure(envoy_listeners.StaticEndpoints(envoy_names.GetAdminListenerName(), staticEndpointPaths)), ), } - for _, se := range staticTlsEndpointPaths { - se.ClusterName = envoyAdminClusterName - } filterChains = append(filterChains, envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(proxy.APIVersion, envoy_common.AnonymousResource). Configure(envoy_listeners.MatchTransportProtocol("tls")). Configure(envoy_listeners.StaticEndpoints(envoy_names.GetAdminListenerName(), staticTlsEndpointPaths)). @@ -122,10 +144,30 @@ func (g AdminProxyGenerator) Generate(ctx context.Context, _ *core_xds.ResourceS } resources.Add(&core_xds.Resource{ - Name: cluster.GetName(), + Name: envoyAdminCluster.GetName(), Origin: OriginAdmin, - Resource: cluster, + Resource: envoyAdminCluster, }) + + if readinessPort > 0 { + adminAddr := proxy.Metadata.GetAdminAddress() + readinessCluster, err := envoy_clusters.NewClusterBuilder(proxy.APIVersion, dppReadinessClusterName). + Configure(envoy_clusters.ProvidedEndpointCluster( + govalidator.IsIPv6(adminAddr), + core_xds.Endpoint{Target: adminAddr, Port: readinessPort})). + Configure(envoy_clusters.DefaultTimeout()). + Build() + if err != nil { + return nil, err + } + + resources.Add(&core_xds.Resource{ + Name: readinessCluster.GetName(), + Origin: OriginAdmin, + Resource: readinessCluster, + }) + } + return resources, nil } diff --git a/pkg/xds/generator/admin_proxy_generator_test.go b/pkg/xds/generator/admin_proxy_generator_test.go index 694e90de1e30..f9fe4e021f58 100644 --- a/pkg/xds/generator/admin_proxy_generator_test.go +++ b/pkg/xds/generator/admin_proxy_generator_test.go @@ -26,6 +26,7 @@ var _ = Describe("AdminProxyGenerator", func() { dataplaneFile string expected string adminAddress string + readinessPort uint32 } DescribeTable("should generate envoy config", @@ -49,9 +50,11 @@ var _ = Describe("AdminProxyGenerator", func() { } proxy := &xds.Proxy{ + Id: *xds.BuildProxyId("default", "test-admin-dpp"), Metadata: &xds.DataplaneMetadata{ - AdminPort: 9901, - AdminAddress: given.adminAddress, + AdminPort: 9901, + AdminAddress: given.adminAddress, + ReadinessPort: given.readinessPort, }, EnvoyAdminMTLSCerts: xds.ServerSideMTLSCerts{ CaPEM: []byte("caPEM"), @@ -93,16 +96,29 @@ var _ = Describe("AdminProxyGenerator", func() { expected: "03.envoy-config.golden.yaml", adminAddress: "::1", }), - Entry("should generate admin resources, unspecified IPv4", testCase{ + Entry("should generate admin resources, unspecified IPv4, readiness port 0", testCase{ dataplaneFile: "04.dataplane.input.yaml", expected: "04.envoy-config.golden.yaml", adminAddress: "0.0.0.0", + readinessPort: 0, }), Entry("should generate admin resources, unspecified IPv6", testCase{ dataplaneFile: "05.dataplane.input.yaml", expected: "05.envoy-config.golden.yaml", adminAddress: "::", }), + Entry("should generate admin resources, IPv4 with readiness port 9902", testCase{ + dataplaneFile: "04.dataplane.input.yaml", + expected: "06.envoy-config.golden.yaml", + adminAddress: "127.0.0.1", + readinessPort: 9902, + }), + Entry("should generate admin resources, IPv6 with readiness port 9400", testCase{ + dataplaneFile: "05.dataplane.input.yaml", + expected: "07.envoy-config.golden.yaml", + adminAddress: "::1", + readinessPort: 9400, + }), ) It("should return error when admin address is not allowed", func() { @@ -117,6 +133,7 @@ var _ = Describe("AdminProxyGenerator", func() { } proxy := &xds.Proxy{ + Id: *xds.BuildProxyId("default", "test-admin-dpp"), Metadata: &xds.DataplaneMetadata{ AdminPort: 9901, AdminAddress: "192.168.0.1", // it's not allowed to use such address diff --git a/pkg/xds/generator/testdata/admin/01.envoy-config.golden.yaml b/pkg/xds/generator/testdata/admin/01.envoy-config.golden.yaml index a92bf4b35d52..b7f238467ac0 100644 --- a/pkg/xds/generator/testdata/admin/01.envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/admin/01.envoy-config.golden.yaml @@ -62,6 +62,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: diff --git a/pkg/xds/generator/testdata/admin/02.envoy-config.golden.yaml b/pkg/xds/generator/testdata/admin/02.envoy-config.golden.yaml index a92bf4b35d52..b7f238467ac0 100644 --- a/pkg/xds/generator/testdata/admin/02.envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/admin/02.envoy-config.golden.yaml @@ -62,6 +62,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: diff --git a/pkg/xds/generator/testdata/admin/03.envoy-config.golden.yaml b/pkg/xds/generator/testdata/admin/03.envoy-config.golden.yaml index 5690585acbd9..2a614cd23da0 100644 --- a/pkg/xds/generator/testdata/admin/03.envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/admin/03.envoy-config.golden.yaml @@ -62,6 +62,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: diff --git a/pkg/xds/generator/testdata/admin/04.envoy-config.golden.yaml b/pkg/xds/generator/testdata/admin/04.envoy-config.golden.yaml index a92bf4b35d52..b7f238467ac0 100644 --- a/pkg/xds/generator/testdata/admin/04.envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/admin/04.envoy-config.golden.yaml @@ -62,6 +62,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: diff --git a/pkg/xds/generator/testdata/admin/05.envoy-config.golden.yaml b/pkg/xds/generator/testdata/admin/05.envoy-config.golden.yaml index 5690585acbd9..2a614cd23da0 100644 --- a/pkg/xds/generator/testdata/admin/05.envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/admin/05.envoy-config.golden.yaml @@ -62,6 +62,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: diff --git a/pkg/xds/generator/testdata/admin/06.envoy-config.golden.yaml b/pkg/xds/generator/testdata/admin/06.envoy-config.golden.yaml new file mode 100644 index 000000000000..2a87f2d228e6 --- /dev/null +++ b/pkg/xds/generator/testdata/admin/06.envoy-config.golden.yaml @@ -0,0 +1,115 @@ +resources: +- name: kuma:envoy:admin + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + altStatName: kuma_envoy_admin + connectTimeout: 5s + loadAssignment: + clusterName: kuma:envoy:admin + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 9901 + name: kuma:envoy:admin + type: STATIC +- name: kuma:readiness + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + altStatName: kuma_readiness + connectTimeout: 5s + loadAssignment: + clusterName: kuma:readiness + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 127.0.0.1 + portValue: 9902 + name: kuma:readiness + type: STATIC +- name: kuma:envoy:admin + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 192.168.0.1 + portValue: 9901 + enableReusePort: false + filterChains: + - filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + routeConfig: + validateClusters: false + virtualHosts: + - domains: + - '*' + name: kuma:envoy:admin + routes: + - match: + prefix: /ready + route: + cluster: kuma:readiness + prefixRewrite: /ready + statPrefix: kuma_envoy_admin + - filterChainMatch: + transportProtocol: tls + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + routeConfig: + validateClusters: false + virtualHosts: + - domains: + - '*' + name: kuma:envoy:admin + routes: + - match: + prefix: /ready + route: + cluster: kuma:readiness + prefixRewrite: /ready + - match: + prefix: / + route: + cluster: kuma:envoy:admin + prefixRewrite: / + statPrefix: kuma_envoy_admin + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + commonTlsContext: + tlsCertificates: + - certificateChain: + inlineBytes: Y2VydFBFTQ== + privateKey: + inlineBytes: a2V5UEVN + validationContext: + matchTypedSubjectAltNames: + - matcher: + exact: kuma-cp + sanType: DNS + trustedCa: + inlineBytes: Y2FQRU0= + requireClientCertificate: true + listenerFilters: + - name: envoy.filters.listener.tls_inspector + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + name: kuma:envoy:admin + trafficDirection: INBOUND diff --git a/pkg/xds/generator/testdata/admin/07.envoy-config.golden.yaml b/pkg/xds/generator/testdata/admin/07.envoy-config.golden.yaml new file mode 100644 index 000000000000..5bc73a119b25 --- /dev/null +++ b/pkg/xds/generator/testdata/admin/07.envoy-config.golden.yaml @@ -0,0 +1,115 @@ +resources: +- name: kuma:envoy:admin + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + altStatName: kuma_envoy_admin + connectTimeout: 5s + loadAssignment: + clusterName: kuma:envoy:admin + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: ::1 + portValue: 9901 + name: kuma:envoy:admin + type: STATIC +- name: kuma:readiness + resource: + '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + altStatName: kuma_readiness + connectTimeout: 5s + loadAssignment: + clusterName: kuma:readiness + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: ::1 + portValue: 9400 + name: kuma:readiness + type: STATIC +- name: kuma:envoy:admin + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 192.168.0.1 + portValue: 9901 + enableReusePort: false + filterChains: + - filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + routeConfig: + validateClusters: false + virtualHosts: + - domains: + - '*' + name: kuma:envoy:admin + routes: + - match: + prefix: /ready + route: + cluster: kuma:readiness + prefixRewrite: /ready + statPrefix: kuma_envoy_admin + - filterChainMatch: + transportProtocol: tls + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + routeConfig: + validateClusters: false + virtualHosts: + - domains: + - '*' + name: kuma:envoy:admin + routes: + - match: + prefix: /ready + route: + cluster: kuma:readiness + prefixRewrite: /ready + - match: + prefix: / + route: + cluster: kuma:envoy:admin + prefixRewrite: / + statPrefix: kuma_envoy_admin + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + commonTlsContext: + tlsCertificates: + - certificateChain: + inlineBytes: Y2VydFBFTQ== + privateKey: + inlineBytes: a2V5UEVN + validationContext: + matchTypedSubjectAltNames: + - matcher: + exact: kuma-cp + sanType: DNS + trustedCa: + inlineBytes: Y2FQRU0= + requireClientCertificate: true + listenerFilters: + - name: envoy.filters.listener.tls_inspector + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + name: kuma:envoy:admin + trafficDirection: INBOUND diff --git a/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml index 2824f329f05d..616c46d556e2 100644 --- a/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml @@ -243,6 +243,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: diff --git a/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml index acee53a91a0c..9457362fb2d2 100644 --- a/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml @@ -286,6 +286,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: diff --git a/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml index 18152ee0b7c9..6ff6067f2b85 100644 --- a/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml @@ -288,6 +288,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: diff --git a/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml index c8ad647c8892..88fe0fcbe5a3 100644 --- a/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml @@ -331,6 +331,11 @@ resources: - '*' name: kuma:envoy:admin routes: + - match: + prefix: /ready + route: + cluster: kuma:envoy:admin + prefixRewrite: /ready - match: prefix: / route: