diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 8942392b..8777b7b3 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -120,13 +120,14 @@ func main() { op, err := operator.New( ctx, operator.NewOperatorConfiguration( + operator.WithNamespace(namespace), operator.WithMetricsAddr(metricsAddr), operator.WithHealthProbeAddr(healthProbeAddr), operator.WithPrometheusImage(imgMap["prometheus"]), operator.WithAlertmanagerImage(imgMap["alertmanager"]), operator.WithThanosSidecarImage(imgMap["thanos"]), operator.WithThanosQuerierImage(imgMap["thanos"]), - operator.WithUIPlugins(namespace, imgMap), + operator.WithUIPluginImages(imgMap), operator.WithFeatureGates(operator.FeatureGates{ OpenShift: operator.OpenShiftFeatureGates{ Enabled: openShiftEnabled, diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 96423450..15f53635 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -8,10 +8,13 @@ import ( "path/filepath" "time" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/client-go/kubernetes" + typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -40,6 +43,7 @@ const ( type Operator struct { manager manager.Manager servingCertController *dynamiccertificates.DynamicServingCertificateController + clientCAController *dynamiccertificates.ConfigMapCAController } type OpenShiftFeatureGates struct { @@ -51,6 +55,7 @@ type FeatureGates struct { } type OperatorConfiguration struct { + Namespace string MetricsAddr string HealthProbeAddr string Prometheus stackctrl.PrometheusConfiguration @@ -61,6 +66,13 @@ type OperatorConfiguration struct { FeatureGates FeatureGates } +func WithNamespace(ns string) func(*OperatorConfiguration) { + return func(oc *OperatorConfiguration) { + oc.Namespace = ns + oc.UIPlugins.ResourcesNamespace = ns + } +} + func WithPrometheusImage(image string) func(*OperatorConfiguration) { return func(oc *OperatorConfiguration) { oc.Prometheus.Image = image @@ -97,10 +109,9 @@ func WithHealthProbeAddr(addr string) func(*OperatorConfiguration) { } } -func WithUIPlugins(namespace string, images map[string]string) func(*OperatorConfiguration) { +func WithUIPluginImages(images map[string]string) func(*OperatorConfiguration) { return func(oc *OperatorConfiguration) { oc.UIPlugins.Images = images - oc.UIPlugins.ResourcesNamespace = namespace } } @@ -120,12 +131,16 @@ func NewOperatorConfiguration(opts ...func(*OperatorConfiguration)) *OperatorCon func New(ctx context.Context, cfg *OperatorConfiguration) (*Operator, error) { restConfig := ctrl.GetConfigOrDie() + scheme := NewScheme(cfg) metricsOpts := metricsserver.Options{ BindAddress: cfg.MetricsAddr, } - var servingCertController *dynamiccertificates.DynamicServingCertificateController + var ( + clientCAController *dynamiccertificates.ConfigMapCAController + servingCertController *dynamiccertificates.DynamicServingCertificateController + ) if cfg.FeatureGates.OpenShift.Enabled { // When running in OpenShift, the server uses HTTPS thanks to the // service CA operator. @@ -162,8 +177,7 @@ func New(ctx context.Context, cfg *OperatorConfiguration) (*Operator, error) { return nil, err } - // ConfigMapCAController automatically reloads the client CA. - clientCAProvider, err := dynamiccertificates.NewDynamicCAFromConfigMapController( + clientCAController, err = dynamiccertificates.NewDynamicCAFromConfigMapController( "client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", @@ -174,19 +188,29 @@ func New(ctx context.Context, cfg *OperatorConfiguration) (*Operator, error) { return nil, fmt.Errorf("failed to initialize client CA controller: %w", err) } + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartStructuredLogging(0) + eventBroadcaster.StartRecordingToSink(&typedv1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) + eventRecorder := record.NewEventRecorderAdapter( + eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: "cluster-observability-operator"}), + ) + servingCertController = dynamiccertificates.NewDynamicServingCertificateController( &tls.Config{ - ClientAuth: tls.NoClientCert, + ClientAuth: tls.RequireAndVerifyClientCert, }, - clientCAProvider, + clientCAController, certKeyProvider, nil, - nil, + eventRecorder, ) if err := servingCertController.RunOnce(); err != nil { return nil, fmt.Errorf("failed to initialize serving certificate controller: %w", err) } + clientCAController.AddListener(servingCertController) + certKeyProvider.AddListener(servingCertController) + metricsOpts.SecureServing = true metricsOpts.TLSOpts = []func(*tls.Config){ func(c *tls.Config) { @@ -198,7 +222,7 @@ func New(ctx context.Context, cfg *OperatorConfiguration) (*Operator, error) { mgr, err := ctrl.NewManager( restConfig, ctrl.Options{ - Scheme: NewScheme(cfg), + Scheme: scheme, Metrics: metricsOpts, HealthProbeBindAddress: cfg.HealthProbeAddr, }) @@ -235,10 +259,15 @@ func New(ctx context.Context, cfg *OperatorConfiguration) (*Operator, error) { return &Operator{ manager: mgr, servingCertController: servingCertController, + clientCAController: clientCAController, }, nil } func (o *Operator) Start(ctx context.Context) error { + if o.clientCAController != nil { + go o.clientCAController.Run(ctx, 1) + } + if o.servingCertController != nil { go o.servingCertController.Run(1, ctx.Done()) } diff --git a/test/e2e/framework/assertions.go b/test/e2e/framework/assertions.go index 1e6f9e7a..4332766b 100644 --- a/test/e2e/framework/assertions.go +++ b/test/e2e/framework/assertions.go @@ -345,14 +345,22 @@ func (f *Framework) getPodMetrics(ctx context.Context, pod *v1.Pod, opts ...func tr.TLSClientConfig = &tls.Config{ ServerName: fmt.Sprintf("observability-operator.%s.svc", pod.Namespace), RootCAs: f.RootCA, + GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + fmt.Printf("client cert: %#v\n", f.MetricsClientCert) + return f.MetricsClientCert, nil + }, } resp, err := (&http.Client{Transport: tr}).Do(req) if err != nil { - return nil, fmt.Errorf("failed to get a response from /metrics: %w", err) + return nil, fmt.Errorf("failed to get a response from %q: %w", req.URL.String(), err) } defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("invalid status code from %q: got %d", req.URL.String(), resp.StatusCode) + } + return io.ReadAll(resp.Body) } diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index eee4c6e1..83c378c0 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -3,6 +3,7 @@ package framework import ( "bytes" "context" + "crypto/tls" "crypto/x509" "fmt" "net/http" @@ -34,6 +35,7 @@ type Framework struct { Retain bool IsOpenshiftCluster bool RootCA *x509.CertPool + MetricsClientCert *tls.Certificate OperatorNamespace string } @@ -74,6 +76,32 @@ func (f *Framework) Setup() error { } f.RootCA = rootCA + // Load the prometheus-k8s TLS client certificate. + var s v1.Secret + key = client.ObjectKey{ + Namespace: "openshift-monitoring", + Name: "metrics-client-certs", + } + if err := f.K8sClient.Get(context.Background(), key, &s); err != nil { + return err + } + + cert, found := s.Data["tls.crt"] + if !found { + return errors.New("failed to find TLS client certificate") + } + + k, found := s.Data["tls.key"] + if !found { + return errors.New("failed to find TLS client key") + } + + c, err := tls.X509KeyPair(cert, k) + if err != nil { + return err + } + f.MetricsClientCert = &c + return nil }