From 69ce9639371c02a7623dd2b74e485dd5294d8bc3 Mon Sep 17 00:00:00 2001 From: rksharma95 Date: Mon, 20 May 2024 21:20:43 +0530 Subject: [PATCH] fix tls rotation logic for controller Signed-off-by: rksharma95 --- .../internal/controller/cluster.go | 2 +- .../internal/controller/resources.go | 147 ++++++------------ 2 files changed, 47 insertions(+), 102 deletions(-) diff --git a/pkg/KubeArmorOperator/internal/controller/cluster.go b/pkg/KubeArmorOperator/internal/controller/cluster.go index b0fc1945e..2f68935c3 100644 --- a/pkg/KubeArmorOperator/internal/controller/cluster.go +++ b/pkg/KubeArmorOperator/internal/controller/cluster.go @@ -701,7 +701,7 @@ func (clusterWatcher *ClusterWatcher) DeleteAllTlsSecrets() error { for _, secret := range tlsSecrets { err := clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Delete(context.Background(), secret, v1.DeleteOptions{}) if err != nil { - clusterWatcher.Log.Errorf("error while deleing secret: %s", secret) + clusterWatcher.Log.Errorf("error while deleing secret: %s, error=%s", secret, err.Error()) return err } } diff --git a/pkg/KubeArmorOperator/internal/controller/resources.go b/pkg/KubeArmorOperator/internal/controller/resources.go index aa0b23ce4..5c339c3e4 100644 --- a/pkg/KubeArmorOperator/internal/controller/resources.go +++ b/pkg/KubeArmorOperator/internal/controller/resources.go @@ -76,6 +76,8 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs, vols = append(vols, common.KubeArmorCaVolume...) volMnts = append(volMnts, common.KubeArmorCaVolumeMount...) common.AddOrReplaceArg("-tlsEnabled=true", "-tlsEnabled=false", &daemonset.Spec.Template.Spec.Containers[0].Args) + } else { + common.AddOrReplaceArg("-tlsEnabled=false", "-tlsEnabled=true", &daemonset.Spec.Template.Spec.Containers[0].Args) } daemonset.Spec.Template.Spec.Volumes = vols daemonset.Spec.Template.Spec.InitContainers[0].VolumeMounts = commonVolMnts @@ -426,7 +428,6 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { var caCert, tlsCrt, tlsKey *bytes.Buffer var kGenErr, err, installErr error RotateTls := false - FirstRun := true srvAccs := []*corev1.ServiceAccount{ addOwnership(deployments.GetServiceAccount(common.Namespace)).(*corev1.ServiceAccount), addOwnership(deployments.GetRelayServiceAccount(common.Namespace)).(*corev1.ServiceAccount), @@ -486,6 +487,8 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { relayServer.Spec.Template.Spec.Volumes = append(relayServer.Spec.Template.Spec.Volumes, common.KubeArmorRelayTlsVolume...) common.AddOrReplaceArg("-tlsEnabled=true", "-tlsEnabled=false", &relayServer.Spec.Template.Spec.Containers[0].Args) + } else { + common.AddOrReplaceArg("-tlsEnabled=false", "-tlsEnabled=true", &relayServer.Spec.Template.Spec.Containers[0].Args) } // update images @@ -631,10 +634,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { installErr = err clusterWatcher.Log.Warnf("Cannot create secret %s, error=%s", secret.Name, err.Error()) } else { - RotateTls = true && !FirstRun - if !FirstRun { - clusterWatcher.Log.Warnf("Secret was created, rotating TLS secrets") - } + caInK8sSecret = secret.Data["ca.crt"] } } else if err == nil { caInK8sSecret = s.Data["ca.crt"] @@ -672,10 +672,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { } else if err == nil { if !bytes.Equal(hook.Webhooks[0].ClientConfig.CABundle, caInK8sSecret) { // rotate - RotateTls = true && !FirstRun - if !FirstRun { - clusterWatcher.Log.Warnf("mutation CA cert does not match secret CA cert, rotating tls secrets") - } + RotateTls = true } } else { installErr = err @@ -698,107 +695,55 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() { clusterWatcher.RotateTlsCerts() RotateTls = false } - if FirstRun { - FirstRun = false - } time.Sleep(10 * time.Second) } } func (clusterWatcher *ClusterWatcher) RotateTlsCerts() { - var suffix string var caCert, tlsCrt, tlsKey *bytes.Buffer var err error - retries := 0 - for { - if retries == 3 { - return - } - retries++ - suffix, err = common.GetFreeRandSuffix(clusterWatcher.Client, common.Namespace) - if err == nil { - clusterWatcher.Log.Infof("Using suffix %s for all new temorary resources", suffix) - break - } - clusterWatcher.Log.Infof("Cannot find a suffix, err=%s, retrying in 3 seconds...", err.Error()) - time.Sleep(3 * time.Second) - } - serviceName := deployments.KubeArmorControllerWebhookServiceName + "-" + suffix - for { - caCert, tlsCrt, tlsKey, err = common.GeneratePki(common.Namespace, serviceName) - if err == nil { - break - } - clusterWatcher.Log.Infof("Couldn't generate TLS secret, retrying in 3 seconds") - time.Sleep(3 * time.Second) - } - tmpsecret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String()) - tmpsecret = addOwnership(tmpsecret).(*corev1.Secret) - tmpsecret.Name = tmpsecret.GetName() + "-" + suffix - _, err = clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Create(context.Background(), tmpsecret, metav1.CreateOptions{}) + + origdeploy, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{}) if err != nil { - clusterWatcher.Log.Warnf("Cannot create secret %s, error=%s", tmpsecret.Name, err.Error()) - } - tmpdeploy := deployments.GetKubeArmorControllerDeployment(common.Namespace) - tmpdeploy = addOwnership(tmpdeploy).(*appsv1.Deployment) - tmpdeploy.Name = tmpdeploy.GetName() + "-" + suffix - for i, s := range tmpdeploy.Spec.Template.Spec.Volumes { - if s.Name == "cert" { - s.Secret.SecretName = tmpsecret.GetName() - tmpdeploy.Spec.Template.Spec.Volumes[i] = s - break - } + clusterWatcher.Log.Warnf("cannot get controller deployment, error=%s", err.Error()) } - selectLabels := tmpdeploy.Spec.Selector.MatchLabels - selectLabels["kubearmor-app"] = suffix - tmpdeploy.Spec.Selector.MatchLabels = selectLabels - origdeploy, _ := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{}) - origdeploy = addOwnership(origdeploy).(*appsv1.Deployment) - tmpdeploy.Spec.Replicas = origdeploy.Spec.Replicas - if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Create(context.Background(), tmpdeploy, metav1.CreateOptions{}); err != nil { - clusterWatcher.Log.Warnf("Cannot create deployment %s, error=%s", tmpdeploy.Name, err.Error()) - } - - time.Sleep(10 * time.Second) - - tmpservice := deployments.GetKubeArmorControllerWebhookService(common.Namespace) - tmpservice = addOwnership(tmpservice).(*corev1.Service) - tmpservice.Name = serviceName - tmpservice.Spec.Selector = selectLabels - if _, err := clusterWatcher.Client.CoreV1().Services(common.Namespace).Create(context.Background(), tmpservice, metav1.CreateOptions{}); err != nil { - clusterWatcher.Log.Warnf("Cannot create deployment %s, error=%s", tmpservice.Name, err.Error()) - } - tmpmutation := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes()) - mutationName := tmpmutation.Name - tmpmutation = addOwnership(tmpmutation).(*v1.MutatingWebhookConfiguration) - tmpmutation.Name = tmpmutation.Name + "-" + suffix - tmpmutation.Webhooks[0].ClientConfig.Service.Name = tmpservice.GetName() - if _, err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), tmpmutation, metav1.CreateOptions{}); err != nil { - clusterWatcher.Log.Warnf("Cannot create mutation webhook %s, error=%s", tmpmutation.Name, err.Error()) - } - clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), mutationName, metav1.DeleteOptions{}) + caCert, tlsCrt, tlsKey, _ = common.GeneratePki(common.Namespace, deployments.KubeArmorControllerWebhookServiceName) - secret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String()) - secret = addOwnership(secret).(*corev1.Secret) - clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Update(context.Background(), secret, metav1.UpdateOptions{}) - - replicas := int32(0) - origdeploy.Spec.Replicas = &replicas - clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), origdeploy, metav1.UpdateOptions{}) - time.Sleep(10 * time.Second) - origdeploy, _ = clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{}) - origdeploy = addOwnership(origdeploy).(*appsv1.Deployment) - origdeploy.Spec.Replicas = tmpdeploy.Spec.Replicas - - clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), origdeploy, metav1.UpdateOptions{}) - mutation := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes()) - mutation = addOwnership(mutation).(*v1.MutatingWebhookConfiguration) - - clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), mutation, metav1.CreateOptions{}) - - clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), tmpmutation.Name, metav1.DeleteOptions{}) - clusterWatcher.Client.CoreV1().Services(common.Namespace).Delete(context.Background(), tmpservice.Name, metav1.DeleteOptions{}) - clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Delete(context.Background(), tmpdeploy.Name, metav1.DeleteOptions{}) - clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Delete(context.Background(), tmpsecret.Name, metav1.DeleteOptions{}) + replicas := origdeploy.Spec.Replicas + + // TODO: Keep CA certificate in k8s secret + + // == CLEANUP == + // scale down controller deployment to 0 + controllerDeployment := deployments.GetKubeArmorControllerDeployment(common.Namespace) + controllerDeployment = addOwnership(controllerDeployment).(*appsv1.Deployment) + zeroReplicas := int32(0) + controllerDeployment.Spec.Replicas = &zeroReplicas + if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), controllerDeployment, metav1.UpdateOptions{}); err != nil { + clusterWatcher.Log.Warnf("cannot scale down controller %s, error=%s", controllerDeployment.Name, err.Error()) + } + // delete mutation webhook configuration + mutationWebhook := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes()) + mutationWebhook = addOwnership(mutationWebhook).(*v1.MutatingWebhookConfiguration) + if err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), mutationWebhook.Name, metav1.DeleteOptions{}); err != nil { + clusterWatcher.Log.Warnf("cannot delete mutation webhook %s, error=%s", mutationWebhook.Name, err.Error()) + } + // == ROTATE == + // update controller tls secret + controllerSecret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String()) + controllerSecret = addOwnership(controllerSecret).(*corev1.Secret) + if _, err := clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Update(context.Background(), controllerSecret, metav1.UpdateOptions{}); err != nil { + clusterWatcher.Log.Warnf("cannot update controller tls secret %s, error=%s", controllerSecret.Name, err.Error()) + } + // == ROLLOUT == + // create mutation webhook configuration + if _, err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), mutationWebhook, metav1.CreateOptions{}); err != nil { + clusterWatcher.Log.Warnf("Cannot create mutation webhook %s, error=%s", mutationWebhook.Name, err.Error()) + } + // scale up controller deployment to previous settings + controllerDeployment.Spec.Replicas = replicas + if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), controllerDeployment, metav1.UpdateOptions{}); err != nil { + clusterWatcher.Log.Warnf("cannot scale down controller %s, error=%s", controllerDeployment.Name, err.Error()) + } clusterWatcher.Log.Info("Tls rotation completed") }