From 1ba0e6efec66ac8ed0e65129ebb77c8b09d78111 Mon Sep 17 00:00:00 2001 From: Jayendra Parsai Date: Wed, 11 Oct 2023 17:31:18 +0530 Subject: [PATCH] Recreate missing Secrets for RepositoryCredentials. --- .../argoproj.io/namespace_reconciler.go | 138 ++++++++++++- .../argoproj.io/namespace_reconciler_test.go | 192 +++++++++++++++++- 2 files changed, 317 insertions(+), 13 deletions(-) diff --git a/cluster-agent/controllers/argoproj.io/namespace_reconciler.go b/cluster-agent/controllers/argoproj.io/namespace_reconciler.go index f4dfb52f58..409698fbc4 100644 --- a/cluster-agent/controllers/argoproj.io/namespace_reconciler.go +++ b/cluster-agent/controllers/argoproj.io/namespace_reconciler.go @@ -72,7 +72,8 @@ func (r *ApplicationReconciler) startTimerForNextCycle(ctx context.Context, name cleanOrphanedCRsfromCluster_Operation(ctx, r.DB, r.Client, log) // Recreate Secrets that are required by Applications, but missing from cluster. - recreateClusterSecrets(ctx, r.DB, r.Client, log) + recreateClusterSecrets_ManagedEnvironments(ctx, r.DB, r.Client, log) + recreateClusterSecrets_RepositoryCredentials(ctx, r.DB, r.Client, log) log.Info(fmt.Sprintf("Namespace Reconciler finished an iteration at %s. "+ "Next iteration will be triggered after %v Minutes", time.Now().String(), namespaceReconcilerInterval)) @@ -499,11 +500,11 @@ func cleanOrphanedCRsfromCluster_Operation(ctx context.Context, dbQueries db.Dat } } -// recreateClusterSecrets goes through list of ManagedEnvironments created in cluster and recreates Secrets that are missing from cluster. -func recreateClusterSecrets(ctx context.Context, dbQueries db.DatabaseQueries, k8sClient client.Client, logger logr.Logger) { +// recreateClusterSecrets_ManagedEnvironments goes through list of ManagedEnvironments created in cluster and recreates Secrets that are missing from cluster. +func recreateClusterSecrets_ManagedEnvironments(ctx context.Context, dbQueries db.DatabaseQueries, k8sClient client.Client, logger logr.Logger) { log := logger.WithValues(sharedutil.Log_JobKey, sharedutil.Log_JobKeyValue). - WithValues(sharedutil.Log_JobTypeKey, "CR_Secret_recreate") + WithValues(sharedutil.Log_JobTypeKey, "CR_Secret_recreate_managedEnv") // First get list of ClusterAccess and Application entries from DB. listOfClusterAccessFromDB := getListOfClusterAccessFromTable(ctx, dbQueries, false, log) @@ -512,7 +513,7 @@ func recreateClusterSecrets(ctx context.Context, dbQueries db.DatabaseQueries, k // Now get list of GitopsEngineInstances from DB for the cluster service is running on. listOfGitopsEngineInstanceFromCluster, err := getListOfGitopsEngineInstancesForCurrentCluster(ctx, dbQueries, k8sClient, log) if err != nil { - log.Error(err, "Error occurred in recreateClusterSecrets while fetching GitopsEngineInstances for cluster.") + log.Error(err, "Error occurred in recreateClusterSecrets_ManagedEnvironments while fetching GitopsEngineInstances for cluster.") return } @@ -548,7 +549,7 @@ func recreateClusterSecrets(ctx context.Context, dbQueries db.DatabaseQueries, k } if err := dbQueries.GetManagedEnvironmentById(ctx, &managedEnvironment); err != nil { - log.Error(err, "Error occurred in recreateClusterSecrets while fetching ManagedEnvironment from DB.:"+managedEnvironment.Managedenvironment_id) + log.Error(err, "Error occurred in recreateClusterSecrets_ManagedEnvironments while fetching ManagedEnvironment from DB.:"+managedEnvironment.Managedenvironment_id) continue } @@ -572,7 +573,7 @@ func recreateClusterSecrets(ctx context.Context, dbQueries db.DatabaseQueries, k // Hence created a dummy Cluster User for internal purpose. var specialClusterUser db.ClusterUser if err := dbQueries.GetOrCreateSpecialClusterUser(ctx, &specialClusterUser); err != nil { - log.Error(err, "Error occurred in recreateClusterSecrets while fetching clusterUser.") + log.Error(err, "Error occurred in recreateClusterSecrets_ManagedEnvironments while fetching clusterUser.") return } @@ -588,14 +589,98 @@ func recreateClusterSecrets(ctx context.Context, dbQueries db.DatabaseQueries, k } if _, _, err := operations.CreateOperation(ctx, false, dbOperationInput, specialClusterUser.Clusteruser_id, instance.Namespace_name, dbQueries, k8sClient, log); err != nil { - log.Error(err, "Error occurred in recreateClusterSecrets while creating Operation.") + log.Error(err, "Error occurred in recreateClusterSecrets_ManagedEnvironments while creating Operation.") continue } log.Info("Operation " + dbOperationInput.Operation_id + " is created to create Secret: managed-env-" + managedEnvironment.Managedenvironment_id) } } else { - log.Error(err, "Error occurred in recreateClusterSecrets while fetching Secret:"+secretName+" from Namespace: "+instance.Namespace_name) + log.Error(err, "Error occurred in recreateClusterSecrets_ManagedEnvironments while fetching Secret:"+secretName+" from Namespace: "+instance.Namespace_name) + } + } + } + } + } +} + +// recreateClusterSecrets_RepositoryCredentials goes through list of RepositoryCredentials created in cluster and recreates Secrets that are missing from cluster. +func recreateClusterSecrets_RepositoryCredentials(ctx context.Context, dbQueries db.DatabaseQueries, k8sClient client.Client, logger logr.Logger) { + + log := logger.WithValues(sharedutil.Log_JobKey, sharedutil.Log_JobKeyValue). + WithValues(sharedutil.Log_JobTypeKey, "CR_Secret_recreate_repoCred") + + // First get list of RepositoryCredentials entries from DB. + listOfRepositoryCredentialsFromDB := getListOfRepositoryCredentialsFromTable(ctx, dbQueries, false, log) + + // Now get list of GitopsEngineInstances from DB for the cluster service is running on. + listOfGitopsEngineInstanceFromCluster, err := getListOfGitopsEngineInstancesForCurrentCluster(ctx, dbQueries, k8sClient, log) + if err != nil { + log.Error(err, "Error occurred in recreateClusterSecrets_RepositoryCredentials while fetching GitopsEngineInstances for cluster.") + return + } + + // map: whether we have processed this namespace already. + // key is namespace UID, value is not used. + // - we should only ever process a namespace once per iteration. + namespacesProcessed := map[string]interface{}{} + + for instanceIndex := range listOfGitopsEngineInstanceFromCluster { + // For each GitOpsEngineinstance, read the namespace and check all the Argo CD secrets in that namespace + instance := listOfGitopsEngineInstanceFromCluster[instanceIndex] // To avoid "Implicit memory aliasing in for loop." error. + + // Sanity check: have we processed this Namespace already + if _, exists := namespacesProcessed[instance.Namespace_uid]; exists { + // Log it an skip. There really should never be any GitOpsEngineInstances with matching namespace UID + log.V(logutil.LogLevel_Warn).Error(nil, "there appears to exist multiple GitOpsEngineInstances targeting the same namespace", "namespaceUID", instance.Namespace_uid) + continue + } + namespacesProcessed[instance.Namespace_uid] = nil + + // Iterate through list of RepositoryCredentials entries from DB and find entry using current GitOpsEngineInstance. + for repositoryCredentialsIndex := range listOfRepositoryCredentialsFromDB { + + repositoryCredentials := listOfRepositoryCredentialsFromDB[repositoryCredentialsIndex] // To avoid "Implicit memory aliasing in for loop." error. + + if repositoryCredentials.EngineClusterID == instance.Gitopsengineinstance_id { + // Skip if RepositoryCredentials is created recently + if time.Since(repositoryCredentials.Created_on) < 30*time.Minute { + continue + } + + // Check if Secret used for this RepositoryCredentials is present in GitOpsEngineInstance namespace. + argoSecret := corev1.Secret{} + + if err := k8sClient.Get(ctx, types.NamespacedName{Name: argosharedutil.GenerateArgoCDRepoCredSecretName(repositoryCredentials), Namespace: instance.Namespace_name}, &argoSecret); err != nil { + + // If Secret is not present, then create Operation to recreate the Secret. + if apierr.IsNotFound(err) { + + log.Info("Secret: " + repositoryCredentials.SecretObj + " not found in Namespace:" + instance.Namespace_name + ", recreating it.") + + // Get Special user from DB because we need ClusterUser for creating Operation and we don't have one. + // Hence created a dummy Cluster User for internal purpose. + var specialClusterUser db.ClusterUser + if err := dbQueries.GetOrCreateSpecialClusterUser(ctx, &specialClusterUser); err != nil { + log.Error(err, "Error occurred in recreateClusterSecrets_RepositoryCredentials while fetching clusterUser.") + return + } + + // We need to recreate Secret, to do that create Operation to inform Argo CD about it. + dbOperationInput := db.Operation{ + Instance_id: instance.Gitopsengineinstance_id, + Resource_id: repositoryCredentials.RepositoryCredentialsID, + Resource_type: db.OperationResourceType_RepositoryCredentials, + } + + if _, _, err := operations.CreateOperation(ctx, false, dbOperationInput, specialClusterUser.Clusteruser_id, instance.Namespace_name, dbQueries, k8sClient, log); err != nil { + log.Error(err, "Error occurred in recreateClusterSecrets_RepositoryCredentials while creating Operation.") + continue + } + + log.Info("Operation " + dbOperationInput.Operation_id + " is created to create Secret: " + repositoryCredentials.SecretObj) + } else { + log.Error(err, "Error occurred in recreateClusterSecrets_RepositoryCredentials while fetching Secret:"+repositoryCredentials.SecretObj+" from Namespace: "+instance.Namespace_name) } } } @@ -714,3 +799,38 @@ func getApplicationRunningInManagedEnvironment(applicationList []db.Application, return false, db.Application{} } + +// getListOfRepositoryCredentialsFromTable loops through RepositoryCredentials in database and returns list of user IDs. +func getListOfRepositoryCredentialsFromTable(ctx context.Context, dbQueries db.DatabaseQueries, skipDelay bool, log logr.Logger) []db.RepositoryCredentials { + + offSet := 0 + var listOfRepositoryCredentialsFromDB []db.RepositoryCredentials + + // Continuously iterate and fetch batches until all entries of table processed. + for { + if offSet != 0 && !skipDelay { + time.Sleep(sleepIntervalsOfBatches) + } + + var tempList []db.RepositoryCredentials + + // Fetch ClusterAccess table entries in batch size as configured above.​ + if err := dbQueries.GetRepositoryCredentialsBatch(ctx, &tempList, appRowBatchSize, offSet); err != nil { + log.Error(err, fmt.Sprintf("Error occurred in cleanOrphanedEntriesfromTable_ClusterUser while fetching batch from Offset: %d to %d: ", + offSet, offSet+appRowBatchSize)) + break + } + + // Break the loop if no entries are left in table to be processed. + if len(tempList) == 0 { + break + } + + listOfRepositoryCredentialsFromDB = append(listOfRepositoryCredentialsFromDB, tempList...) + + // Skip processed entries in next iteration + offSet += appRowBatchSize + } + + return listOfRepositoryCredentialsFromDB +} diff --git a/cluster-agent/controllers/argoproj.io/namespace_reconciler_test.go b/cluster-agent/controllers/argoproj.io/namespace_reconciler_test.go index 5dd65bdd32..b6097d299c 100644 --- a/cluster-agent/controllers/argoproj.io/namespace_reconciler_test.go +++ b/cluster-agent/controllers/argoproj.io/namespace_reconciler_test.go @@ -750,7 +750,7 @@ var _ = Describe("Namespace Reconciler Tests.", func() { }) }) - Context("Testing for recreateClusterSecrets function.", func() { + Context("Testing for recreateClusterSecrets_ManagedEnvironments function.", func() { var log logr.Logger var ctx context.Context @@ -879,7 +879,7 @@ var _ = Describe("Namespace Reconciler Tests.", func() { By("Call function to recreate Secret if missing from cluster.") - recreateClusterSecrets(ctx, dbq, k8sClient, log) + recreateClusterSecrets_ManagedEnvironments(ctx, dbq, k8sClient, log) By("Get list of Operations after calling function.") @@ -913,7 +913,7 @@ var _ = Describe("Namespace Reconciler Tests.", func() { By("Call function to recreate Secret if missing from cluster.") - recreateClusterSecrets(ctx, dbq, k8sClient, log) + recreateClusterSecrets_ManagedEnvironments(ctx, dbq, k8sClient, log) By("Get list of Operations after calling function.") @@ -942,7 +942,7 @@ var _ = Describe("Namespace Reconciler Tests.", func() { By("Call function to recreate Secret if missing from cluster.") - recreateClusterSecrets(ctx, dbq, k8sClient, log) + recreateClusterSecrets_ManagedEnvironments(ctx, dbq, k8sClient, log) By("Get list of Operations after calling function.") @@ -951,4 +951,188 @@ var _ = Describe("Namespace Reconciler Tests.", func() { Expect(operation).To(HaveLen(1)) }) }) + + Context("Testing for recreateClusterSecrets_RepositoryCredentials function.", func() { + + var log logr.Logger + var ctx context.Context + var secret corev1.Secret + var dbq db.AllDatabaseQueries + var k8sClient client.WithWatch + var kubeSystemNamepace corev1.Namespace + var repositoryCredentials db.RepositoryCredentials + + BeforeEach(func() { + err := db.SetupForTestingDBGinkgo() + Expect(err).ToNot(HaveOccurred()) + + ctx = context.Background() + log = logger.FromContext(ctx) + + dbq, err = db.NewUnsafePostgresDBQueries(true, true) + Expect(err).ToNot(HaveOccurred()) + + scheme, _, _, _, err := tests.GenericTestSetup() + Expect(err).ToNot(HaveOccurred()) + + err = appv1.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + + By("Create fake kube client.") + + k8sClient = fake.NewClientBuilder().WithScheme(scheme).Build() + + By("Create kube-system namespace.") + + kubeSystemNamepace = corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + UID: "test-" + uuid.NewUUID(), + }, + } + err = k8sClient.Create(ctx, &kubeSystemNamepace) + Expect(err).ToNot(HaveOccurred()) + + By("Create GitopsEngineCluster.") + + gitopsEngineCluster, created, err := dbutil.GetOrCreateGitopsEngineClusterByKubeSystemNamespaceUID(ctx, string(kubeSystemNamepace.UID), dbq, log) + Expect(err).ToNot(HaveOccurred()) + Expect(gitopsEngineCluster).ToNot(BeNil()) + Expect(created).To(BeTrue()) + + By("Create ClusterCredentials entry in DB.") + + clusterCredentials := db.ClusterCredentials{ + Clustercredentials_cred_id: "test-creds-" + string(uuid.NewUUID()), + Host: "host", + Kube_config: "kube-config", + Kube_config_context: "kube-config-context", + Serviceaccount_bearer_token: "serviceaccount_bearer_token", + Serviceaccount_ns: "Serviceaccount_ns", + } + err = dbq.CreateClusterCredentials(ctx, &clusterCredentials) + Expect(err).ToNot(HaveOccurred()) + + By("Create GitopsEngineInstance entry in DB.") + + gitopsEngineInstance := db.GitopsEngineInstance{ + Gitopsengineinstance_id: "test-id" + string(uuid.NewUUID()), + Namespace_name: "test-ns-" + string(uuid.NewUUID()), + Namespace_uid: "test-ns-" + string(uuid.NewUUID()), + EngineCluster_id: gitopsEngineCluster.Gitopsenginecluster_id, + } + err = dbq.CreateGitopsEngineInstance(ctx, &gitopsEngineInstance) + Expect(err).ToNot(HaveOccurred()) + + By("Create RepositoryCredentials entry in DB.") + + repositoryCredentials = db.RepositoryCredentials{ + RepositoryCredentialsID: "test-repo-" + string(uuid.NewUUID()), + UserID: "test-user", + PrivateURL: "https://test-private-url", + AuthUsername: "test-auth-username", + AuthPassword: "test-auth-password", + AuthSSHKey: "test-auth-ssh-key", + SecretObj: "test-secret-obj", + EngineClusterID: gitopsEngineInstance.Gitopsengineinstance_id, + } + err = dbq.CreateRepositoryCredentials(ctx, &repositoryCredentials) + Expect(err).ToNot(HaveOccurred()) + + By("Create Secret CR definition.") + + secret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: argosharedutil.GenerateArgoCDRepoCredSecretName(repositoryCredentials), + Namespace: gitopsEngineInstance.Namespace_name, + }, + } + }) + + It("Should not create Operation for Secret, even if it is missing in Cluster, since RepositoryCredentials is created recently.", func() { + + defer dbq.CloseDatabase() + + By("Get list of Operations before calling function.") + + var operation []db.Operation + + err := dbq.ListOperationsByResourceIdAndTypeAndOwnerId(ctx, repositoryCredentials.RepositoryCredentialsID, db.OperationResourceType_RepositoryCredentials, &operation, db.SpecialClusterUserName) + Expect(err).ToNot(HaveOccurred()) + Expect(operation).To(BeEmpty()) + + By("Call function to recreate Secret if missing from cluster.") + + recreateClusterSecrets_RepositoryCredentials(ctx, dbq, k8sClient, log) + + By("Get list of Operations after calling function.") + + err = dbq.ListOperationsByResourceIdAndTypeAndOwnerId(ctx, repositoryCredentials.RepositoryCredentialsID, db.OperationResourceType_RepositoryCredentials, &operation, db.SpecialClusterUserName) + Expect(err).ToNot(HaveOccurred()) + Expect(operation).To(BeEmpty()) + }) + + It("Should not create Operation for Secret, since it is already present in Cluster.", func() { + + defer dbq.CloseDatabase() + + By("Set 'Created_on' field more than 30 Minutes.") + + repositoryCredentials.Created_on = time.Now().Add(time.Duration(-(31 * time.Minute))) + err := dbq.UpdateRepositoryCredentials(ctx, &repositoryCredentials) + Expect(err).ToNot(HaveOccurred()) + + By("Create Secret CR in cluster.") + + err = k8sClient.Create(ctx, &secret) + Expect(err).ToNot(HaveOccurred()) + + By("Get list of Operations before calling function.") + + var operation []db.Operation + + err = dbq.ListOperationsByResourceIdAndTypeAndOwnerId(ctx, repositoryCredentials.RepositoryCredentialsID, db.OperationResourceType_RepositoryCredentials, &operation, db.SpecialClusterUserName) + Expect(err).ToNot(HaveOccurred()) + Expect(operation).To(BeEmpty()) + + By("Call function to recreate Secret if missing from cluster.") + + recreateClusterSecrets_RepositoryCredentials(ctx, dbq, k8sClient, log) + + By("Get list of Operations after calling function.") + + err = dbq.ListOperationsByResourceIdAndTypeAndOwnerId(ctx, repositoryCredentials.RepositoryCredentialsID, db.OperationResourceType_RepositoryCredentials, &operation, db.SpecialClusterUserName) + Expect(err).ToNot(HaveOccurred()) + Expect(operation).To(BeEmpty()) + }) + + It("Should create Operation to recreate Secret, since it is not present in Cluster.", func() { + + defer dbq.CloseDatabase() + + By("Set 'Created_on' field more than 30 Minutes.") + + repositoryCredentials.Created_on = time.Now().Add(time.Duration(-(31 * time.Minute))) + err := dbq.UpdateRepositoryCredentials(ctx, &repositoryCredentials) + Expect(err).ToNot(HaveOccurred()) + + By("Get list of Operations before calling function.") + + var operation []db.Operation + + err = dbq.ListOperationsByResourceIdAndTypeAndOwnerId(ctx, repositoryCredentials.RepositoryCredentialsID, db.OperationResourceType_RepositoryCredentials, &operation, db.SpecialClusterUserName) + Expect(err).ToNot(HaveOccurred()) + Expect(operation).To(BeEmpty()) + + By("Call function to recreate Secret if missing from cluster.") + + recreateClusterSecrets_RepositoryCredentials(ctx, dbq, k8sClient, log) + + By("Get list of Operations after calling function.") + + err = dbq.ListOperationsByResourceIdAndTypeAndOwnerId(ctx, repositoryCredentials.RepositoryCredentialsID, db.OperationResourceType_RepositoryCredentials, &operation, db.SpecialClusterUserName) + Expect(err).ToNot(HaveOccurred()) + Expect(operation).To(HaveLen(1)) + }) + }) })