Merge branch 'dev'

charts/kratix/templates/distribution.yaml

@@ -349,7 +349,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  WC_IMG: syntasso/kratix-platform-pipeline-adapter:4c9f634941e90291fb5ae93e9fc65bd6aac9ebb3
+  WC_IMG: syntasso/kratix-platform-pipeline-adapter:13524824e7f974f868c1f092c1e8d51f62a11deb
 kind: ConfigMap
 metadata:
   name: kratix-platform-wc-img-config
@@ -410,7 +410,7 @@ spec:
                 configMapKeyRef:
                   key: WC_IMG
                   name: kratix-platform-wc-img-config
-          image: syntasso/kratix-platform:4c9f634941e90291fb5ae93e9fc65bd6aac9ebb3
+          image: syntasso/kratix-platform:13524824e7f974f868c1f092c1e8d51f62a11deb
           livenessProbe:
             httpGet:
               path: /healthz

config/manager/kustomization.yaml

@@ -20,4 +20,4 @@ configMapGenerator:
 images:
 - name: controller
   newName: syntasso/kratix-platform
-  newTag: 4c9f634941e90291fb5ae93e9fc65bd6aac9ebb3
+  newTag: 13524824e7f974f868c1f092c1e8d51f62a11deb

controllers/destination_controller.go

@@ -109,7 +109,10 @@ func (r *DestinationReconciler) createResourcePathWithExample(writer writers.Sta
 	}
 	nsBytes, _ := yaml.Marshal(kratixConfigMap)
 
-	return writer.WriteObject(filepath.Join(resourcesDir, "kratix-canary-configmap.yaml"), nsBytes)
+	return writer.WriteDirWithObjects(writers.PreserveExistingContentsInDir, resourcesDir, platformv1alpha1.Workload{
+		Filepath: "kratix-canary-configmap.yaml",
+		Content:  string(nsBytes),
+	})
 }
 
 func (r *DestinationReconciler) createDependenciesPathWithExample(writer writers.StateStoreWriter) error {
@@ -122,7 +125,10 @@ func (r *DestinationReconciler) createDependenciesPathWithExample(writer writers
 	}
 	nsBytes, _ := yaml.Marshal(kratixNamespace)
 
-	return writer.WriteObject(filepath.Join(dependenciesDir, "kratix-canary-namespace.yaml"), nsBytes)
+	return writer.WriteDirWithObjects(writers.PreserveExistingContentsInDir, dependenciesDir, platformv1alpha1.Workload{
+		Filepath: "kratix-canary-namespace.yaml",
+		Content:  string(nsBytes),
+	})
 }
 
 // SetupWithManager sets up the controller with the Manager.

controllers/scheduler.go

@@ -45,9 +45,10 @@ func (r *Scheduler) ReconcileDestination() error {
 func (r *Scheduler) UpdateWorkPlacement(work *platformv1alpha1.Work, workPlacement *platformv1alpha1.WorkPlacement) error {
 	workPlacement.Spec.Workloads = work.Spec.Workloads
 	if err := r.Client.Update(context.Background(), workPlacement); err != nil {
-		r.Log.Error(err, "Error updating WorkPlacement: "+workPlacement.Name)
+		r.Log.Error(err, "Error updating WorkPlacement", "workplacement", workPlacement.Name)
 		return err
 	}
+	r.Log.Info("Successfully updated WorkPlacement workloads", "workplacement", workPlacement.Name)
 	return nil
 }
 
@@ -131,12 +132,14 @@ func (r *Scheduler) createWorkplacementsForTargetDestinations(work *platformv1al
 
 		if err := r.Client.Create(context.Background(), &workPlacement); err != nil {
 			if errors.IsAlreadyExists(err) {
+				r.Log.Info("WorkPlacement already exists, skipping", "workplacement", workPlacement.Name, "destination", targetDestinationName)
 				continue
 			}
 
-			r.Log.Error(err, "Error creating new WorkPlacement: "+workPlacement.Name)
+			r.Log.Error(err, "Error creating new WorkPlacement", "workplacement", workPlacement.Name)
 			return err
 		}
+		r.Log.Info("WorkPlacement created", "workplacement", workPlacement.Name, "destination", targetDestinationName)
 	}
 	return nil
 }

controllers/workplacement_controller.go

@@ -124,13 +124,10 @@ func (r *WorkPlacementReconciler) deleteWorkPlacement(ctx context.Context, write
 }
 
 func (r *WorkPlacementReconciler) writeWorkloadsToStateStore(writer writers.StateStoreWriter, workPlacement v1alpha1.WorkPlacement, logger logr.Logger) error {
-	for _, workload := range workPlacement.Spec.Workloads {
-		filepath := filepath.Join(getDir(workPlacement), workload.Filepath)
-		err := writer.WriteObject(filepath, []byte(workload.Content))
-		if err != nil {
-			logger.Error(err, "Error writing resources to repository", "filepath", filepath)
-			return err
-		}
+	err := writer.WriteDirWithObjects(writers.DeleteExistingContentsInDir, getDir(workPlacement), workPlacement.Spec.Workloads...)
+	if err != nil {
+		logger.Error(err, "Error writing resources to repository")
+		return err
 	}
 
 	return nil

distribution/kratix.yaml

@@ -804,7 +804,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  WC_IMG: syntasso/kratix-platform-pipeline-adapter:4c9f634941e90291fb5ae93e9fc65bd6aac9ebb3
+  WC_IMG: syntasso/kratix-platform-pipeline-adapter:13524824e7f974f868c1f092c1e8d51f62a11deb
 kind: ConfigMap
 metadata:
   name: kratix-platform-wc-img-config
@@ -865,7 +865,7 @@ spec:
             configMapKeyRef:
               key: WC_IMG
               name: kratix-platform-wc-img-config
-        image: syntasso/kratix-platform:4c9f634941e90291fb5ae93e9fc65bd6aac9ebb3
+        image: syntasso/kratix-platform:13524824e7f974f868c1f092c1e8d51f62a11deb
         livenessProbe:
           httpGet:
             path: /healthz

distribution/single-cluster/install-all-in-one.yaml

@@ -804,7 +804,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  WC_IMG: syntasso/kratix-platform-pipeline-adapter:4c9f634941e90291fb5ae93e9fc65bd6aac9ebb3
+  WC_IMG: syntasso/kratix-platform-pipeline-adapter:13524824e7f974f868c1f092c1e8d51f62a11deb
 kind: ConfigMap
 metadata:
   name: kratix-platform-wc-img-config
@@ -865,7 +865,7 @@ spec:
             configMapKeyRef:
               key: WC_IMG
               name: kratix-platform-wc-img-config
-        image: syntasso/kratix-platform:4c9f634941e90291fb5ae93e9fc65bd6aac9ebb3
+        image: syntasso/kratix-platform:13524824e7f974f868c1f092c1e8d51f62a11deb
         livenessProbe:
           httpGet:
             path: /healthz

lib/writers/git.go

@@ -68,125 +68,139 @@ func NewGitWriter(logger logr.Logger, stateStoreSpec platformv1alpha1.GitStateSt
 	}, nil
 }
 
-func (g *GitWriter) WriteObject(fileName string, toWrite []byte) error {
-	log := g.Log.WithValues(
-		"dir", g.path,
-		"fileName", fileName,
-		"branch", g.gitServer.Branch,
-	)
-	if len(toWrite) == 0 {
-		log.Info("Empty byte[]. Nothing to write to Git")
-		return nil
-	}
-
-	localTmpDir, err := createLocalDirectory()
+func (g *GitWriter) setupLocalDirectoryWithRepo(logger logr.Logger) (string, *git.Repository, *git.Worktree, error) {
+	localTmpDir, err := createLocalDirectory(logger)
 	if err != nil {
-		log.Error(err, "could not create temporary repository directory")
-		return err
+		logger.Error(err, "could not create temporary repository directory")
+		return "", nil, nil, err
 	}
-	defer os.RemoveAll(filepath.Dir(localTmpDir))
 
-	repo, err := g.cloneRepo(localTmpDir)
+	repo, err := g.cloneRepo(localTmpDir, logger)
 	if err != nil {
-		log.Error(err, "could not clone repository")
-		return err
+		logger.Error(err, "could not clone repository")
+		return "", nil, nil, err
 	}
 
 	worktree, err := repo.Worktree()
 	if err != nil {
-		log.Error(err, "could not access repo worktree")
-		return err
+		logger.Error(err, "could not access repo worktree")
+		return "", nil, nil, err
 	}
+	return localTmpDir, repo, worktree, nil
+}
 
-	workTreeFilePath := filepath.Join(g.path, fileName)
-	absoluteFilePath := filepath.Join(localTmpDir, workTreeFilePath)
-
-	//We need to protect against paths containg `..`
-	//filepath.Join expands any '../' in the path to the actual, e.g. /tmp/foo/../ resolves to /tmp/
-	//To ensure they can't write to files on disk outside of the tmp git repostiroy we check the absolute path
-	//returned by `filepath.Join` is still contained with the git repository:
-	// Note: This means `../` can still be used, but only if the end result is still contained within the git repository
-	if !strings.HasPrefix(absoluteFilePath, localTmpDir) {
-		log.Error(nil, "path of file to write is not located within the git repostiory", "absolutePath", absoluteFilePath, "tmpDir", localTmpDir, "repoPath", workTreeFilePath, "path", g.path)
-		return nil //We don't want to retry as this isn't a recoverable error. Log error and return nil.
-	}
+func (g *GitWriter) WriteDirWithObjects(deleteExistingContentsInDir bool, subDir string, toWrite ...platformv1alpha1.Workload) error {
+	dirInGitRepo := filepath.Join(g.path, subDir)
+	logger := g.Log.WithValues(
+		"dir", dirInGitRepo,
+		"branch", g.gitServer.Branch,
+	)
 
-	if os.MkdirAll(filepath.Dir(absoluteFilePath), 0700); err != nil {
-		log.Error(err, "could not generate local directories")
-		return err
+	if len(toWrite) == 0 {
+		logger.Info("Empty workloads. Nothing to write to Git")
+		return nil
 	}
 
-	if err := ioutil.WriteFile(absoluteFilePath, toWrite, 0644); err != nil {
-		log.Error(err, "could not write to file")
+	localTmpDir, repo, worktree, err := g.setupLocalDirectoryWithRepo(logger)
+	if err != nil {
 		return err
 	}
+	defer os.RemoveAll(filepath.Dir(localTmpDir))
 
-	if _, err := worktree.Add(workTreeFilePath); err != nil {
-		log.Error(err, "could not add file to worktree")
-		return err
+	if deleteExistingContentsInDir {
+		logger.Info("checking if any existing directories needs to be deleted")
+		if _, err := worktree.Filesystem.Lstat(dirInGitRepo); err == nil {
+			logger.Info("deleting existing content")
+			if _, err := worktree.Remove(dirInGitRepo); err != nil {
+				logger.Error(err, "could not add directory deletion to worktree", "dir", dirInGitRepo)
+				return err
+			}
+		}
 	}
 
-	if err := g.commitAndPush(repo, worktree, Add, workTreeFilePath, log); err != nil {
-		return err
-	}
+	var filesCommitted []string
+	for _, item := range toWrite {
+		//worker-cluster/resources/<rr-namespace>/<promise-name>/<rr-name>/foo/bar/baz.yaml
+		worktreeFilePath := filepath.Join(dirInGitRepo, item.Filepath)
+		logger := logger.WithValues(
+			"filepath", worktreeFilePath,
+		)
+
+		///tmp/git-dir/worker-cluster/resources/<rr-namespace>/<promise-name>/<rr-name>/foo/bar/baz.yaml
+		absoluteFilePath := filepath.Join(localTmpDir, worktreeFilePath)
+
+		//We need to protect against paths containg `..`
+		//filepath.Join expands any '../' in the path to the actual, e.g. /tmp/foo/../ resolves to /tmp/
+		//To ensure they can't write to files on disk outside of the tmp git repostiroy we check the absolute path
+		//returned by `filepath.Join` is still contained with the git repository:
+		// Note: This means `../` can still be used, but only if the end result is still contained within the git repository
+		if !strings.HasPrefix(absoluteFilePath, localTmpDir) {
+			logger.Error(nil, "path of file to write is not located within the git repostiory", "absolutePath", absoluteFilePath, "tmpDir", localTmpDir)
+			return nil //We don't want to retry as this isn't a recoverable error. Log error and return nil.
+		}
 
-	return nil
-}
+		if os.MkdirAll(filepath.Dir(absoluteFilePath), 0700); err != nil {
+			logger.Error(err, "could not generate local directories")
+			return err
+		}
 
-func (g *GitWriter) RemoveObject(fileName string) error {
-	log := g.Log.WithValues("dir", g.path, "fileName", fileName)
+		if err := os.WriteFile(absoluteFilePath, []byte(item.Content), 0644); err != nil {
+			logger.Error(err, "could not write to file")
+			return err
+		}
 
-	repoPath, err := createLocalDirectory()
-	if err != nil {
-		log.Error(err, "could not create temporary repository directory")
-		return err
+		if _, err := worktree.Add(worktreeFilePath); err != nil {
+			logger.Error(err, "could not add file to worktree")
+			return err
+		}
+		filesCommitted = append(filesCommitted, worktreeFilePath)
 	}
-	defer os.RemoveAll(filepath.Dir(repoPath))
 
-	repo, err := g.cloneRepo(repoPath)
-	if err != nil {
-		log.Error(err, "could not clone repository")
-		return err
-	}
+	return g.commitAndPush(repo, worktree, Add, filesCommitted, logger)
+}
 
-	worktree, err := repo.Worktree()
+func (g *GitWriter) RemoveObject(filePath string) error {
+	logger := g.Log.WithValues("dir", g.path, "filepath", filePath)
+
+	localTmpDir, repo, worktree, err := g.setupLocalDirectoryWithRepo(logger)
 	if err != nil {
-		log.Error(err, "could not access repo worktree")
 		return err
 	}
+	defer os.RemoveAll(filepath.Dir(localTmpDir))
 
-	objectFileName := filepath.Join(g.path, fileName)
-	if _, err := worktree.Filesystem.Lstat(objectFileName); err == nil {
-		if _, err := worktree.Remove(objectFileName); err != nil {
-			log.Error(err, "could not remove file from worktree")
+	worktreeFilepath := filepath.Join(g.path, filePath)
+	if _, err := worktree.Filesystem.Lstat(worktreeFilepath); err == nil {
+		if _, err := worktree.Remove(worktreeFilepath); err != nil {
+			logger.Error(err, "could not remove file from worktree")
 			return err
 		}
-		log.Info("successfully deleted file from worktree")
+		logger.Info("successfully deleted file from worktree")
 	} else {
-		log.Info("file does not exist on worktree, nothing to delete")
+		logger.Info("file does not exist on worktree, nothing to delete")
 		return nil
 	}
 
-	if err := g.commitAndPush(repo, worktree, Delete, objectFileName, log); err != nil {
+	if err := g.commitAndPush(repo, worktree, Delete, []string{worktreeFilepath}, logger); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (g *GitWriter) push(repo *git.Repository, log logr.Logger) error {
+func (g *GitWriter) push(repo *git.Repository, logger logr.Logger) error {
 	err := repo.Push(&git.PushOptions{
 		RemoteName:      "origin",
 		Auth:            g.gitServer.Auth,
 		InsecureSkipTLS: true,
 	})
 	if err != nil {
-		log.Error(err, "could not push to remote")
+		logger.Error(err, "could not push to remote")
 		return err
 	}
 	return nil
 }
 
-func (g *GitWriter) cloneRepo(localRepoFilePath string) (*git.Repository, error) {
+func (g *GitWriter) cloneRepo(localRepoFilePath string, logger logr.Logger) (*git.Repository, error) {
+	logger.Info("cloning repo")
 	return git.PlainClone(localRepoFilePath, false, &git.CloneOptions{
 		Auth:            g.gitServer.Auth,
 		URL:             g.gitServer.URL,
@@ -198,38 +212,43 @@ func (g *GitWriter) cloneRepo(localRepoFilePath string) (*git.Repository, error)
 	})
 }
 
-func (g *GitWriter) commitAndPush(repo *git.Repository, worktree *git.Worktree, action, fileToAdd string, log logr.Logger) error {
+func (g *GitWriter) commitAndPush(repo *git.Repository, worktree *git.Worktree, action string, filesToAdd []string, logger logr.Logger) error {
 	status, err := worktree.Status()
 	if err != nil {
-		log.Error(err, "could not get worktree status")
+		logger.Error(err, "could not get worktree status")
 		return err
 	}
 
 	if status.IsClean() {
-		log.Info("no changes to be committed")
+		logger.Info("no changes to be committed")
 		return nil
 	}
 
 	//should fileToAdd be here at all? is it valuable? specifically the fileToAdd parameter
-	_, err = worktree.Commit(fmt.Sprintf("%s: %s", action, fileToAdd), &git.CommitOptions{
+	logger.Info("commiting changes", "filesAdded", filesToAdd)
+	_, err = worktree.Commit(fmt.Sprintf("%s: %v", action, filesToAdd), &git.CommitOptions{
 		Author: &object.Signature{
 			Name:  g.author.Name,
 			Email: g.author.Email,
 			When:  time.Now(),
 		},
 	})
+
 	if err != nil {
-		log.Error(err, "could not commit file to worktree")
+		logger.Error(err, "could not commit file to worktree")
 		return err
 	}
 
-	if err := g.push(repo, log); err != nil {
+	logger.Info("pushing changes")
+	if err := g.push(repo, logger); err != nil {
+		logger.Error(err, "could not push changes")
 		return err
 	}
 	return nil
 }
 
-func createLocalDirectory() (string, error) {
+func createLocalDirectory(logger logr.Logger) (string, error) {
+	logger.Info("creating local directory")
 	dir, err := ioutil.TempDir("", "kratix-repo")
 	if err != nil {
 		return "", err