Add ability to add custom manifests to the rendered output (#704)

* Explain kev runConfig type fields

* Simplify RenderWithConvertor method args

* Adds new render command flag for passing additional manifests to the rendered set

* Pass additionalManifests to the Render method

* Handle additional manifests passed with kev render --additional-manifest flag and add them to the rendered output

* go mod

* Update ci config

.circleci/config.yml

@@ -9,7 +9,7 @@ jobs:
   build:
     executor:
       name: go/default
-      tag: '1.14'
+      tag: '1.19'
     steps:
      - checkout
      - go/load-cache

cmd/kev/cmd/render.go

@@ -70,6 +70,13 @@ func init() {
 		"Target environment for which deployment files should be rendered",
 	)
 
+	flags.StringSliceP(
+		"additional-manifests",
+		"a",
+		[]string{},
+		"Additional Kubernetes manifests to be included in the output",
+	)
+
 	rootCmd.AddCommand(renderCmd)
 }
 
@@ -79,6 +86,7 @@ func runRenderCmd(cmd *cobra.Command, _ []string) error {
 	dir, _ := cmd.Flags().GetString("dir")
 	envs, _ := cmd.Flags().GetStringSlice("environment")
 	verbose, _ := cmd.Root().Flags().GetBool("verbose")
+	additionalManifests, _ := cmd.Flags().GetStringSlice("additional-manifests")
 
 	// The working directory is always the current directory.
 	// This ensures created manifest yaml entries are portable between users and require no path fixing.
@@ -88,6 +96,7 @@ func runRenderCmd(cmd *cobra.Command, _ []string) error {
 		kev.WithAppName(rootCmd.Use),
 		kev.WithManifestFormat(format),
 		kev.WithManifestsAsSingleFile(singleFile),
+		kev.WithAdditionalManifests(additionalManifests),
 		kev.WithOutputDir(dir),
 		kev.WithEnvs(envs),
 		kev.WithLogVerbose(verbose),

go.mod

@@ -22,6 +22,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.19.7
 	k8s.io/apimachinery v0.19.7
+	k8s.io/client-go v0.19.7
 )
 
 require (
@@ -181,7 +182,6 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/client-go v0.19.7 // indirect
 	k8s.io/klog/v2 v2.5.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20210113233702-8566a335510f // indirect
 	k8s.io/utils v0.0.0-20200729134348-d5654de09c73 // indirect

pkg/kev/converter/converter.go

@@ -30,6 +30,7 @@ type Converter interface {
 		dir, workDir string,
 		projects map[string]*composego.Project,
 		files map[string][]string,
+		additionalManifests []string,
 		rendered map[string][]byte,
 		excluded map[string][]string) (map[string]string, error)
 }

pkg/kev/converter/dummy/converter.go

@@ -37,6 +37,7 @@ func (c *Dummy) Render(singleFile bool,
 	dir, workDir string,
 	projects map[string]*composego.Project,
 	files map[string][]string,
+	additionalManifests []string,
 	rendered map[string][]byte,
 	excluded map[string][]string) (map[string]string, error) {
 

pkg/kev/converter/kubernetes/converter.go

@@ -56,11 +56,13 @@ func (c *K8s) Render(singleFile bool,
 	dir, workDir string,
 	projects map[string]*composego.Project,
 	files map[string][]string,
+	additionalFiles []string,
 	rendered map[string][]byte,
 	excluded map[string][]string) (map[string]string, error) {
 
 	renderOutputPaths := map[string]string{}
 	envs := getSortedEnvs(projects)
+
 	for _, env := range envs {
 		project := projects[env]
 
@@ -120,7 +122,7 @@ func (c *K8s) Render(singleFile bool,
 		}
 
 		// @step Produce objects
-		err = PrintList(objects, convertOpts, rendered)
+		err = PrintList(objects, convertOpts, additionalFiles, rendered)
 		if err != nil {
 			return nil, errors.Wrapf(err, "Could not render %s manifests to disk, details:\n", Name)
 		}

pkg/kev/converter/kubernetes/utils.go

@@ -24,7 +24,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -45,6 +44,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes/scheme"
 )
 
 // Selector used as labels and selector
@@ -75,27 +75,41 @@ func (env EnvSort) Swap(i, j int) {
 
 // PrintList prints k8s objects
 // @orig: https://github.com/kubernetes/kompose/blob/master/pkg/transformer/kubernetes/k8sutils.go#L153
-func PrintList(objects []runtime.Object, opt ConvertOptions, rendered map[string][]byte) error {
+func PrintList(objects []runtime.Object, opt ConvertOptions, additionalManifests []string, rendered map[string][]byte) error {
 
 	var f *os.File
 	dirName := getDirName(opt)
 	log.Debugf("Target Dir: %s", dirName)
 
 	// Check if output file is a directory
 	isDirVal, err := isDir(opt.OutFile)
+
 	if err != nil {
 		log.Error("Directory check failed")
 		return err
 	}
+
 	if opt.CreateChart {
 		isDirVal = true
 	}
+
 	if !isDirVal {
+		// cleanup target directory before creating a new file
+		if err := os.RemoveAll(filepath.Dir(dirName)); err != nil {
+			return err
+		}
+
+		if err := os.MkdirAll(filepath.Dir(dirName), 0755); err != nil {
+			return err
+		}
+
+		// create a new output file
 		f, err = createOutFile(opt.OutFile)
 		if err != nil {
 			log.Error("Error creating output file")
 			return err
 		}
+
 		defer func(f *os.File) {
 			err := f.Close()
 			if err != nil {
@@ -104,18 +118,16 @@ func PrintList(objects []runtime.Object, opt ConvertOptions, rendered map[string
 		}(f)
 	}
 
-	var files []string
-	var indent int
-
+	indent := 2
 	if opt.YAMLIndent > 0 {
 		indent = opt.YAMLIndent
-	} else {
-		indent = 2
 	}
 
 	// @step print to stdout, or to a single file - it will return a list object
 	if opt.ToStdout || f != nil {
+
 		list := &v1.List{}
+
 		// convert objects to versioned and add them to list
 		for _, object := range objects {
 			versionedObject, err := convertToVersion(object, schema.GroupVersion{})
@@ -125,6 +137,20 @@ func PrintList(objects []runtime.Object, opt ConvertOptions, rendered map[string
 
 			list.Items = append(list.Items, runtime.RawExtension{Object: versionedObject})
 		}
+
+		// if additional manifests files are specified, add them to the generated objects list
+		if len(additionalManifests) > 0 {
+			for _, extraManifest := range additionalManifests {
+
+				ro, err := fileToRuntimeObject(extraManifest)
+				if err != nil {
+					return err
+				}
+
+				list.Items = append(list.Items, runtime.RawExtension{Object: ro})
+			}
+		}
+
 		// version list itself
 		listVersion := schema.GroupVersion{Group: "", Version: "v1"}
 		list.Kind = "List"
@@ -140,16 +166,17 @@ func PrintList(objects []runtime.Object, opt ConvertOptions, rendered map[string
 			return err
 		}
 
-		printVal, err := print("", dirName, "", data, opt.ToStdout, opt.GenerateJSON, f)
+		file, err := print(dirName, "", "", data, opt.ToStdout, opt.GenerateJSON, f)
 		if err != nil {
 			log.Error("Printing manifests failed")
 			return err
 		}
 
-		files = append(files, printVal)
-		rendered[printVal] = data
+		rendered[file] = data
+
 	} else {
 		// @step output directory specified - print all objects individually to that directory
+
 		finalDirName := dirName
 
 		// if that's a chart it'll spit things out to "templates" subdir
@@ -165,13 +192,27 @@ func PrintList(objects []runtime.Object, opt ConvertOptions, rendered map[string
 			return err
 		}
 
-		var file string
+		// if additional manifests files are specified, add them to the generated objects
+		if len(additionalManifests) > 0 {
+			for _, extraManifest := range additionalManifests {
+
+				ro, err := fileToRuntimeObject(extraManifest)
+				if err != nil {
+					return err
+				}
+
+				objects = append(objects, ro)
+			}
+		}
+
 		// create a separate file for each provider
 		for _, v := range objects {
+
 			versionedObject, err := convertToVersion(v, schema.GroupVersion{})
 			if err != nil {
 				return err
 			}
+
 			data, err := marshal(versionedObject, opt.GenerateJSON, indent)
 			if err != nil {
 				return err
@@ -197,60 +238,87 @@ func PrintList(objects []runtime.Object, opt ConvertOptions, rendered map[string
 				// Use reflect to access ObjectMeta struct inside runtime.Object.
 				// cast it to correct type - meta.ObjectMeta
 				objectMeta = val.FieldByName("ObjectMeta").Interface().(meta.ObjectMeta)
-
 			}
 
-			file, err = print(objectMeta.Name, finalDirName, strings.ToLower(typeMeta.Kind), data, opt.ToStdout, opt.GenerateJSON, f)
+			file, err := print(finalDirName, objectMeta.Name, strings.ToLower(typeMeta.Kind), data, opt.ToStdout, opt.GenerateJSON, f)
 			if err != nil {
 				log.Error("Printing manifests failed")
 				return err
 			}
 
-			files = append(files, file)
 			rendered[file] = data
 		}
 	}
 	// @step for helm output generate chart directory structure
 	if opt.CreateChart {
-		err = generateHelm(dirName)
-		if err != nil {
+		if err = generateHelm(dirName); err != nil {
 			log.Error("Couldn't generate HELM chart")
 			return err
 		}
 	}
+
 	return nil
 }
 
+// fileToRuntimeObject reads a file and converts its contents to a runtime.Object
+func fileToRuntimeObject(file string) (runtime.Object, error) {
+	// @step: create a new decoder
+	decoder := scheme.Codecs.UniversalDeserializer()
+
+	// @step: read the file
+	path, err := filepath.Abs(file)
+	if err != nil {
+		return nil, err
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// @step: decode the file
+	runtimeObject, _, err := decoder.Decode(data, nil, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("malformed manifest file %s", file))
+	}
+
+	return runtimeObject, nil
+}
+
 // print either renders to stdout or to file/s
 // @orig: https://github.com/kubernetes/kompose/blob/master/pkg/transformer/utils.go#L176
-func print(name, path string, trailing string, data []byte, toStdout, generateJSON bool, f *os.File) (string, error) {
+func print(path, name, kind string, data []byte, toStdout, generateJSON bool, f *os.File) (string, error) {
 	file := ""
+
 	if generateJSON {
-		file = fmt.Sprintf("%s-%s.json", name, trailing)
+		file = fmt.Sprintf("%s-%s.json", name, kind)
 	} else {
-		file = fmt.Sprintf("%s-%s.yaml", name, trailing)
+		file = fmt.Sprintf("%s-%s.yaml", name, kind)
 	}
+
 	if toStdout {
 		_, _ = fmt.Fprintf(os.Stdout, "%s\n", string(data))
 		return "", nil
+
 	} else if f != nil {
 		// Write all content to a single file f
 		if _, err := f.WriteString(fmt.Sprintf("%s\n", string(data))); err != nil {
 			log.Error("Couldn't write manifests content to a single file")
 			return "", err
 		}
 		_ = f.Sync()
+
 	} else {
 		// Write content separately to each file
 		file = filepath.Join(path, file)
-		if err := ioutil.WriteFile(file, []byte(data), 0644); err != nil {
+		if err := os.WriteFile(file, []byte(data), 0644); err != nil {
 			log.ErrorWithFields(log.Fields{
 				"file": file,
 			}, "Failed to write content to a file")
 			return "", err
 		}
 		log.Debugf("%s file %q created", Name, file)
 	}
+
 	return file, nil
 }
 
@@ -284,7 +352,7 @@ func generateHelm(dirName string) error {
 
 	// @step Create the readme file
 	readme := "This chart was created by Kompose\n"
-	err = ioutil.WriteFile(dirName+string(os.PathSeparator)+"README.md", []byte(readme), 0644)
+	err = os.WriteFile(dirName+string(os.PathSeparator)+"README.md", []byte(readme), 0644)
 	if err != nil {
 		return err
 	}
@@ -308,7 +376,7 @@ home:
 	var chartData bytes.Buffer
 	_ = t.Execute(&chartData, details)
 
-	err = ioutil.WriteFile(dirName+string(os.PathSeparator)+"Chart.yaml", chartData.Bytes(), 0644)
+	err = os.WriteFile(dirName+string(os.PathSeparator)+"Chart.yaml", chartData.Bytes(), 0644)
 	if err != nil {
 		return err
 	}
@@ -503,7 +571,7 @@ func durationStrToSecondsInt(s string) (*int32, error) {
 // getContentFromFile gets the content from the file..
 // @orig: https://github.com/kubernetes/kompose/blob/master/pkg/transformer/kubernetes/k8sutils.go#L775
 func getContentFromFile(file string) (string, error) {
-	fileBytes, err := ioutil.ReadFile(file)
+	fileBytes, err := os.ReadFile(file)
 	if err != nil {
 		log.ErrorWithFields(log.Fields{
 			"file": file,

pkg/kev/kev.go

@@ -64,7 +64,7 @@ func RenderProjectWithOptions(workingDir string, opts ...Options) error {
 		return err
 	}
 
-	return printRenderProjectWithOptionsSuccess(runner, results, envs, runner.config.ManifestFormat)
+	return printRenderProjectWithOptionsSuccess(runner, results, envs)
 }
 
 // DevWithOptions runs a continuous development cycle detecting project updates and