From 0f1876dba572d3efab8486ac0ca426a49d4d044c Mon Sep 17 00:00:00 2001 From: nikhilsbhat Date: Wed, 31 Jul 2024 08:46:05 +0530 Subject: [PATCH] Add a "build" command that only constructs YAML files by substituting imports, rather than merging them like the "import" command does --- .golangci.yml | 3 ++ Dockerfile | 2 +- cmd/commands.go | 80 +++++++++++++++++++++++++++++++----- cmd/register.go | 1 + docs/doc/yamll.md | 5 ++- docs/doc/yamll_build.md | 35 ++++++++++++++++ docs/doc/yamll_import.md | 2 +- docs/doc/yamll_tree.md | 4 +- docs/doc/yamll_version.md | 2 +- pkg/yamll/build.go | 58 ++++++++++++++++++++++++++ pkg/yamll/dependency_test.go | 8 ++++ pkg/yamll/git.go | 3 ++ pkg/yamll/yamll.go | 10 +++++ 13 files changed, 195 insertions(+), 18 deletions(-) create mode 100644 docs/doc/yamll_build.md create mode 100644 pkg/yamll/build.go diff --git a/.golangci.yml b/.golangci.yml index a8bce51..93808de 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -18,6 +18,9 @@ linters: - dupword - tagalign - testpackage + - perfsprint + - testifylint + - mnd issues: exclude-rules: diff --git a/Dockerfile b/Dockerfile index fc9fb9e..854c6d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ### Description: Dockerfile for yamll -FROM alpine:3.16 +FROM alpine:3.20.2 COPY yamll / diff --git a/cmd/commands.go b/cmd/commands.go index 8cbf7c3..a597b9e 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -21,7 +21,7 @@ func getRootCommand() *cobra.Command { Short: "A utility to facilitate the inclusion of sub-YAML files as libraries.", Long: `It identifies imports declared in YAML files and merges them to generate a single final YAML file, similar to importing libraries in programming.`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Usage() }, } @@ -48,7 +48,7 @@ func getImportCommand() *cobra.Command { yamll import --file path/to/file.yaml --no-validation yamll import --file path/to/file.yaml --effective`, PreRunE: setCLIClient, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { cfg := yamll.New(yamllCfg.Merge, yamllCfg.LogLevel, yamllCfg.Limiter, cliCfg.Files...) cfg.SetLogger() logger = cfg.GetLogger() @@ -105,14 +105,72 @@ yamll import --file path/to/file.yaml --effective`, return importCommand } +func getBuildCommand() *cobra.Command { + buildCommand := &cobra.Command{ + Use: "build [flags]", + Short: "Builds YAML files substituting imports", + Long: "Builds YAML by substituting all anchors and aliases defined in sub-YAML files defined as libraries", + Example: `yamll build --file path/to/file.yaml`, + PreRunE: setCLIClient, + RunE: func(_ *cobra.Command, _ []string) error { + cfg := yamll.New(yamllCfg.Merge, yamllCfg.LogLevel, yamllCfg.Limiter, cliCfg.Files...) + cfg.SetLogger() + logger = cfg.GetLogger() + + out, err := cfg.YamlBuild() + if err != nil { + logger.Error("errored generating final yaml", slog.Any("err", err)) + } + + if !cliCfg.NoValidation { + logger.Debug("validating final yaml for syntax") + var data interface{} + err = yaml.Unmarshal([]byte(out), &data) + if err != nil { + logger.Error("the final rendered YAML file is not a valid yaml", slog.Any("error", err)) + logger.Error("rendering the final YAML encountered an error. skip validation to view the broken file.") + + os.Exit(1) + } + } + + if !cliCfg.NoColor { + render := renderer.GetRenderer(nil, nil, false, true, false, false, false) + coloredFinalData, err := render.Color(renderer.TypeYAML, string(out)) + if err != nil { + logger.Error("color coding yaml errored", slog.Any("error", err)) + } else { + out = yamll.Yaml(coloredFinalData) + } + } + + if _, err = writer.Write([]byte(out)); err != nil { + return err + } + + return nil + }, + } + + buildCommand.SilenceErrors = true + registerCommonFlags(buildCommand) + + buildCommand.PersistentFlags().StringVarP(&cliCfg.ToFile, "to-file", "", "", + "name of the file to which the final imported yaml should be written to") + buildCommand.PersistentFlags().BoolVarP(&cliCfg.NoValidation, "no-validation", "", false, + "when enabled it skips validating the final generated YAML file") + + return buildCommand +} + func getTreeCommand() *cobra.Command { - importCommand := &cobra.Command{ + treeCommand := &cobra.Command{ Use: "tree [flags]", - Short: "builds dependency trees from sub-YAML files defined as libraries", + Short: "Builds dependency trees from sub-YAML files defined as libraries", Long: "Identifies dependencies and builds the dependency tree for the base yaml", Example: `yamll tree --file path/to/file.yaml`, PreRunE: setCLIClient, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { cfg := yamll.New(yamllCfg.Merge, yamllCfg.LogLevel, yamllCfg.Limiter, cliCfg.Files...) cfg.SetLogger() logger = cfg.GetLogger() @@ -125,10 +183,10 @@ func getTreeCommand() *cobra.Command { }, } - importCommand.SilenceErrors = true - registerCommonFlags(importCommand) + treeCommand.SilenceErrors = true + registerCommonFlags(treeCommand) - return importCommand + return treeCommand } func versionConfig(_ *cobra.Command, _ []string) error { @@ -138,10 +196,10 @@ func versionConfig(_ *cobra.Command, _ []string) error { os.Exit(1) } - writer := bufio.NewWriter(os.Stdout) + versionWriter := bufio.NewWriter(os.Stdout) versionInfo := fmt.Sprintf("%s \n", strings.Join([]string{"yamll version", string(buildInfo)}, ": ")) - if _, err = writer.WriteString(versionInfo); err != nil { + if _, err = versionWriter.WriteString(versionInfo); err != nil { logger.Error(err.Error()) os.Exit(1) } @@ -152,7 +210,7 @@ func versionConfig(_ *cobra.Command, _ []string) error { logger.Error(err.Error()) os.Exit(1) } - }(writer) + }(versionWriter) return nil } diff --git a/cmd/register.go b/cmd/register.go index cc6b336..0bc004d 100644 --- a/cmd/register.go +++ b/cmd/register.go @@ -29,6 +29,7 @@ func getYamllCommands() *cobra.Command { command := new(yamllCommands) command.commands = append(command.commands, getImportCommand()) command.commands = append(command.commands, getTreeCommand()) + command.commands = append(command.commands, getBuildCommand()) command.commands = append(command.commands, getVersionCommand()) return command.prepareCommands() diff --git a/docs/doc/yamll.md b/docs/doc/yamll.md index 3480281..77d74d9 100644 --- a/docs/doc/yamll.md +++ b/docs/doc/yamll.md @@ -22,8 +22,9 @@ yamll [command] [flags] ### SEE ALSO +* [yamll build](yamll_build.md) - Builds YAML files substituting imports * [yamll import](yamll_import.md) - Imports defined sub-YAML files as libraries -* [yamll tree](yamll_tree.md) - builds dependency trees from sub-YAML files defined as libraries +* [yamll tree](yamll_tree.md) - Builds dependency trees from sub-YAML files defined as libraries * [yamll version](yamll_version.md) - Command to fetch the version of YAMLL installed -###### Auto generated by spf13/cobra on 6-Jul-2024 +###### Auto generated by spf13/cobra on 27-Jul-2024 diff --git a/docs/doc/yamll_build.md b/docs/doc/yamll_build.md new file mode 100644 index 0000000..59434b3 --- /dev/null +++ b/docs/doc/yamll_build.md @@ -0,0 +1,35 @@ +## yamll build + +Builds YAML files substituting imports + +### Synopsis + +Builds YAML by substituting all anchors and aliases defined in sub-YAML files defined as libraries + +``` +yamll build [flags] +``` + +### Examples + +``` +yamll build --file path/to/file.yaml +``` + +### Options + +``` + -f, --file stringArray root yaml files to be used for importing + -h, --help help for build + --limiter string limiters to separate the yaml files post merging (default "---") + --log-level string log level for the yamll (default "INFO") + --no-color when enabled the output would not be color encoded + --no-validation when enabled it skips validating the final generated YAML file + --to-file string name of the file to which the final imported yaml should be written to +``` + +### SEE ALSO + +* [yamll](yamll.md) - A utility to facilitate the inclusion of sub-YAML files as libraries. + +###### Auto generated by spf13/cobra on 27-Jul-2024 diff --git a/docs/doc/yamll_import.md b/docs/doc/yamll_import.md index 52abb04..71553b7 100644 --- a/docs/doc/yamll_import.md +++ b/docs/doc/yamll_import.md @@ -36,4 +36,4 @@ yamll import --file path/to/file.yaml --effective * [yamll](yamll.md) - A utility to facilitate the inclusion of sub-YAML files as libraries. -###### Auto generated by spf13/cobra on 6-Jul-2024 +###### Auto generated by spf13/cobra on 27-Jul-2024 diff --git a/docs/doc/yamll_tree.md b/docs/doc/yamll_tree.md index 032a7da..c3075e1 100644 --- a/docs/doc/yamll_tree.md +++ b/docs/doc/yamll_tree.md @@ -1,6 +1,6 @@ ## yamll tree -builds dependency trees from sub-YAML files defined as libraries +Builds dependency trees from sub-YAML files defined as libraries ### Synopsis @@ -30,4 +30,4 @@ yamll tree --file path/to/file.yaml * [yamll](yamll.md) - A utility to facilitate the inclusion of sub-YAML files as libraries. -###### Auto generated by spf13/cobra on 6-Jul-2024 +###### Auto generated by spf13/cobra on 27-Jul-2024 diff --git a/docs/doc/yamll_version.md b/docs/doc/yamll_version.md index 2b5caf1..d2fc408 100644 --- a/docs/doc/yamll_version.md +++ b/docs/doc/yamll_version.md @@ -29,4 +29,4 @@ yamll version [flags] * [yamll](yamll.md) - A utility to facilitate the inclusion of sub-YAML files as libraries. -###### Auto generated by spf13/cobra on 6-Jul-2024 +###### Auto generated by spf13/cobra on 27-Jul-2024 diff --git a/pkg/yamll/build.go b/pkg/yamll/build.go new file mode 100644 index 0000000..2c82a5e --- /dev/null +++ b/pkg/yamll/build.go @@ -0,0 +1,58 @@ +package yamll + +import ( + "fmt" + "strings" + + "github.com/goccy/go-yaml" + "github.com/nikhilsbhat/yamll/pkg/errors" +) + +func (yamlRoutes YamlRoutes) Build() (Yaml, error) { + anchorRefs := strings.NewReader(yamlRoutes.getRawData()) + + var output []byte + + for _, dependencyRoute := range yamlRoutes { + if !dependencyRoute.Root { + continue + } + + yamlMap := make(Data) + + decodeOpts := []yaml.DecodeOption{ + yaml.UseOrderedMap(), + yaml.Strict(), + yaml.ReferenceReaders(anchorRefs), + } + + encodeOpts := []yaml.EncodeOption{ + yaml.Indent(yamlIndent), + yaml.IndentSequence(true), + yaml.UseLiteralStyleIfMultiline(true), + } + + if err := yaml.UnmarshalWithOptions([]byte(dependencyRoute.DataRaw), &yamlMap, decodeOpts...); err != nil { + return "", &errors.YamllError{Message: fmt.Sprintf("error deserialising YAML file: %v", err)} + } + + yamlOut, err := yaml.MarshalWithOptions(yamlMap, encodeOpts...) + if err != nil { + return "", &errors.YamllError{Message: fmt.Sprintf("serialising YAML file %s errored : %v", dependencyRoute.File, err)} + } + + output = yamlOut + } + + return Yaml(output), nil +} + +func (yamlRoutes YamlRoutes) getRawData() string { + var rawData string + + for _, dependencyRoute := range yamlRoutes { + rawData += fmt.Sprintf("---\n%s\n", dependencyRoute.DataRaw) + } + + return rawData +} diff --git a/pkg/yamll/dependency_test.go b/pkg/yamll/dependency_test.go index f75b3fd..fd044f3 100644 --- a/pkg/yamll/dependency_test.go +++ b/pkg/yamll/dependency_test.go @@ -20,20 +20,25 @@ func Test_getDependencyData2(t *testing.T) { ##++https://run.mocky.io/v3/92e08b25-dd1f-4dd0-bc55-9649b5b896c9` stringReader := strings.NewReader(absYamlFilePath) + scanner := bufio.NewScanner(stringReader) + t.Setenv("USERNAME", "nikhil") t.Setenv("PASSWORD", "super-secret-password") cfg := New(false, "DEBUG", "") + cfg.SetLogger() dependencies := make([]*Dependency, 0) + for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "##++") { dependency, err := cfg.getDependencyData(line) assert.NoError(t, err) + dependencies = append(dependencies, dependency) } } @@ -52,6 +57,7 @@ func TestDependency_ReadData(t *testing.T) { } cfg := New(false, "DEBUG", "") + cfg.SetLogger() out, err := dependency.readData(false, cfg.GetLogger()) @@ -82,9 +88,11 @@ func TestConfig_ResolveDependencies2(t *testing.T) { func TestDependency_Git(t *testing.T) { t.Run("", func(t *testing.T) { cfg := New(false, "DEBUG", "") + cfg.SetLogger() t.Setenv("GITHUB_TOKEN", "testkey") + dependency := Dependency{ // Path: "git+https://github.com/nikhilsbhat/yamll@main?path=internal/fixtures/base.yaml", Path: "git+ssh://git@github.com:nikhilsbhat/yamll@main?path=internal/fixtures/base.yaml", diff --git a/pkg/yamll/git.go b/pkg/yamll/git.go index 5044a81..800175c 100644 --- a/pkg/yamll/git.go +++ b/pkg/yamll/git.go @@ -115,13 +115,16 @@ func (dependency *Dependency) getGitMetaData() (*gitMeta, error) { } gitBaseURL = fmt.Sprintf("git@%v", gitParsedURL[1]) + remainingPath = fmt.Sprintf("https://%v@%v", gitParsedURL[1], gitParsedURL[2]) } else { gitParsedURL := strings.SplitN(dependency.Path, "@", 2) if len(gitParsedURL) != 2 { return nil, &errors.YamllError{Message: fmt.Sprintf("unable to parse git url '%s'", dependency.Path)} } + gitBaseURL = gitParsedURL[0] + remainingPath = dependency.Path } diff --git a/pkg/yamll/yamll.go b/pkg/yamll/yamll.go index a10bb89..d081728 100644 --- a/pkg/yamll/yamll.go +++ b/pkg/yamll/yamll.go @@ -99,6 +99,16 @@ func (cfg *Config) YamlTree(color bool) error { return err } +// YamlBuild builds YAML by substituting all anchors and aliases defined in sub-YAML files defined as libraries. +func (cfg *Config) YamlBuild() (Yaml, error) { + dependencyRoutes, err := cfg.ResolveDependencies(make(map[string]*YamlData), cfg.Files...) + if err != nil { + return "", &errors.YamllError{Message: fmt.Sprintf("fetching dependency tree errored with: '%v'", err)} + } + + return YamlRoutes(dependencyRoutes).Build() +} + // New returns new instance of Config with passed parameters. func New(effective bool, logLevel, limiter string, paths ...string) *Config { dependencies := make([]*Dependency, 0)