Include locked melange config in control section

This adds a .melange.json file to the control section of each built APK
which includes the locked (resolved and apko-solved) melange
configuration used to build the package.

Out of an abundance of caution, this also strips comments from the runs
of any pipelines to avoid embedding them in the APKs.

I've also hoisted the "If" field in pipeline to the top to match how
that's usually used (as the first field, if present).

Signed-off-by: Jon Johnson <jon.johnson@chainguard.dev>

examples/one-arch.yaml

@@ -30,10 +30,10 @@ test:
       packages:
         - busybox
   pipeline:
-    - if: ${{targets.architecture == "x86_64"}}
+    - if: ${{build.arch}} == "x86_64"
       runs: |
         echo hello test
-    - if: ${{targets.architecture == "arm64"}}
+    - if: ${{build.arch}} == "aarch64"
       runs: |
         echo "BAD ARCHITECTURE"
         exit 1

go.mod

@@ -48,6 +48,7 @@ require (
 	gopkg.in/ini.v1 v1.67.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f
+	mvdan.cc/sh/v3 v3.10.0
 	sigs.k8s.io/release-utils v0.8.5
 	sigs.k8s.io/yaml v1.4.0
 )
@@ -153,6 +154,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/term v0.5.0 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.15.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
@@ -223,5 +225,4 @@ require (
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	k8s.io/apimachinery v0.31.2 // indirect
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
-	mvdan.cc/sh/v3 v3.8.0 // indirect
 )

go.sum

@@ -117,8 +117,8 @@ github.com/containerd/stargz-snapshotter/estargz v0.16.1/go.mod h1:uyr4BfYfOj3G9
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
 github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
-github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0=
+github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f h1:eHnXnuK47UlSTOQexbzxAZfekVz6i+LKRdj1CU5DPaM=
 github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
 github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8=
@@ -163,8 +163,6 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
-github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -216,6 +214,8 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr
 github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
 github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
 github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
@@ -353,6 +353,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
+github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=
 github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -740,8 +742,8 @@ k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f h1:0LQagt0gDpKqvIkAMPaRGc
 k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f/go.mod h1:S9tOR0FxgyusSNR+MboCuiDpVWkAifZvaYI1Q2ubgro=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-mvdan.cc/sh/v3 v3.8.0 h1:ZxuJipLZwr/HLbASonmXtcvvC9HXY9d2lXZHnKGjFc8=
-mvdan.cc/sh/v3 v3.8.0/go.mod h1:w04623xkgBVo7/IUK89E0g8hBykgEpN0vgOj3RJr6MY=
+mvdan.cc/sh/v3 v3.10.0 h1:v9z7N1DLZ7owyLM/SXZQkBSXcwr2IGMm2LY2pmhVXj4=
+mvdan.cc/sh/v3 v3.10.0/go.mod h1:z/mSSVyLFGZzqb3ZIKojjyqIx/xbmz/UHdCSv9HmqXY=
 sigs.k8s.io/release-utils v0.8.5 h1:FUtFqEAN621gSXv0L7kHyWruBeS7TUU9aWf76olX7uQ=
 sigs.k8s.io/release-utils v0.8.5/go.mod h1:qsm5bdxdgoHkD8HsXpgme2/c3mdsNaiV53Sz2HmKeJA=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

pkg/build/build.go

@@ -291,6 +291,9 @@ func (b *Build) Close(ctx context.Context) error {
 
 // buildGuest invokes apko to build the guest environment, returning a reference to the image
 // loaded by the OCI Image loader.
+//
+// NB: This has side effects! This mutates Build by overwriting Configuration.Environment with
+// a locked version (packages resolved to versions) so we can record which packages were used.
 func (b *Build) buildGuest(ctx context.Context, imgConfig apko_types.ImageConfiguration, guestFS apkofs.FullFS) (string, error) {
 	log := clog.FromContext(ctx)
 	ctx, span := otel.Tracer("melange").Start(ctx, "buildGuest")
@@ -308,15 +311,34 @@ func (b *Build) buildGuest(ctx context.Context, imgConfig apko_types.ImageConfig
 		}...)
 	}
 
-	bc, err := apko_build.New(ctx, guestFS,
-		apko_build.WithImageConfiguration(imgConfig),
+	// Work around LockImageConfiguration assuming multi-arch.
+	imgConfig.Archs = []apko_types.Architecture{b.Arch}
+
+	opts := []apko_build.Option{apko_build.WithImageConfiguration(imgConfig),
 		apko_build.WithArch(b.Arch),
 		apko_build.WithExtraKeys(b.ExtraKeys),
 		apko_build.WithExtraBuildRepos(b.ExtraRepos),
 		apko_build.WithExtraPackages(b.ExtraPackages),
 		apko_build.WithCache(b.ApkCacheDir, false, apk.NewCache(true)),
 		apko_build.WithTempDir(tmp),
-		apko_build.WithIgnoreSignatures(b.IgnoreSignatures))
+		apko_build.WithIgnoreSignatures(b.IgnoreSignatures),
+	}
+
+	locked, warn, err := apko_build.LockImageConfiguration(ctx, imgConfig, opts...)
+	if err != nil {
+		return "", fmt.Errorf("unable to lock image configuration: %w", err)
+	}
+
+	for k, v := range warn {
+		log.Warnf("Unable to lock package %s: %s", k, v)
+	}
+
+	// Overwrite the environment with the locked one.
+	b.Configuration.Environment = *locked
+
+	opts = append(opts, apko_build.WithImageConfiguration(*locked))
+
+	bc, err := apko_build.New(ctx, guestFS, opts...)
 	if err != nil {
 		return "", fmt.Errorf("unable to create build context: %w", err)
 	}

pkg/build/compile.go

@@ -16,16 +16,19 @@ package build
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"maps"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"chainguard.dev/melange/pkg/cond"
 	"chainguard.dev/melange/pkg/config"
 	"chainguard.dev/melange/pkg/util"
 	"github.com/chainguard-dev/clog"
 	"gopkg.in/yaml.v3"
+	"mvdan.cc/sh/v3/syntax"
 )
 
 const unidentifiablePipeline = "???"
@@ -277,6 +280,12 @@ func (c *Compiled) compilePipeline(ctx context.Context, sm *SubstitutionMap, pip
 		return fmt.Errorf("mutating runs: %w", err)
 	}
 
+	// Drop any comments to avoid leaking things into .melange.json.
+	pipeline.Runs, err = stripComments(pipeline.Runs)
+	if err != nil {
+		return fmt.Errorf("stripping runs comments: %w", err)
+	}
+
 	if pipeline.If != "" {
 		pipeline.If, err = util.MutateAndQuoteStringFromMap(mutated, pipeline.If)
 		if err != nil {
@@ -360,3 +369,30 @@ func (c *Compiled) gatherDeps(ctx context.Context, pipeline *config.Pipeline) er
 
 	return nil
 }
+
+func stripComments(runs string) (string, error) {
+	parser := syntax.NewParser(syntax.KeepComments(false))
+	printer := syntax.NewPrinter()
+
+	builder := strings.Builder{}
+
+	// The KeepComments(false) option drops comments, including the shebang.
+	// We don't want to do that, so keep the first line if it starts with #!
+	if idx := strings.IndexRune(runs, '\n'); idx != -1 {
+		firstLine := runs[0 : idx+1]
+		if strings.HasPrefix(firstLine, "#!") {
+			builder.WriteString(firstLine)
+		}
+	}
+
+	var perr error
+	if err := parser.Stmts(strings.NewReader(runs), func(stmt *syntax.Stmt) bool {
+		perr = printer.Print(&builder, stmt)
+		builder.WriteRune('\n')
+		return perr == nil
+	}); err != nil || perr != nil {
+		return "", errors.Join(err, perr)
+	}
+
+	return builder.String(), nil
+}

pkg/build/compile_test.go

@@ -52,6 +52,7 @@ func TestInheritWorkdir(t *testing.T) {
 				WorkDir: "/work",
 				Pipeline: []config.Pipeline{{}, {
 					WorkDir: "/do-not-inherit",
+					Runs:    "#!/bin/bash\n# hunter2\necho $SECRET",
 				}},
 			}},
 		},
@@ -67,6 +68,9 @@ func TestInheritWorkdir(t *testing.T) {
 	if got, want := build.Configuration.Pipeline[0].Pipeline[1].WorkDir, "/do-not-inherit"; want != got {
 		t.Fatalf("workdir[1]: want %q, got %q", want, got)
 	}
+	if got, want := build.Configuration.Pipeline[0].Pipeline[1].Runs, "#!/bin/bash\necho $SECRET\n"; want != got {
+		t.Fatalf("runs[1]: should strip comments, want %q, got %q", want, got)
+	}
 }
 
 func TestCompileTest(t *testing.T) {

pkg/build/package.go

@@ -42,6 +42,7 @@ import (
 	"github.com/chainguard-dev/clog"
 	"github.com/psanford/memfs"
 	"go.opentelemetry.io/otel"
+	"gopkg.in/yaml.v3"
 )
 
 // pgzip's default is GOMAXPROCS(0)
@@ -210,6 +211,17 @@ func (pc *PackageBuild) generateControlSection(ctx context.Context) ([]byte, err
 		return nil, fmt.Errorf("unable to build control FS: %w", err)
 	}
 
+	var melangeBuf bytes.Buffer
+	enc := yaml.NewEncoder(&melangeBuf)
+	enc.SetIndent(2) // To align with `yam` a little better.
+
+	if err := enc.Encode(pc.Build.Configuration); err != nil {
+		return nil, fmt.Errorf("marshalling config: %w", err)
+	}
+	if err := fsys.WriteFile(".melange.yaml", melangeBuf.Bytes(), 0644); err != nil {
+		return nil, fmt.Errorf("writing .melange.yaml: %w", err)
+	}
+
 	if scriptlets := pc.Scriptlets; scriptlets != nil {
 		if scriptlets.Trigger.Script != "" {
 			// #nosec G306 -- scriptlets must be executable

pkg/config/config.go

@@ -361,6 +361,8 @@ type PipelineAssertions struct {
 }
 
 type Pipeline struct {
+	// Optional: A condition to evaluate before running the pipeline
+	If string `json:"if,omitempty" yaml:"if,omitempty"`
 	// Optional: A user defined name for the pipeline
 	Name string `json:"name,omitempty" yaml:"name,omitempty"`
 	// Optional: A named reusable pipeline to run
@@ -386,8 +388,6 @@ type Pipeline struct {
 	Needs *Needs `json:"needs,omitempty" yaml:"needs,omitempty"`
 	// Optional: Labels to apply to the pipeline
 	Label string `json:"label,omitempty" yaml:"label,omitempty"`
-	// Optional: A condition to evaluate before running the pipeline
-	If string `json:"if,omitempty" yaml:"if,omitempty"`
 	// Optional: Assertions to evaluate whether the pipeline was successful
 	Assertions *PipelineAssertions `json:"assertions,omitempty" yaml:"assertions,omitempty"`
 	// Optional: The working directory of the pipeline