From 533589a561dfdbc829e731a780410215673fe53c Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Tue, 5 Nov 2024 15:47:12 -0800 Subject: [PATCH] Include locked melange config in control section This adds a .melange.json file to the control section of each built APK which includes the locked (resolved and apko-solved) melange configuration used to build the package. Out of an abundance of caution, this also strips comments from the runs of any pipelines to avoid embedding them in the APKs. Signed-off-by: Jon Johnson --- examples/one-arch.yaml | 4 ++-- go.mod | 3 ++- go.sum | 18 ++++++++++-------- pkg/build/build.go | 28 +++++++++++++++++++++++++--- pkg/build/compile.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/build/compile_test.go | 4 ++++ pkg/build/package.go | 12 ++++++++++++ 7 files changed, 91 insertions(+), 14 deletions(-) diff --git a/examples/one-arch.yaml b/examples/one-arch.yaml index 653de707a..5de833f76 100644 --- a/examples/one-arch.yaml +++ b/examples/one-arch.yaml @@ -30,10 +30,10 @@ test: packages: - busybox pipeline: - - if: ${{targets.architecture == "x86_64"}} + - if: ${{build.arch}} == "x86_64" runs: | echo hello test - - if: ${{targets.architecture == "arm64"}} + - if: ${{build.arch}} == "aarch64" runs: | echo "BAD ARCHITECTURE" exit 1 diff --git a/go.mod b/go.mod index 8f30bfe44..2f069dd7b 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f + mvdan.cc/sh/v3 v3.10.0 sigs.k8s.io/release-utils v0.8.5 sigs.k8s.io/yaml v1.4.0 ) @@ -153,6 +154,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.15.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect @@ -223,5 +225,4 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect k8s.io/apimachinery v0.31.2 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - mvdan.cc/sh/v3 v3.8.0 // indirect ) diff --git a/go.sum b/go.sum index 4d29ad50c..47a536c6f 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9N github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= -github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0= +github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f h1:eHnXnuK47UlSTOQexbzxAZfekVz6i+LKRdj1CU5DPaM= github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8= @@ -165,8 +165,6 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -218,6 +216,8 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= @@ -355,6 +355,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -415,8 +417,8 @@ github.com/psanford/memfs v0.0.0-20241019191636-4ef911798f9b/go.mod h1:tcaRap0jS github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= @@ -749,8 +751,8 @@ k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f h1:0LQagt0gDpKqvIkAMPaRGc k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f/go.mod h1:S9tOR0FxgyusSNR+MboCuiDpVWkAifZvaYI1Q2ubgro= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -mvdan.cc/sh/v3 v3.8.0 h1:ZxuJipLZwr/HLbASonmXtcvvC9HXY9d2lXZHnKGjFc8= -mvdan.cc/sh/v3 v3.8.0/go.mod h1:w04623xkgBVo7/IUK89E0g8hBykgEpN0vgOj3RJr6MY= +mvdan.cc/sh/v3 v3.10.0 h1:v9z7N1DLZ7owyLM/SXZQkBSXcwr2IGMm2LY2pmhVXj4= +mvdan.cc/sh/v3 v3.10.0/go.mod h1:z/mSSVyLFGZzqb3ZIKojjyqIx/xbmz/UHdCSv9HmqXY= sigs.k8s.io/release-utils v0.8.5 h1:FUtFqEAN621gSXv0L7kHyWruBeS7TUU9aWf76olX7uQ= sigs.k8s.io/release-utils v0.8.5/go.mod h1:qsm5bdxdgoHkD8HsXpgme2/c3mdsNaiV53Sz2HmKeJA= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/pkg/build/build.go b/pkg/build/build.go index aefce5641..def828327 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -274,6 +274,9 @@ func (b *Build) Close(ctx context.Context) error { // buildGuest invokes apko to build the guest environment, returning a reference to the image // loaded by the OCI Image loader. +// +// NB: This has side effects! This mutates Build by overwriting Configuration.Environment with +// a locked version (packages resolved to versions) so we can record which packages were used. func (b *Build) buildGuest(ctx context.Context, imgConfig apko_types.ImageConfiguration, guestFS apkofs.FullFS) (string, error) { log := clog.FromContext(ctx) ctx, span := otel.Tracer("melange").Start(ctx, "buildGuest") @@ -291,15 +294,34 @@ func (b *Build) buildGuest(ctx context.Context, imgConfig apko_types.ImageConfig }...) } - bc, err := apko_build.New(ctx, guestFS, - apko_build.WithImageConfiguration(imgConfig), + // Work around LockImageConfiguration assuming multi-arch. + imgConfig.Archs = []apko_types.Architecture{b.Arch} + + opts := []apko_build.Option{apko_build.WithImageConfiguration(imgConfig), apko_build.WithArch(b.Arch), apko_build.WithExtraKeys(b.ExtraKeys), apko_build.WithExtraBuildRepos(b.ExtraRepos), apko_build.WithExtraPackages(b.ExtraPackages), apko_build.WithCache(b.ApkCacheDir, false, apk.NewCache(true)), apko_build.WithTempDir(tmp), - apko_build.WithIgnoreSignatures(b.IgnoreSignatures)) + apko_build.WithIgnoreSignatures(b.IgnoreSignatures), + } + + locked, warn, err := apko_build.LockImageConfiguration(ctx, imgConfig, opts...) + if err != nil { + return "", fmt.Errorf("unable to lock image configuration: %w", err) + } + + for k, v := range warn { + log.Warnf("Unable to lock package %s: %s", k, v) + } + + // Overwrite the environment with the locked one. + b.Configuration.Environment = *locked + + opts = append(opts, apko_build.WithImageConfiguration(*locked)) + + bc, err := apko_build.New(ctx, guestFS, opts...) if err != nil { return "", fmt.Errorf("unable to create build context: %w", err) } diff --git a/pkg/build/compile.go b/pkg/build/compile.go index 351b914df..d79ad6216 100644 --- a/pkg/build/compile.go +++ b/pkg/build/compile.go @@ -16,16 +16,19 @@ package build import ( "context" + "errors" "fmt" "maps" "os" "path/filepath" + "strings" "chainguard.dev/melange/pkg/cond" "chainguard.dev/melange/pkg/config" "chainguard.dev/melange/pkg/util" "github.com/chainguard-dev/clog" "gopkg.in/yaml.v3" + "mvdan.cc/sh/v3/syntax" ) const unidentifiablePipeline = "???" @@ -277,6 +280,12 @@ func (c *Compiled) compilePipeline(ctx context.Context, sm *SubstitutionMap, pip return fmt.Errorf("mutating runs: %w", err) } + // Drop any comments to avoid leaking things into .melange.json. + pipeline.Runs, err = stripComments(pipeline.Runs) + if err != nil { + return fmt.Errorf("stripping runs comments: %w", err) + } + if pipeline.If != "" { pipeline.If, err = util.MutateAndQuoteStringFromMap(mutated, pipeline.If) if err != nil { @@ -360,3 +369,30 @@ func (c *Compiled) gatherDeps(ctx context.Context, pipeline *config.Pipeline) er return nil } + +func stripComments(runs string) (string, error) { + parser := syntax.NewParser(syntax.KeepComments(false)) + printer := syntax.NewPrinter() + + builder := strings.Builder{} + + // The KeepComments(false) option drops comments, including the shebang. + // We don't want to do that, so keep the first line if it starts with #! + if idx := strings.IndexRune(runs, '\n'); idx != -1 { + firstLine := runs[0 : idx+1] + if strings.HasPrefix(firstLine, "#!") { + builder.WriteString(firstLine) + } + } + + var perr error + if err := parser.Stmts(strings.NewReader(runs), func(stmt *syntax.Stmt) bool { + perr = printer.Print(&builder, stmt) + builder.WriteRune('\n') + return perr == nil + }); err != nil || perr != nil { + return "", errors.Join(err, perr) + } + + return builder.String(), nil +} diff --git a/pkg/build/compile_test.go b/pkg/build/compile_test.go index 336b1075a..9c7c05972 100644 --- a/pkg/build/compile_test.go +++ b/pkg/build/compile_test.go @@ -52,6 +52,7 @@ func TestInheritWorkdir(t *testing.T) { WorkDir: "/work", Pipeline: []config.Pipeline{{}, { WorkDir: "/do-not-inherit", + Runs: "#!/bin/bash\n# hunter2\necho $SECRET", }}, }}, }, @@ -67,6 +68,9 @@ func TestInheritWorkdir(t *testing.T) { if got, want := build.Configuration.Pipeline[0].Pipeline[1].WorkDir, "/do-not-inherit"; want != got { t.Fatalf("workdir[1]: want %q, got %q", want, got) } + if got, want := build.Configuration.Pipeline[0].Pipeline[1].Runs, "#!/bin/bash\necho $SECRET\n"; want != got { + t.Fatalf("runs[1]: should strip comments, want %q, got %q", want, got) + } } func TestCompileTest(t *testing.T) { diff --git a/pkg/build/package.go b/pkg/build/package.go index 2bb45ee61..45a164002 100644 --- a/pkg/build/package.go +++ b/pkg/build/package.go @@ -42,6 +42,7 @@ import ( "github.com/chainguard-dev/clog" "github.com/psanford/memfs" "go.opentelemetry.io/otel" + "gopkg.in/yaml.v3" ) // pgzip's default is GOMAXPROCS(0) @@ -210,6 +211,17 @@ func (pc *PackageBuild) generateControlSection(ctx context.Context) ([]byte, err return nil, fmt.Errorf("unable to build control FS: %w", err) } + var melangeBuf bytes.Buffer + enc := yaml.NewEncoder(&melangeBuf) + enc.SetIndent(2) // To align with `yam` a little better. + + if err := enc.Encode(pc.Build.Configuration); err != nil { + return nil, fmt.Errorf("marshalling config: %w", err) + } + if err := fsys.WriteFile(".melange.yaml", melangeBuf.Bytes(), 0644); err != nil { + return nil, fmt.Errorf("writing .melange.yaml: %w", err) + } + if scriptlets := pc.Scriptlets; scriptlets != nil { if scriptlets.Trigger.Script != "" { // #nosec G306 -- scriptlets must be executable