From ac9600ce62ca1f45f91daaacdee9c9e6113953ea Mon Sep 17 00:00:00 2001 From: Christopher Hunter Date: Wed, 5 Jul 2023 16:33:08 -0700 Subject: [PATCH] refactor: use BOSH release manifest reader - refator: add test coverage for TestReadBOSHReleaseManifestsFromTarballs - feat: add Stemcell method on BOSHReleaseManifest - fix linter errors - refactor: use new stemcell utility --- internal/builder/release_manifest_reader.go | 73 +++---------------- .../builder/release_manifest_reader_test.go | 6 +- pkg/tile/bosh_release.go | 8 ++ pkg/tile/bosh_release_test.go | 11 +++ 4 files changed, 31 insertions(+), 67 deletions(-) diff --git a/internal/builder/release_manifest_reader.go b/internal/builder/release_manifest_reader.go index bb62e5b80..de6a4111f 100644 --- a/internal/builder/release_manifest_reader.go +++ b/internal/builder/release_manifest_reader.go @@ -1,18 +1,14 @@ package builder import ( - "archive/tar" - "compress/gzip" "crypto/sha1" + "encoding/hex" "fmt" - "io" - "path/filepath" - "strings" - "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" - - "gopkg.in/yaml.v2" + "github.com/pivotal-cf/kiln/pkg/tile" + "io" + "path/filepath" ) type ReleaseManifest struct { @@ -24,17 +20,6 @@ type ReleaseManifest struct { StemcellVersion string `yaml:"-"` } -// inputReleaseManifest is a subset of release.MF -type inputReleaseManifest struct { - Name string `yaml:"name"` - Version string `yaml:"version"` - CompiledPackages []compiledPackage `yaml:"compiled_packages"` -} - -type compiledPackage struct { - Stemcell string `yaml:"stemcell"` -} - type ReleaseManifestReader struct { fs billy.Filesystem } @@ -54,54 +39,14 @@ func (r ReleaseManifestReader) Read(releaseTarball string) (Part, error) { } defer closeAndIgnoreError(file) - // TODO: use component.ReadReleaseManifest - // we could not do it yet due to a circular package reference where we import builder in the local release source - - gr, err := gzip.NewReader(file) - if err != nil { - return Part{}, err - } - defer closeAndIgnoreError(gr) - - tr := tar.NewReader(gr) - - var header *tar.Header - for { - header, err = tr.Next() - if err != nil { - if err == io.EOF { - return Part{}, fmt.Errorf("could not find release.MF in %q", releaseTarball) - } - - return Part{}, fmt.Errorf("error while reading %q: %s", releaseTarball, err) - } - - if filepath.Base(header.Name) == "release.MF" { - break - } - } - - var inputReleaseManifest inputReleaseManifest - inputReleaseManifestContents, err := io.ReadAll(tr) - if err != nil { - return Part{}, err // NOTE: cannot replicate this error scenario in a test - } - - err = yaml.Unmarshal(inputReleaseManifestContents, &inputReleaseManifest) + inputReleaseManifest, err := tile.ReadProductTemplatePartFromBOSHReleaseTarball(file) if err != nil { return Part{}, err } - var stemcellOS, stemcellVersion string - compiledPackages := inputReleaseManifest.CompiledPackages - if len(compiledPackages) > 0 { - inputStemcell := inputReleaseManifest.CompiledPackages[0].Stemcell - stemcellParts := strings.Split(inputStemcell, "/") - if len(stemcellParts) != 2 { - return Part{}, fmt.Errorf("Invalid format for compiled package stemcell inside release.MF (expected 'os/version'): %s", inputStemcell) - } - stemcellOS = stemcellParts[0] - stemcellVersion = stemcellParts[1] + stemcellOS, stemcellVersion, stemcellOK := inputReleaseManifest.Stemcell() + if !stemcellOK && len(inputReleaseManifest.CompiledPackages) > 0 { + return Part{}, fmt.Errorf("%s/%s has invalid stemcell: %q", inputReleaseManifest.Name, inputReleaseManifest.Version, inputReleaseManifest.CompiledPackages[0].Stemcell) } outputReleaseManifest := ReleaseManifest{ @@ -123,7 +68,7 @@ func (r ReleaseManifestReader) Read(releaseTarball string) (Part, error) { return Part{}, err // NOTE: cannot replicate this error scenario in a test } - outputReleaseManifest.SHA1 = fmt.Sprintf("%x", hash.Sum(nil)) + outputReleaseManifest.SHA1 = hex.EncodeToString(hash.Sum(nil)) return Part{ File: releaseTarball, diff --git a/internal/builder/release_manifest_reader_test.go b/internal/builder/release_manifest_reader_test.go index e6fd9b5fb..5edf97793 100644 --- a/internal/builder/release_manifest_reader_test.go +++ b/internal/builder/release_manifest_reader_test.go @@ -183,7 +183,7 @@ version: 1.2.3 Expect(gw.Close()).NotTo(HaveOccurred()) _, err = reader.Read(tarball.Name()) - Expect(err).To(MatchError(fmt.Sprintf("could not find release.MF in %q", tarball.Name()))) + Expect(err).To(MatchError("failed to find release.MF in tarball")) }) }) @@ -220,7 +220,7 @@ version: 1.2.3 Expect(err).NotTo(HaveOccurred()) _, err = reader.Read(tarball.Name()) - Expect(err).To(MatchError(fmt.Sprintf("could not find release.MF in %q", tarball.Name()))) + Expect(err).To(MatchError("failed to find release.MF in tarball")) }) }) @@ -242,7 +242,7 @@ version: 1.2.3 Expect(err).NotTo(HaveOccurred()) _, err = reader.Read(tarball.Name()) - Expect(err).To(MatchError(fmt.Sprintf("error while reading %q: unexpected EOF", tarball.Name()))) + Expect(err).To(MatchError("unexpected EOF")) }) }) diff --git a/pkg/tile/bosh_release.go b/pkg/tile/bosh_release.go index 9002f984b..67dafd0c2 100644 --- a/pkg/tile/bosh_release.go +++ b/pkg/tile/bosh_release.go @@ -12,6 +12,7 @@ import ( "io/fs" "os" "path" + "strings" "golang.org/x/exp/slices" "gopkg.in/yaml.v2" @@ -104,6 +105,13 @@ type BOSHReleaseManifest struct { Packages []BOSHReleasePackage `yaml:"packages"` } +func (mf BOSHReleaseManifest) Stemcell() (string, string, bool) { + if len(mf.CompiledPackages) == 0 { + return "", "", false + } + return strings.Cut(mf.CompiledPackages[0].Stemcell, "/") +} + type BOSHReleaseTarball struct { Manifest BOSHReleaseManifest diff --git a/pkg/tile/bosh_release_test.go b/pkg/tile/bosh_release_test.go index c82c0d692..6f779399c 100644 --- a/pkg/tile/bosh_release_test.go +++ b/pkg/tile/bosh_release_test.go @@ -2,6 +2,7 @@ package tile_test import ( "bytes" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "io" "os" @@ -33,6 +34,16 @@ func TestReadReleaseFromFile(t *testing.T) { please.Expect(err).NotTo(HaveOccurred()) } +func TestReadBOSHReleaseManifestsFromTarballs(t *testing.T) { + boshReleases, err := tile.ReadBOSHReleaseManifestsFromTarballs(os.DirFS("testdata"), "bpm-1.1.21-ubuntu-xenial-621.463.tgz", "bpm-1.1.21.tgz") + require.NoError(t, err) + require.Len(t, boshReleases, 2) + assert.Equal(t, "be5b1710f33128f6c864eae1d97effddb94dd3ac", boshReleases[0].SHA1) + assert.Equal(t, "519b78f2f3333a7b9c000bbef325e12a2f36996d", boshReleases[1].SHA1) + assert.Equal(t, "bpm-1.1.21-ubuntu-xenial-621.463.tgz", boshReleases[0].FilePath) + assert.Equal(t, "bpm-1.1.21.tgz", boshReleases[1].FilePath) +} + func TestReadProductTemplatePartFromBOSHReleaseTarball(t *testing.T) { t.Run("when the release is compiled", func(t *testing.T) { f, err := os.Open(filepath.Join("testdata", "bpm-1.1.21-ubuntu-xenial-621.463.tgz"))