add function to tile package for reading metadata from a tanzunet product file

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
+	github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270
 	github.com/aws/aws-sdk-go v1.44.95
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cloudfoundry/bosh-cli v6.4.1+incompatible
@@ -31,8 +32,10 @@ require (
 	github.com/opencontainers/image-spec v1.0.3-0.20220303224323-02efb9a75ee1
 	github.com/pivotal-cf-experimental/gomegamatchers v0.0.0-20180326192815-e36bfcc98c3a
 	github.com/pivotal-cf/go-pivnet/v2 v2.0.11
+	github.com/pivotal-cf/go-pivnet/v7 v7.0.2
 	github.com/pivotal-cf/jhanda v0.0.0-20200619200912-8de8eb943a43
 	github.com/pivotal-cf/om v0.0.0-20211027143906-30b10602e528
+	github.com/snabb/httpreaderat v1.0.1
 	github.com/stretchr/testify v1.8.2
 	golang.org/x/crypto v0.7.0
 	golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2

go.sum

@@ -118,6 +118,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270 h1:JIxGEMs4E5Zb6R7z2C5IgecI0mkqS97WAEF31wUbYTM=
+github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270/go.mod h1:2XtVRGCw/HthOLxU0Qw6o6jSJrcEoOb2OCCl8gQYvGw=
 github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.36.19/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.44.95 h1:QwmA+PeR6v4pF0f/dPHVPWGAshAhb9TnGZBTM5uKuI8=
@@ -628,6 +630,8 @@ github.com/pivotal-cf/go-pivnet v1.0.3/go.mod h1:rvEzWli4NJQhX7Z3z0DiEQXsPwC+uE/
 github.com/pivotal-cf/go-pivnet/v2 v2.0.11 h1:6tzC4zOr7acFPevfP32+8rEU567JlDobqiIejFf8aOc=
 github.com/pivotal-cf/go-pivnet/v2 v2.0.11/go.mod h1:Ormn4YO2MooU40LNFwECoLd8XZFwbRcysET+OAn7FLg=
 github.com/pivotal-cf/go-pivnet/v6 v6.0.2/go.mod h1:ymI4gZvp8lf3GNaf1Re+JRV18zU0w8TLBDQB/t8SaFs=
+github.com/pivotal-cf/go-pivnet/v7 v7.0.2 h1:8IYUIXmFEJpuTSIuOM5K4z0tJLPULDPpKJfn+wWLhqk=
+github.com/pivotal-cf/go-pivnet/v7 v7.0.2/go.mod h1:h868WmuLUuavwNHoC6FyGI3hK/6rP3/HCmM8sYLuC8c=
 github.com/pivotal-cf/jhanda v0.0.0-20200619200912-8de8eb943a43 h1:SYEUxVbqz3U7pJP/tlaXaD08w0rZVuPvCFRodnw/TLY=
 github.com/pivotal-cf/jhanda v0.0.0-20200619200912-8de8eb943a43/go.mod h1:UXciri1Yqno0IdXxEzwMF91nnwYMPoN95goHWxVtWq8=
 github.com/pivotal-cf/om v0.0.0-20211027143906-30b10602e528 h1:XYJJ3cozlyyEs9IhoQLA2ZE9GpT4qrtM8V9FhXF4PqY=
@@ -700,6 +704,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/snabb/httpreaderat v1.0.1 h1:whlb+vuZmyjqVop8x1EKOg05l2NE4z9lsMMXjmSUCnY=
+github.com/snabb/httpreaderat v1.0.1/go.mod h1:lpbGrKDWF37yvRbtRvQsbesS6Ty5c83t8ztannPoMsA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=

pkg/tile/metadata.go

@@ -5,7 +5,11 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"net/http"
 	"os"
+
+	bufferReaderAt "github.com/avvmoto/buf-readerat"
+	"github.com/snabb/httpreaderat"
 )
 
 func ReadMetadataFromFile(tilePath string) ([]byte, error) {
@@ -50,6 +54,21 @@ func ReadMetadataFromFS(dir fs.FS) ([]byte, error) {
 	return buf, nil
 }
 
+const (
+	// readerAtCacheSize is 1mb
+	readerAtCacheSize = 1 << 20
+)
+
+// ReadMetadataFromServer can download the metadata from a product file on TanzuNet.
+func ReadMetadataFromServer(client *http.Client, req *http.Request) ([]byte, error) {
+	ra, err := httpreaderat.New(client, req, nil)
+	if err != nil {
+		return nil, err
+	}
+	bufRa := bufferReaderAt.NewBufReaderAt(ra, readerAtCacheSize)
+	return ReadMetadataFromZip(bufRa, ra.Size())
+}
+
 func closeAndIgnoreError(c io.Closer) {
 	_ = c.Close()
 }

pkg/tile/metadata_test.go

@@ -1,36 +1,79 @@
 package tile_test
 
 import (
+	"context"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"testing/fstest"
 
-	. "github.com/onsi/gomega"
-	"gopkg.in/yaml.v2"
+	"github.com/pivotal-cf/go-pivnet/v7"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
 
 	"github.com/pivotal-cf/kiln/pkg/tile"
 )
 
 func TestReadMetadataFromFile(t *testing.T) {
-	please := NewWithT(t)
-
 	metadataBytes, err := tile.ReadMetadataFromFile("testdata/tile-0.1.2.pivotal")
-	please.Expect(err).NotTo(HaveOccurred())
+	require.NoError(t, err)
 
 	var metadata struct {
 		Name string `yaml:"name"`
 	}
 	err = yaml.Unmarshal(metadataBytes, &metadata)
-	please.Expect(err).NotTo(HaveOccurred(), string(metadataBytes))
+	require.NoError(t, err)
 
-	please.Expect(metadata.Name).To(Equal("hello"), string(metadataBytes))
+	assert.Equal(t, metadata.Name, "hello")
 }
 
 func TestNonStandardMetadataFilename(t *testing.T) {
 	fileFS := fstest.MapFS{
 		"metadata/banana.yml": &fstest.MapFile{Data: []byte(`{name: "banana"}`)},
 	}
 	buf, err := tile.ReadMetadataFromFS(fileFS)
-	please := NewWithT(t)
-	please.Expect(err).NotTo(HaveOccurred())
-	please.Expect(string(buf)).To(Equal(`{name: "banana"}`))
+	require.NoError(t, err)
+
+	assert.Equal(t, string(buf), "{name: \"banana\"}")
+}
+
+func TestReadMetadataFromServer(t *testing.T) {
+	httpClient, productFile := setupReadMetadataFromServer(t)
+
+	// create http.Request from pivnet.ProductFile
+	downloadLink, err := productFile.DownloadLink()
+	require.NoError(t, err)
+	ctx := context.Background()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadLink, nil)
+	require.NoError(t, err)
+	// on a real request you need to set Authorization and User-Agent headers
+
+	metadataBytes, err := tile.ReadMetadataFromServer(httpClient, req)
+	require.NoError(t, err)
+
+	var metadata struct {
+		Name string `yaml:"name"`
+	}
+	err = yaml.Unmarshal(metadataBytes, &metadata)
+	require.NoError(t, err)
+
+	assert.Equal(t, "hello", metadata.Name)
+}
+
+func setupReadMetadataFromServer(t *testing.T) (*http.Client, pivnet.ProductFile) {
+	t.Helper()
+	fileServer := http.FileServer(http.Dir("testdata"))
+	server := httptest.NewServer(fileServer)
+	t.Cleanup(server.Close)
+
+	// productFile should come from an API request:
+	// _ = pivnet.ProductFilesService.ListForRelease
+	productFile := pivnet.ProductFile{
+		Links: &pivnet.Links{
+			Download: map[string]string{"href": server.URL + "/tile-0.1.2.pivotal"},
+		},
+	}
+
+	return server.Client(), productFile
 }