Skip to content

Commit

Permalink
s3: fix test
Browse files Browse the repository at this point in the history
  • Loading branch information
@cavedon @neolynx
cavedon authored and neolynx committed Jan 21, 2024
1 parent 5541ac9 commit 2a5c4be
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 17 deletions.
33 changes: 20 additions & 13 deletions s3/public_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"

. "gopkg.in/check.v1"

Expand Down Expand Up @@ -50,6 +52,17 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
s.srv.Quit()
}

func (s *PublishedStorageSuite) checkGetRequestsEqual(c *C, prefix string, expectedGetRequestUris []string) {
getRequests := make([]string, 0, len(s.srv.Requests))
for _, r := range s.srv.Requests {
if r.Method == "GET" && strings.HasPrefix(r.RequestURI, prefix) {
getRequests = append(getRequests, r.RequestURI)
}
}
sort.Strings(getRequests)
c.Check(getRequests, DeepEquals, expectedGetRequestUris)
}

func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
resp, err := s.storage.s3.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(s.storage.bucket),
Expand Down Expand Up @@ -301,7 +314,6 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
// 2nd link from pool, providing wrong path for source file
//
// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
s.prefixedStorage.pathCache = nil
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
c.Check(err, IsNil)

Expand Down Expand Up @@ -335,7 +347,7 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) {
c.Check(err, IsNil)

// Check only one listing request was done to the server
s.checkGetRequestsEqual(c, "/test?", []string{"/test?max-keys=1000&prefix="})
s.checkGetRequestsEqual(c, "/test?", []string{"/test?list-type=2&max-keys=1000&prefix=pool%2F"})

s.srv.Requests = nil
// Publish two packages at a different prefix
Expand All @@ -345,10 +357,8 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) {
err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)

// Check only one listing request was done to the server
s.checkGetRequestsEqual(c, "/test?", []string{
"/test?max-keys=1000&prefix=publish-prefix%2F",
})
// Check no listing request was done to the server (pathCache is used)
s.checkGetRequestsEqual(c, "/test?", []string{})

s.srv.Requests = nil
// Publish two packages at a prefixed storage
Expand All @@ -360,24 +370,21 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) {

// Check only one listing request was done to the server
s.checkGetRequestsEqual(c, "/test?", []string{
"/test?max-keys=1000&prefix=lala%2F",
"/test?list-type=2&max-keys=1000&prefix=lala%2Flala%2Fpool%2F",
})

s.srv.Requests = nil
// Publish two packages at a prefixed storage plus a publish prefix.
s.srv.Requests = nil
err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)

err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)

// Check only one listing request was done to the server
s.checkGetRequestsEqual(c, "/test?", []string{
"/test?max-keys=1000&prefix=lala%2Fpublish-prefix%2F",
})
// Check no listing request was done to the server (pathCache is used)
s.checkGetRequestsEqual(c, "/test?", []string{})

// This step checks that files already exists in S3 and skip upload (which would fail if not skipped).
s.prefixedStorage.pathCache = nil
err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false)
c.Check(err, IsNil)
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false)
Expand Down
21 changes: 17 additions & 4 deletions s3/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ func (c *Config) send409Conflict() bool {
return false
}

// Request stores the method and URI of an HTTP request.
type Request struct {
Method string
RequestURI string
}

// Server is a fake S3 server for testing purposes.
// All of the data for the server is kept in memory.
type Server struct {
Expand All @@ -68,6 +74,8 @@ type Server struct {
mu sync.Mutex
buckets map[string]*bucket
config *Config
// Requests holds a log of all requests received by the server.
Requests []Request
}

type bucket struct {
Expand Down Expand Up @@ -140,6 +148,7 @@ func (srv *Server) serveHTTP(w http.ResponseWriter, req *http.Request) {
if debug {
log.Printf("s3test %q %q", req.Method, req.URL)
}
srv.Requests = append(srv.Requests, Request{req.Method, req.RequestURI})
a := &action{
srv: srv,
w: w,
Expand Down Expand Up @@ -330,6 +339,10 @@ type Owner struct {
DisplayName string
}

type CommonPrefix struct {
Prefix string
}

// The ListResp type holds the results of a List bucket operation.
type ListResp struct {
Name string
Expand All @@ -344,7 +357,7 @@ type ListResp struct {
// http://goo.gl/YjQTc
IsTruncated bool
Contents []Key
CommonPrefixes []string `xml:">Prefix"`
CommonPrefixes []CommonPrefix
}

// The Key type represents an item stored in an S3 bucket.
Expand Down Expand Up @@ -403,7 +416,7 @@ func (r bucketResource) get(a *action) interface{} {
MaxKeys: maxKeys,
}

var prefixes []string
var prefixes []CommonPrefix
for _, obj := range objs {
if !strings.HasPrefix(obj.name, prefix) {
continue
Expand All @@ -413,7 +426,7 @@ func (r bucketResource) get(a *action) interface{} {
if delimiter != "" {
if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 {
name = obj.name[:len(prefix)+i+len(delimiter)]
if prefixes != nil && prefixes[len(prefixes)-1] == name {
if prefixes != nil && prefixes[len(prefixes)-1].Prefix == name {
continue
}
isPrefix = true
Expand All @@ -427,7 +440,7 @@ func (r bucketResource) get(a *action) interface{} {
break
}
if isPrefix {
prefixes = append(prefixes, name)
prefixes = append(prefixes, CommonPrefix{name})
} else {
// Contents contains only keys not found in CommonPrefixes
resp.Contents = append(resp.Contents, obj.s3Key())
Expand Down

0 comments on commit 2a5c4be

Please sign in to comment.