From d62ddcc5ea7e856c964e4368d20624e87733ef7f Mon Sep 17 00:00:00 2001 From: Ludovico Cavedon Date: Fri, 7 Apr 2023 17:43:31 -0700 Subject: [PATCH] s3: fix test --- s3/public_test.go | 33 ++++++++++++++++++++------------- s3/server_test.go | 21 +++++++++++++++++---- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/s3/public_test.go b/s3/public_test.go index a3586325a..5d96b5b6e 100644 --- a/s3/public_test.go +++ b/s3/public_test.go @@ -6,6 +6,8 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" + "strings" . "gopkg.in/check.v1" @@ -50,6 +52,17 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) { s.srv.Quit() } +func (s *PublishedStorageSuite) checkGetRequestsEqual(c *C, prefix string, expectedGetRequestUris []string) { + getRequests := make([]string, 0, len(s.srv.Requests)) + for _, r := range s.srv.Requests { + if r.Method == "GET" && strings.HasPrefix(r.RequestURI, prefix) { + getRequests = append(getRequests, r.RequestURI) + } + } + sort.Strings(getRequests) + c.Check(getRequests, DeepEquals, expectedGetRequestUris) +} + func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte { resp, err := s.storage.s3.GetObject(context.TODO(), &s3.GetObjectInput{ Bucket: aws.String(s.storage.bucket), @@ -301,7 +314,6 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { // 2nd link from pool, providing wrong path for source file // // this test should check that file already exists in S3 and skip upload (which would fail if not skipped) - s.prefixedStorage.pathCache = nil err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) c.Check(err, IsNil) @@ -335,7 +347,7 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) { c.Check(err, IsNil) // Check only one listing request was done to the server - s.checkGetRequestsEqual(c, "/test?", []string{"/test?max-keys=1000&prefix="}) + s.checkGetRequestsEqual(c, "/test?", []string{"/test?list-type=2&max-keys=1000&prefix=pool%2F"}) s.srv.Requests = nil // Publish two packages at a different prefix @@ -345,10 +357,8 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) { err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) - // Check only one listing request was done to the server - s.checkGetRequestsEqual(c, "/test?", []string{ - "/test?max-keys=1000&prefix=publish-prefix%2F", - }) + // Check no listing request was done to the server (pathCache is used) + s.checkGetRequestsEqual(c, "/test?", []string{}) s.srv.Requests = nil // Publish two packages at a prefixed storage @@ -360,24 +370,21 @@ func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) { // Check only one listing request was done to the server s.checkGetRequestsEqual(c, "/test?", []string{ - "/test?max-keys=1000&prefix=lala%2F", + "/test?list-type=2&max-keys=1000&prefix=lala%2Flala%2Fpool%2F", }) - s.srv.Requests = nil // Publish two packages at a prefixed storage plus a publish prefix. + s.srv.Requests = nil err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) - // Check only one listing request was done to the server - s.checkGetRequestsEqual(c, "/test?", []string{ - "/test?max-keys=1000&prefix=lala%2Fpublish-prefix%2F", - }) + // Check no listing request was done to the server (pathCache is used) + s.checkGetRequestsEqual(c, "/test?", []string{}) // This step checks that files already exists in S3 and skip upload (which would fail if not skipped). - s.prefixedStorage.pathCache = nil err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false) c.Check(err, IsNil) err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false) diff --git a/s3/server_test.go b/s3/server_test.go index 573097abe..08c290406 100644 --- a/s3/server_test.go +++ b/s3/server_test.go @@ -59,6 +59,12 @@ func (c *Config) send409Conflict() bool { return false } +// Request stores the method and URI of an HTTP request. +type Request struct { + Method string + RequestURI string +} + // Server is a fake S3 server for testing purposes. // All of the data for the server is kept in memory. type Server struct { @@ -68,6 +74,8 @@ type Server struct { mu sync.Mutex buckets map[string]*bucket config *Config + // Requests holds a log of all requests received by the server. + Requests []Request } type bucket struct { @@ -140,6 +148,7 @@ func (srv *Server) serveHTTP(w http.ResponseWriter, req *http.Request) { if debug { log.Printf("s3test %q %q", req.Method, req.URL) } + srv.Requests = append(srv.Requests, Request{req.Method, req.RequestURI}) a := &action{ srv: srv, w: w, @@ -330,6 +339,10 @@ type Owner struct { DisplayName string } +type CommonPrefix struct { + Prefix string +} + // The ListResp type holds the results of a List bucket operation. type ListResp struct { Name string @@ -344,7 +357,7 @@ type ListResp struct { // http://goo.gl/YjQTc IsTruncated bool Contents []Key - CommonPrefixes []string `xml:">Prefix"` + CommonPrefixes []CommonPrefix } // The Key type represents an item stored in an S3 bucket. @@ -403,7 +416,7 @@ func (r bucketResource) get(a *action) interface{} { MaxKeys: maxKeys, } - var prefixes []string + var prefixes []CommonPrefix for _, obj := range objs { if !strings.HasPrefix(obj.name, prefix) { continue @@ -413,7 +426,7 @@ func (r bucketResource) get(a *action) interface{} { if delimiter != "" { if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 { name = obj.name[:len(prefix)+i+len(delimiter)] - if prefixes != nil && prefixes[len(prefixes)-1] == name { + if prefixes != nil && prefixes[len(prefixes)-1].Prefix == name { continue } isPrefix = true @@ -427,7 +440,7 @@ func (r bucketResource) get(a *action) interface{} { break } if isPrefix { - prefixes = append(prefixes, name) + prefixes = append(prefixes, CommonPrefix{name}) } else { // Contents contains only keys not found in CommonPrefixes resp.Contents = append(resp.Contents, obj.s3Key())