Fix/s3 pubilshing

AUTHORS

@@ -57,3 +57,4 @@ List of contributors, in chronological order:
 * Nicolas Dostert (https://github.com/acdn-ndostert)
 * Ryan Gonzalez (https://github.com/refi64)
 * Paul Cacheux (https://github.com/paulcacheux)
+* Nic Waller (https://github.com/sf-nwaller)

aptly/interfaces.go

@@ -70,7 +70,7 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
+	LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file

azure/public.go

@@ -22,7 +22,7 @@ import (
 type PublishedStorage struct {
 	container azblob.ContainerURL
 	prefix    string
-	pathCache map[string]string
+	pathCache map[string]map[string]string
 }
 
 // Check interface
@@ -174,31 +174,38 @@ func (storage *PublishedStorage) Remove(path string) error {
 
 // LinkFromPool links package file from pool to dist's pool location
 //
-// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// publishedPrefix is desired prefix for the location in the pool.
+// publishedRelParh is desired location in pool (like pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
-	relPath := filepath.Join(publishedDirectory, fileName)
-	poolPath := filepath.Join(storage.prefix, relPath)
+	relFilePath := filepath.Join(publishedRelPath, fileName)
+	prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	poolPath := filepath.Join(storage.prefix, prefixRelFilePath)
 
 	if storage.pathCache == nil {
-		paths, md5s, err := storage.internalFilelist("")
+		storage.pathCache = make(map[string]map[string]string)
+	}
+	pathCache := storage.pathCache[publishedPrefix]
+	if pathCache == nil {
+		paths, md5s, err := storage.internalFilelist(publishedPrefix)
 		if err != nil {
 			return fmt.Errorf("error caching paths under prefix: %s", err)
 		}
 
-		storage.pathCache = make(map[string]string, len(paths))
+		pathCache = make(map[string]string, len(paths))
 
 		for i := range paths {
-			storage.pathCache[paths[i]] = md5s[i]
+			pathCache[paths[i]] = md5s[i]
 		}
+		storage.pathCache[publishedPrefix] = pathCache
 	}
 
-	destinationMD5, exists := storage.pathCache[relPath]
+	destinationMD5, exists := pathCache[relFilePath]
 	sourceMD5 := sourceChecksums.MD5
 
 	if exists {
@@ -221,9 +228,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
 	}
 	defer source.Close()
 
-	err = storage.putFile(relPath, source, sourceMD5)
+	err = storage.putFile(prefixRelFilePath, source, sourceMD5)
 	if err == nil {
-		storage.pathCache[relPath] = sourceMD5
+		pathCache[relFilePath] = sourceMD5
 	} else {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
 	}

azure/public_test.go

@@ -300,45 +300,45 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 
 	// first link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// duplicate link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict and force
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
 
 	// for prefixed storage:
 	// first link from pool
-	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	// 2nd link from pool, providing wrong path for source file
 	//
 	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
 	s.prefixedStorage.pathCache = nil
-	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with nested file name
-	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))

cmd/publish_snapshot.go

@@ -162,8 +162,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
-		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
-			"the same package pool.\n")
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
 	}
 
 	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)

deb/index_files.go

@@ -143,19 +143,20 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	}
 
 	if signer != nil {
+		gpgExt := ".gpg"
 		if file.detachedSign {
-			err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+			err = signer.DetachedSign(file.tempFilename, file.tempFilename+gpgExt)
 			if err != nil {
 				return fmt.Errorf("unable to detached sign file: %s", err)
 			}
 
 			if file.parent.suffix != "" {
-				file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
-					filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+				file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt)] =
+					filepath.Join(file.parent.basePath, file.relativePath+gpgExt)
 			}
 
-			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
-				file.tempFilename+".gpg")
+			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt),
+				file.tempFilename+gpgExt)
 			if err != nil {
 				return fmt.Errorf("unable to publish file: %s", err)
 			}

deb/package.go

@@ -621,9 +621,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 			return err
 		}
 
-		publishedDirectory := filepath.Join(prefix, relPath)
-
-		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
+		err = publishedStorage.LinkFromPool(prefix, relPath, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
 		if err != nil {
 			return err
 		}

files/public.go

@@ -118,16 +118,17 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 
 // LinkFromPool links package file from pool to dist's pool location
 //
-// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// publishedPrefix is desired prefix for the location in the pool.
+// publishedRelParh is desired location in pool (like pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
 // sourcePath is a relative path to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
 	baseName := filepath.Base(fileName)
-	poolPath := filepath.Join(storage.rootPath, publishedDirectory, filepath.Dir(fileName))
+	poolPath := filepath.Join(storage.rootPath, publishedPrefix, publishedRelPath, filepath.Dir(fileName))
 
 	err := os.MkdirAll(poolPath, 0777)
 	if err != nil {

files/public_test.go

@@ -233,7 +233,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		c.Assert(err, IsNil)
 
 		// Test using hardlinks
-		err = s.storage.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		err = s.storage.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
@@ -243,7 +243,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		c.Check(int(info.Nlink), Equals, 3)
 
 		// Test using symlinks
-		err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		err = s.storageSymlink.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err = os.Lstat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename))
@@ -254,7 +254,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		c.Check(int(info.Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK))
 
 		// Test using copy with checksum verification
-		err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		err = s.storageCopy.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err = os.Stat(filepath.Join(s.storageCopy.rootPath, t.prefix, t.expectedFilename))
@@ -264,7 +264,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		c.Check(int(info.Nlink), Equals, 1)
 
 		// Test using copy with size verification
-		err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		err = s.storageCopySize.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err = os.Stat(filepath.Join(s.storageCopySize.rootPath, t.prefix, t.expectedFilename))
@@ -289,37 +289,37 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 	nlinks := int(st.Sys().(*syscall.Stat_t).Nlink)
 
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different")
 
 	st, err = pool.Stat(srcPoolPath)
 	c.Assert(err, IsNil)
 	c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks)
 
 	// linking with force
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
 	c.Check(err, IsNil)
 
 	st, err = pool.Stat(srcPoolPath)
 	c.Assert(err, IsNil)
 	c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks+1)
 
 	// Test using symlinks
-	err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	err = s.storageSymlink.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different")
 
-	err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
+	err = s.storageSymlink.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
 	c.Check(err, IsNil)
 
 	// Test using copy with checksum verification
-	err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	err = s.storageCopy.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different")
 
-	err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
+	err = s.storageCopy.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
 	c.Check(err, IsNil)
 
 	// Test using copy with size verification (this will NOT detect the difference)
-	err = s.storageCopySize.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	err = s.storageCopySize.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
 	c.Check(err, IsNil)
 }
 

s3/public.go

@@ -66,10 +66,11 @@ func NewPublishedStorageRaw(
 	plusWorkaround, disabledMultiDel, forceVirtualHostedStyle bool,
 	config *aws.Config,
 ) (*PublishedStorage, error) {
-
 	var acl types.ObjectCannedACL
-	if defaultACL == "" {
+	if defaultACL == "" || defaultACL == "private" {
 		acl = types.ObjectCannedACLPrivate
+	} else if defaultACL == "public-read" {
+		acl = types.ObjectCannedACLPublicRead
 	} else if defaultACL == "none" {
 		acl = ""
 	}
@@ -232,6 +233,9 @@ func (storage *PublishedStorage) Remove(path string) error {
 		// try to remove workaround version, but don't care about result
 		_ = storage.Remove(strings.Replace(path, "+", " ", -1))
 	}
+
+	delete(storage.pathCache, path)
+
 	return nil
 }
 
@@ -258,6 +262,7 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
 			if err != nil {
 				return fmt.Errorf("error deleting path %s from %s: %s", filelist[i], storage, err)
 			}
+			delete(storage.pathCache, filepath.Join(path, filelist[i]))
 		}
 	} else {
 		numParts := (len(filelist) + page - 1) / page
@@ -289,6 +294,9 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
 			if err != nil {
 				return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
 			}
+			for i := range part {
+				delete(storage.pathCache, filepath.Join(path, part[i]))
+			}
 		}
 	}
 
@@ -297,19 +305,21 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
 
 // LinkFromPool links package file from pool to dist's pool location
 //
-// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// publishedPrefix is desired prefix for the location in the pool.
+// publishedRelParh is desired location in pool (like pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
+	publishedDirectory := filepath.Join(publishedPrefix, publishedRelPath)
 	relPath := filepath.Join(publishedDirectory, fileName)
 	poolPath := filepath.Join(storage.prefix, relPath)
 
 	if storage.pathCache == nil {
-		paths, md5s, err := storage.internalFilelist("", true)
+		paths, md5s, err := storage.internalFilelist(filepath.Join(storage.prefix, publishedPrefix, "pool"), true)
 		if err != nil {
 			return errors.Wrap(err, "error caching paths under prefix")
 		}
@@ -490,6 +500,17 @@ func (storage *PublishedStorage) FileExists(path string) (bool, error) {
 			return false, nil
 		}
 
+		// falback in case the above condidition fails
+		var opErr *smithy.OperationError
+		if errors.As(err, &opErr) {
+			var ae smithy.APIError
+			if errors.As(err, &ae) {
+				if ae.ErrorCode() == "NotFound" {
+					return false, nil
+				}
+			}
+		}
+
 		return false, err
 	}