From e8f1cf2a95ab84c30c797c27743552165789abec Mon Sep 17 00:00:00 2001 From: Kevin Martin Date: Thu, 11 Jul 2024 16:13:43 -0400 Subject: [PATCH 1/6] Fix broken MD5 hash cache. This change fixes issues causing no matches to occur for existing objects at S3 in some cases. This issue caused all versions of a package in a repo to be uploaded again to S3. Lookups in the cache of file MD5s were failing as the key in the map had its prefix stripped while the key used for the lookup did not have the prefix stripped. This change stops stripping the prefix when building the cache. Retrieving object MD5s from object metadata (as opposed to the etag that comes with the object list) were failing as the key in the metadata map is now "md5" (all lowercase) vs. "Md5" that was used pre-v2 AWS SDK. --- s3/public.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/s3/public.go b/s3/public.go index 47623a001..f2b8702cd 100644 --- a/s3/public.go +++ b/s3/public.go @@ -191,7 +191,7 @@ func (storage *PublishedStorage) getMD5(path string) (string, error) { return "", err } - return output.Metadata["Md5"], nil + return output.Metadata["md5"], nil } // putFile uploads file-like object to @@ -431,11 +431,7 @@ func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaro continue } - if prefix == "" { - paths = append(paths, *key.Key) - } else { - paths = append(paths, (*key.Key)[len(prefix):]) - } + paths = append(paths, *key.Key) md5s = append(md5s, strings.Replace(*key.ETag, "\"", "", -1)) } } From e64398142f1c7b039d9e853dfa0b4e3f558b3df4 Mon Sep 17 00:00:00 2001 From: Kevin Martin Date: Thu, 11 Jul 2024 16:16:45 -0400 Subject: [PATCH 2/6] Update authors for new contributor. --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 70a75464b..78ff9dc6d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -63,3 +63,4 @@ List of contributors, in chronological order: * Ramón N.Rodriguez (https://github.com/runitonmetal) * Golf Hu (https://github.com/hudeng-go) * Cookie Fei (https://github.com/wuhuang26) +* Kevin Martin (https://github.com/melraidin) From 80ff35708dc8a676479e3b29716c4b6de8d8aea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 2 Oct 2024 20:26:54 +0200 Subject: [PATCH 3/6] system-tests: install swag only if needed --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3ef5169ff..aa0ac6f1e 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ docker-test: ## Run system tests @rm -f aptly.test go generate # install and initialize swagger - go install github.com/swaggo/swag/cmd/swag@latest + test -f $(BINPATH)/swag || go install github.com/swaggo/swag/cmd/swag@latest PATH=$(BINPATH)/:$(PATH) swag init # build coverage binary go test -v -coverpkg="./..." -c -tags testruncli From e7d37af8a38e1ae2fdcf3732873efa133a760978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 2 Oct 2024 21:26:37 +0200 Subject: [PATCH 4/6] revert not removing prefix --- s3/public.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/s3/public.go b/s3/public.go index f2b8702cd..2a5f76a91 100644 --- a/s3/public.go +++ b/s3/public.go @@ -430,8 +430,11 @@ func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaro /// from listing continue } - - paths = append(paths, *key.Key) + if prefix == "" { + paths = append(paths, *key.Key) + } else { + paths = append(paths, (*key.Key)[len(prefix):]) + } md5s = append(md5s, strings.Replace(*key.ETag, "\"", "", -1)) } } From 5224e01d49a9ebba0ce9dd2c7966083c3912eeca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 2 Oct 2024 21:26:57 +0200 Subject: [PATCH 5/6] make swagger quiet --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index aa0ac6f1e..839c7720d 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ prepare: ## Install go module dependencies go mod download # install and initialize swagger go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q go mod verify go mod tidy -v go generate @@ -71,7 +71,7 @@ docker-test: ## Run system tests go generate # install and initialize swagger test -f $(BINPATH)/swag || go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q # build coverage binary go test -v -coverpkg="./..." -c -tags testruncli @echo Running python tests ... @@ -126,7 +126,7 @@ releasetype: # Print release type (ci/release) build: ## Build aptly # install and initialize swagger unset GOBIN; go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q # prepare go mod tidy go generate @@ -137,7 +137,7 @@ dev-server: prepare ## Run dev-server go install github.com/air-verse/air@v1.52.3 cp debian/aptly.conf /var/lib/aptly/.aptly.conf sed -i /enableSwaggerEndpoint/s/false/true/ /var/lib/aptly/.aptly.conf - PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142 + PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142 dpkg: ## Build debian packages @test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1) @@ -145,7 +145,7 @@ dpkg: ## Build debian packages GOPATH=$$PWD/.go go generate -v # install and initialize swagger go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q # set debian version @if [ "`make -s releasetype`" = "ci" ]; then \ echo CI Build, setting version... ; \ @@ -169,7 +169,7 @@ binaries: ## Build binary releases (FreeBSD, MacOS, Linux tar) @make version > VERSION # install and initialize swagger GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest - PATH=$(BINPATH)/:$(PATH) swag init + PATH=$(BINPATH)/:$(PATH) swag init -q # build aptly GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static' # install From 80ab393f405dc4417e010c3c99e66f92ba95c315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 2 Oct 2024 21:27:21 +0200 Subject: [PATCH 6/6] fix test depending on gpg1 keys --- system/t06_publish/repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index fd3409d68..7e4b10c51 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -671,12 +671,12 @@ class PublishRepo26Test(BaseTest): """ publish repo: sign with passphrase """ - skipTest = "Failing on CI" fixtureCmds = [ "aptly repo create local-repo", "aptly repo add local-repo ${files}", + "gpg --import --batch --passphrase verysecret ${files}/aptly_passphrase.sec" ] - runCmd = "aptly publish repo -keyring=${files}/aptly_passphrase.pub -secret-keyring=${files}/aptly_passphrase.sec -passphrase=verysecret -distribution=maverick local-repo" + runCmd = "aptly publish repo -batch -keyring=${files}/aptly_passphrase.pub -passphrase=verysecret -distribution=maverick local-repo" gold_processor = BaseTest.expand_environ def outputMatchPrepare(_, s):