improve and test grab downloader

aptly-dev · Oct 4, 2024 · f0ad0f9 · f0ad0f9
1 parent d6a156b
commit f0ad0f9
Show file tree

Hide file tree

Showing 4 changed files with 106 additions and 5 deletions.
diff --git a/http/grab.go b/http/grab.go
@@ -59,14 +59,13 @@ func (d *GrabDownloader) DownloadWithChecksum(ctx context.Context, url string, d
 			// Success
 			break
 		}
-		d.log("Download Error: %v\n", err)
 		if retryableError(err) {
 			maxTries--
-			d.log("Retrying download %s: %d\n", url, maxTries)
+			d.log("Retrying %d %s\n", maxTries, url)
 			time.Sleep(delay)
 		} else {
 			// Can't retry
-			d.log("Cannot retry download %s\n", url)
+			d.log("Error (retrying): HTTP code %s while fetching %s\n", err, url)
 			break
 		}
 	}
@@ -115,8 +114,7 @@ func (d *GrabDownloader) maybeSetupChecksum(req *grab.Request, expected *utils.C
 }
 
 func (d *GrabDownloader) download(_ context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
-	// TODO clean up dest dir on permanent failure
-	d.log("Download %s -> %s\n", url, destination)
+	d.log("Downloading: %s\n", url)
 
 	req, err := grab.NewRequest(destination, url)
 	if err != nil {

diff --git a/system/t04_mirror/UpdateMirror26Test_gold b/system/t04_mirror/UpdateMirror26Test_gold
@@ -0,0 +1,33 @@
+
+
+Applying filter...
+Building download queue...
+Download queue: 1 items (3.35 KiB)
+Downloading & parsing package files...
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
+Mirror `grab` has been successfully updated.
+Packages filtered: 50604 -> 1.
+Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
+Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
+gpgv:                issuer "debian-release@lists.debian.org"
+gpgv:                using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138
+gpgv:                using RSA key 067E3C456BAE240ACEE88F6FEF0F382A1A7B6500
+gpgv:                using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC
+gpgv:                using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9
+gpgv: Good signature from "Debian Archive Automatic Signing Key (10/buster) <ftpmaster@debian.org>"
+gpgv: Good signature from "Debian Archive Automatic Signing Key (11/bullseye) <ftpmaster@debian.org>"
+gpgv: Good signature from "Debian Archive Automatic Signing Key (9/stretch) <ftpmaster@debian.org>"
+gpgv: Good signature from "Debian Stable Release Key (9/stretch) <debian-release@lists.debian.org>"
+gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC
+gpgv: Signature made Sat Aug 14 07:43:25 2021 UTC
+gpgv: Signature made Sat Aug 14 08:26:43 2021 UTC
+gpgv: Signature made Sat Aug 14 08:46:19 2021 UTC
diff --git a/system/t04_mirror/UpdateMirror27Test_gold b/system/t04_mirror/UpdateMirror27Test_gold
@@ -0,0 +1,39 @@
+
+  server returned 404 Not Found
+Applying filter...
+Building download queue...
+Download queue: 2 items (6.70 KiB)
+Downloading & parsing package files...
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-amd64/Packages.gz
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-amd64/Packages.gz
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
+ERROR: unable to update: download errors:
+Packages filtered: 76844 -> 2.
+Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
+Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
+Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
+Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
+gpgv:                issuer "debian-release@lists.debian.org"
+gpgv:                using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138
+gpgv:                using RSA key 067E3C456BAE240ACEE88F6FEF0F382A1A7B6500
+gpgv:                using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC
+gpgv:                using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9
+gpgv: Good signature from "Debian Archive Automatic Signing Key (10/buster) <ftpmaster@debian.org>"
+gpgv: Good signature from "Debian Archive Automatic Signing Key (11/bullseye) <ftpmaster@debian.org>"
+gpgv: Good signature from "Debian Archive Automatic Signing Key (9/stretch) <ftpmaster@debian.org>"
+gpgv: Good signature from "Debian Stable Release Key (9/stretch) <debian-release@lists.debian.org>"
+gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC
+gpgv: Signature made Sat Aug 14 07:43:25 2021 UTC
+gpgv: Signature made Sat Aug 14 08:26:43 2021 UTC
+gpgv: Signature made Sat Aug 14 08:46:19 2021 UTC
diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py
@@ -451,3 +451,34 @@ class UpdateMirror25Test(BaseTest):
     ]
     runCmd = "aptly mirror update -keyring=aptlytest.gpg mirror19"
     outputMatchPrepare = filterOutSignature
+
+
+class UpdateMirror26Test(BaseTest):
+    """
+    update mirrors: regular update, grab downloader
+    """
+    configOverride = {"downloader": "grab"}
+    sortOutput = True
+    longTest = False
+    fixtureGpg = True
+    fixtureCmds = [
+        "aptly mirror create -architectures=i386 -keyring=aptlytest.gpg -filter=libboost-program-options-dev grab http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian stretch main",
+    ]
+    runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab"
+    outputMatchPrepare = filterOutRedirects
+
+
+class UpdateMirror27Test(BaseTest):
+    """
+    update mirrors: failing update, grab downloader
+    """
+    configOverride = {"downloader": "grab"}
+    sortOutput = True
+    longTest = False
+    fixtureGpg = True
+    fixtureCmds = [
+        "aptly mirror create -architectures=amd64,i386 -keyring=aptlytest.gpg -filter=libboost-program-options-dev grab-fail http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian stretch main",
+    ]
+    runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab-fail"
+    outputMatchPrepare = filterOutRedirects
+    expectedCode = 1