test: add containerd e2e testings for API v2

Signed-off-by: Gaius <gaius.qi@gmail.com>

.github/workflows/check-size.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [main, release-*]
   pull_request:
-    branches: [amain, release-*]
+    branches: [main, release-*]
 
 env:
   GO_VERSION: '1.21'

.github/workflows/ci.yml

@@ -5,7 +5,7 @@ on:
     branches: [main, release-*]
     paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
   pull_request:
-    branches: [amain, release-*]
+    branches: [main, release-*]
     paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
   schedule:
     - cron: '0 4 * * *'

.github/workflows/codeql-analysis.yml

@@ -5,7 +5,7 @@ on:
     branches: [main, release-*]
     paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
   pull_request:
-    branches: [amain, release-*]
+    branches: [main, release-*]
     paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
   schedule:
     - cron: '0 4 * * *'

.github/workflows/compatibility-e2e-v1.yml

@@ -5,7 +5,7 @@ on:
     branches: [main, release-*]
     paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
   pull_request:
-    branches: [amain, release-*]
+    branches: [main, release-*]
     paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
   schedule:
     - cron: '0 4 * * *'

.github/workflows/e2e-v1.yml

@@ -5,7 +5,7 @@ on:
     branches: [main, release-*]
     paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
   pull_request:
-    branches: [amain, release-*]
+    branches: [main, release-*]
     paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
   schedule:
     - cron: '0 4 * * *'

.github/workflows/lint.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [main, release-*]
   pull_request:
-    branches: [amain, release-*]
+    branches: [main, release-*]
 
 env:
   GO_VERSION: '1.21'

.github/workflows/nydus-e2e.yml

@@ -5,7 +5,7 @@ on:
     branches: [main, release-*]
     paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
   pull_request:
-    branches: [amain, release-*]
+    branches: [main, release-*]
     paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
   schedule:
     - cron: '0 4 * * *'

.gitignore

@@ -67,3 +67,5 @@ artifacts
 .run
 deploy/docker-compose/log
 deploy/docker-compose/config
+test/e2e/v1/*.log
+test/e2e/v2/*.log

test/e2e/v1/manager/preheat.go

@@ -38,10 +38,15 @@ import (
 	"d7y.io/dragonfly/v2/test/e2e/v1/util"
 )
 
+type taskMetadata struct {
+	ID     string
+	Sha256 string
+}
+
 var _ = Describe("Preheat with manager", func() {
 	Context("preheat", func() {
 		It("preheat files should be ok", Label("preheat", "file"), func() {
-			var seedPeerPods [3]*util.PodExec
+			seedPeerPods := make([]*util.PodExec, 3)
 			for i := 0; i < 3; i++ {
 				seedPeerPods[i] = getSeedPeerExec(i)
 			}
@@ -55,7 +60,7 @@ var _ = Describe("Preheat with manager", func() {
 				out, err := util.DockerCommand("sha256sum", v).CombinedOutput()
 				fmt.Println("original sha256sum: " + string(out))
 				Expect(err).NotTo(HaveOccurred())
-				sha256sum1 := strings.Split(string(out), " ")[0]
+				taskSha256 := strings.Split(string(out), " ")[0]
 
 				// preheat file
 				req, err := structure.StructToMap(types.CreatePreheatJobRequest{
@@ -80,31 +85,31 @@ var _ = Describe("Preheat with manager", func() {
 				Expect(done).Should(BeTrue())
 
 				// generate task_id, also the filename
-				seedPeerTaskID := idgen.TaskIDV1(url, &commonv1.UrlMeta{})
-				fmt.Println(seedPeerTaskID)
+				taskID := idgen.TaskIDV1(url, &commonv1.UrlMeta{})
+				fmt.Println(taskID)
 
-				sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID)
+				sha256sum, err := calculateSha256ByTaskID(seedPeerPods, taskID)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(sha256sum1).To(Equal(sha256sum))
+				Expect(taskSha256).To(Equal(sha256sum))
 			}
 		})
 
 		It("preheat image should be ok", Label("preheat", "image"), func() {
 			url := "https://index.docker.io/v2/dragonflyoss/busybox/manifests/1.35.0"
 			fmt.Println("download image: " + url)
 
-			var (
-				seedPeerTaskIDs = []string{
-					"b6922209dc9616f8736a860e93c3cd7288a4e801517f88eec3df514606d18cdf",
-					"c0dfae864ae65c285676063eb148d0a0064d5c6c39367fee0bcc1f3700c39c31",
-				}
-				sha256sum1 = []string{
-					"a711f05d33845e2e9deffcfcc5adf082d7c6e97e3e3a881d193d9aae38f092a8",
-					"f643e116a03d9604c344edb345d7592c48cc00f2a4848aaf773411f4fb30d2f5",
-				}
-			)
-
-			var seedPeerPods [3]*util.PodExec
+			taskMetadatas := []taskMetadata{
+				{
+					ID:     "b6922209dc9616f8736a860e93c3cd7288a4e801517f88eec3df514606d18cdf",
+					Sha256: "a711f05d33845e2e9deffcfcc5adf082d7c6e97e3e3a881d193d9aae38f092a8",
+				},
+				{
+					ID:     "c0dfae864ae65c285676063eb148d0a0064d5c6c39367fee0bcc1f3700c39c31",
+					Sha256: "f643e116a03d9604c344edb345d7592c48cc00f2a4848aaf773411f4fb30d2f5",
+				},
+			}
+
+			seedPeerPods := make([]*util.PodExec, 3)
 			for i := 0; i < 3; i++ {
 				seedPeerPods[i] = getSeedPeerExec(i)
 			}
@@ -132,10 +137,10 @@ var _ = Describe("Preheat with manager", func() {
 			done := waitForDone(job, fsPod)
 			Expect(done).Should(BeTrue())
 
-			for i, seedPeerTaskID := range seedPeerTaskIDs {
-				sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID)
+			for _, taskMetadata := range taskMetadatas {
+				sha256sum, err := calculateSha256ByTaskID(seedPeerPods, taskMetadata.ID)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(sha256sum1[i]).To(Equal(sha256sum))
+				Expect(taskMetadata.Sha256).To(Equal(sha256sum))
 			}
 		})
 
@@ -148,24 +153,30 @@ var _ = Describe("Preheat with manager", func() {
 			url := "https://index.docker.io/v2/dragonflyoss/scheduler/manifests/v2.1.0"
 			fmt.Println("download image: " + url)
 
-			var (
-				seedPeerTaskIDs = []string{
-					"c8ca6a17354d3a79397eef26803e5af84d00a3fd64b0f823922086a31ebdee18",
-					"b8de5865e2ebf537279683adfbdb5f858b0c7212e5744a1df233086496c245d7",
-					"e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806",
-					"7da0721fd078dd46a63298747ffde8fcbe12b53378f282c9def693615ac7993e",
-					"3639c8c5712e77acd3751142c83150c0a12284a54fa41224a1c7acc0e343020d",
-				}
-				sha256sum1 = []string{
-					"f1f1039835051ecc04909f939530e86a20f02d2ce5ad7a81c0fa3616f7303944",
-					"c1d6d1b2d5a367259e6e51a7f4d1ccd66a28cc9940d6599d8a8ea9544dd4b4a8",
-					"871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7",
-					"f1a1d290795d904815786e41d39a41dc1af5de68a9e9020baba8bd83b32d8f95",
-					"f1ffc4b5459e82dc8e7ddd1d1a2ec469e85a1f076090c22851a1f2ce6f71e1a6",
-				}
-			)
-
-			var seedPeerPods [3]*util.PodExec
+			taskMetadatas := []taskMetadata{
+				{
+					ID:     "c8ca6a17354d3a79397eef26803e5af84d00a3fd64b0f823922086a31ebdee18",
+					Sha256: "f1f1039835051ecc04909f939530e86a20f02d2ce5ad7a81c0fa3616f7303944",
+				},
+				{
+					ID:     "b8de5865e2ebf537279683adfbdb5f858b0c7212e5744a1df233086496c245d7",
+					Sha256: "c1d6d1b2d5a367259e6e51a7f4d1ccd66a28cc9940d6599d8a8ea9544dd4b4a8",
+				},
+				{
+					ID:     "e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806",
+					Sha256: "871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7",
+				},
+				{
+					ID:     "7da0721fd078dd46a63298747ffde8fcbe12b53378f282c9def693615ac7993e",
+					Sha256: "f1a1d290795d904815786e41d39a41dc1af5de68a9e9020baba8bd83b32d8f95",
+				},
+				{
+					ID:     "3639c8c5712e77acd3751142c83150c0a12284a54fa41224a1c7acc0e343020d",
+					Sha256: "f1ffc4b5459e82dc8e7ddd1d1a2ec469e85a1f076090c22851a1f2ce6f71e1a6",
+				},
+			}
+
+			seedPeerPods := make([]*util.PodExec, 3)
 			for i := 0; i < 3; i++ {
 				seedPeerPods[i] = getSeedPeerExec(i)
 			}
@@ -194,10 +205,10 @@ var _ = Describe("Preheat with manager", func() {
 			done := waitForDone(job, fsPod)
 			Expect(done).Should(BeTrue())
 
-			for i, seedPeerTaskID := range seedPeerTaskIDs {
-				sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID)
+			for _, taskMetadata := range taskMetadatas {
+				sha256sum, err := calculateSha256ByTaskID(seedPeerPods, taskMetadata.ID)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(sha256sum1[i]).To(Equal(sha256sum))
+				Expect(taskMetadata.Sha256).To(Equal(sha256sum))
 			}
 		})
 
@@ -210,24 +221,30 @@ var _ = Describe("Preheat with manager", func() {
 			url := "https://index.docker.io/v2/dragonflyoss/scheduler/manifests/v2.1.0"
 			fmt.Println("download image: " + url)
 
-			var (
-				seedPeerTaskIDs = []string{
-					"9869dbb01ac214e90e4ae667e42d50210c2ff1e63292d73b14f0a7a2226c0320",
-					"ab049caee13f77d91568d954a5d32f5d2354497cab098887a8a663656daa9840",
-					"e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806",
-					"a26e1ac8b70926f45766fcf886f23a833793c39c62237bcda9ffeb158131c0d6",
-					"7376f665077e91cd0dc410c00242ab88775e3eae19eca4b7b3a29ded14fc3754",
-				}
-				sha256sum1 = []string{
-					"a0d7a8f11f7e25ca59f0bf470187dd9aa27e7ca951cf67a53c750deea5d3b076",
-					"a880266d3b77f75696023df2da1ef66c3c565e0f70596242395c9e68de955c7c",
-					"871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7",
-					"9b5952218d7711195c6c6fbddbef2780507d20851ca68845d180397d1348f0d8",
-					"889f4c960ac4ff70774e9c4cfa64efc4823ade0702d0f96c20ff0054ffbbe504",
-				}
-			)
-
-			var seedPeerPods [3]*util.PodExec
+			taskMetadatas := []taskMetadata{
+				{
+					ID:     "9869dbb01ac214e90e4ae667e42d50210c2ff1e63292d73b14f0a7a2226c0320",
+					Sha256: "a0d7a8f11f7e25ca59f0bf470187dd9aa27e7ca951cf67a53c750deea5d3b076",
+				},
+				{
+					ID:     "ab049caee13f77d91568d954a5d32f5d2354497cab098887a8a663656daa9840",
+					Sha256: "a880266d3b77f75696023df2da1ef66c3c565e0f70596242395c9e68de955c7c",
+				},
+				{
+					ID:     "e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806",
+					Sha256: "871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7",
+				},
+				{
+					ID:     "a26e1ac8b70926f45766fcf886f23a833793c39c62237bcda9ffeb158131c0d6",
+					Sha256: "9b5952218d7711195c6c6fbddbef2780507d20851ca68845d180397d1348f0d8",
+				},
+				{
+					ID:     "7376f665077e91cd0dc410c00242ab88775e3eae19eca4b7b3a29ded14fc3754",
+					Sha256: "889f4c960ac4ff70774e9c4cfa64efc4823ade0702d0f96c20ff0054ffbbe504",
+				},
+			}
+
+			seedPeerPods := make([]*util.PodExec, 3)
 			for i := 0; i < 3; i++ {
 				seedPeerPods[i] = getSeedPeerExec(i)
 			}
@@ -256,10 +273,10 @@ var _ = Describe("Preheat with manager", func() {
 			done := waitForDone(job, fsPod)
 			Expect(done).Should(BeTrue())
 
-			for i, seedPeerTaskID := range seedPeerTaskIDs {
-				sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID)
+			for _, taskMetadata := range taskMetadatas {
+				sha256sum, err := calculateSha256ByTaskID(seedPeerPods, taskMetadata.ID)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(sha256sum1[i]).To(Equal(sha256sum))
+				Expect(taskMetadata.Sha256).To(Equal(sha256sum))
 			}
 		})
 	})
@@ -294,18 +311,18 @@ func waitForDone(preheat *models.Job, pod *util.PodExec) bool {
 	}
 }
 
-func checkPreheatResult(seedPeerPods [3]*util.PodExec, seedPeerTaskID string) (string, error) {
+func calculateSha256ByTaskID(pods []*util.PodExec, taskID string) (string, error) {
 	var sha256sum string
-	for _, seedPeer := range seedPeerPods {
-		taskDir := fmt.Sprintf("%s/%s", seedPeerDataPath, seedPeerTaskID)
-		if _, err := seedPeer.Command("ls", taskDir).CombinedOutput(); err != nil {
+	for _, pod := range pods {
+		taskDir := fmt.Sprintf("%s/%s", seedPeerDataPath, taskID)
+		if _, err := pod.Command("ls", taskDir).CombinedOutput(); err != nil {
 			// if the directory does not exist, skip this seed peer
 			fmt.Printf("directory %s does not exist: %s\n", taskDir, err.Error())
 			continue
 		}
 
 		// calculate digest of downloaded file
-		out, err := seedPeer.Command("sh", "-c", fmt.Sprintf("sha256sum %s/*/%s", taskDir, "data")).CombinedOutput()
+		out, err := pod.Command("sh", "-c", fmt.Sprintf("sha256sum %s/*/%s", taskDir, "data")).CombinedOutput()
 		fmt.Println("preheat sha256sum: " + string(out))
 		Expect(err).NotTo(HaveOccurred())
 		sha256sum = strings.Split(string(out), " ")[0]