Merge pull request #153 from sawsa307/refactor-cleanup

Create test Framework.
GoogleCloudPlatform · Aug 24, 2023 · 6e6ed8f · 6e6ed8f
2 parents 5980adf + 9788365
commit 6e6ed8f
Show file tree

Hide file tree

Showing 6 changed files with 296 additions and 175 deletions.
diff --git a/Makefile b/Makefile
@@ -15,7 +15,7 @@
 PROJECT_ID ?= $(shell gcloud config get-value project 2>&1 | head -n 1)
 BOSKOS_RESOURCE_TYPE ?= gke-internal-project
 RUN_IN_PROW ?= false
-LOCATION ?= us-central1-c
+ZONE ?= us-central1-c
 NUM_NODES ?= 3
 TEST_TO_RUN ?= .*
 JOB_NAME ?= gke-networking-recipe-e2e
@@ -33,7 +33,7 @@ test: bin/recipes-test
 		--boskos-resource-type=$(BOSKOS_RESOURCE_TYPE) \
 		--test-project-id=$(PROJECT_ID) \
 		--cluster-name=$(CLUSTER_NAME) \
-		--location=$(LOCATION) \
+		--zone=$(ZONE) \
 		--num-nodes=$(NUM_NODES) \
 		-test.run=$(TEST_TO_RUN) \
 

diff --git a/test/main_test.go b/test/main_test.go
@@ -24,33 +24,23 @@ import (
 	"testing"
 
 	"github.com/GoogleCloudPlatform/gke-networking-recipes/test/utils"
-	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
 	"k8s.io/client-go/tools/clientcmd"
-	backendconfigclient "k8s.io/ingress-gce/pkg/backendconfig/client/clientset/versioned"
-	frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned"
 	"k8s.io/klog/v2"
-	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
-	ctrlLog "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 var (
 	flags struct {
 		kubeconfig      string
 		testProjectID   string
 		testClusterName string
-		location        string
+		zone            string
 		numOfNodes      int
 		// Test infrastructure flags.
 		boskosResourceType string
 		inProw             bool
 		deleteCluster      bool
 	}
-	Framework struct {
-		Client               ctrlClient.Client
-		BackendConfigClient  *backendconfigclient.Clientset
-		FrontendConfigClient *frontendconfigclient.Clientset
-		Cloud                cloud.Cloud
-	}
+	framework *utils.Framework
 )
 
 func init() {
@@ -59,7 +49,7 @@ func init() {
 	flag.BoolVar(&flags.inProw, "run-in-prow", false, "is the test running in PROW")
 	flag.StringVar(&flags.testProjectID, "test-project-id", "", "Project ID of the test cluster")
 	flag.StringVar(&flags.testClusterName, "cluster-name", "", "Name of the test cluster")
-	flag.StringVar(&flags.location, "location", "", "Location of the test cluster")
+	flag.StringVar(&flags.zone, "zone", "", "Zone of the test cluster")
 	flag.IntVar(&flags.numOfNodes, "num-nodes", 3, "The number of nodes to be created in each of the cluster's zones")
 	flag.BoolVar(&flags.deleteCluster, "delete-cluster", false, "if the cluster is deleted after test runs")
 }
@@ -81,8 +71,8 @@ func TestMain(m *testing.M) {
 		flags.testClusterName = "gke-networking-recipes-" + randSuffix
 	}
 
-	if flags.location == "" {
-		fmt.Fprintln(os.Stderr, "--location must be set to run the test")
+	if flags.zone == "" {
+		fmt.Fprintln(os.Stderr, "--zone must be set to run the test")
 		os.Exit(1)
 	}
 
@@ -100,61 +90,62 @@ func TestMain(m *testing.M) {
 
 		if _, ok := os.LookupEnv("USER"); !ok {
 			if err := os.Setenv("USER", "prow"); err != nil {
-				klog.Fatalf("failed to set user in prow to prow: %v", err)
+				klog.Fatalf("failed to set user in prow to prow: %v, want nil", err)
 			}
 		}
 	}
 
-	output, _ := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput()
+	output, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput()
+	if err != nil {
+		klog.Fatalf("failed to get gcloud project: %q: %v, want nil", string(output), err)
+	}
 	oldProject := strings.TrimSpace(string(output))
 	klog.Infof("Using project %s for testing. Restore to existing project %s after testing.", project, oldProject)
 
 	if err := setEnvProject(project); err != nil {
-		klog.Fatalf("failed to set project environment to %q: %v", project, err)
+		klog.Fatalf("setEnvProject(%q) failed: %v, want nil", project, err)
 	}
 
 	// After the test, reset the project
 	defer func() {
 		if err := setEnvProject(oldProject); err != nil {
-			klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
+			klog.Errorf("setEnvProject(%q) failed: %v, want nil", oldProject, err)
 		}
 	}()
 
-	klog.Infof("setupCluster(%q, %q, %d)", flags.location, flags.testClusterName, flags.numOfNodes)
-	if err := setupCluster(flags.location, flags.testClusterName, flags.numOfNodes); err != nil {
-		klog.Fatalf("setupCluster(%q, %q, %d) = %v", flags.location, flags.testClusterName, flags.numOfNodes, err)
+	clusterConfig := utils.ClusterConfig{
+		Name:       flags.testClusterName,
+		Zone:       flags.zone,
+		NumOfNodes: flags.numOfNodes,
+	}
+	klog.Infof("EnsureCluster(%+v)", clusterConfig)
+	err = utils.EnsureCluster(clusterConfig)
+	if err != nil {
+		klog.Fatalf("EnsureCluster(%+v) = %v, want nil", clusterConfig, err)
 	}
-	klog.Infof("getCredential(%q, %q)", flags.location, flags.testClusterName)
-	if err := getCredential(flags.location, flags.testClusterName); err != nil {
-		klog.Fatalf("getCredential(%q, %q) = %v", flags.location, flags.testClusterName, err)
+	klog.Infof("GetCredentials(%+v)", clusterConfig)
+	if err := utils.GetCredentials(clusterConfig); err != nil {
+		klog.Fatalf("GetCredentials(%+v) = %v, want nil", clusterConfig, err)
 	}
 	if flags.deleteCluster {
 		defer func() {
-			klog.Infof("deleteCluster(%q, %q)", flags.location, flags.testClusterName)
-			if err := deleteCluster(flags.location, flags.testClusterName); err != nil {
-				klog.Errorf("deleteCluster(%q, %q) = %v", flags.location, flags.testClusterName, err)
+			klog.Infof("DeleteCluster(%+v)", clusterConfig)
+			if err := utils.DeleteCluster(clusterConfig); err != nil {
+				klog.Errorf("DeleteCluster(%+v) = %v, want nil", clusterConfig, err)
 			}
 		}()
 	}
 
 	klog.Infof("Using kubeconfig %q", flags.kubeconfig)
 	kubeconfig, err := clientcmd.BuildConfigFromFlags("", flags.kubeconfig)
 	if err != nil {
-		klog.Fatalf("Error creating kubernetes clients from %q: %v", flags.kubeconfig, err)
+		klog.Fatalf("BuildConfigFromFlags(%q) = %v, want nil", flags.kubeconfig, err)
 	}
-	ctrlLog.SetLogger(klog.NewKlogr())
-	client, err := ctrlClient.New(kubeconfig, ctrlClient.Options{})
-	if err != nil {
-		klog.Errorf("Failed to create kubernetes client: %v", err)
-	}
-	Framework.Client = client
-	Framework.BackendConfigClient = backendconfigclient.NewForConfigOrDie(kubeconfig)
-	Framework.FrontendConfigClient = frontendconfigclient.NewForConfigOrDie(kubeconfig)
 
-	Framework.Cloud, err = newCloud(project)
-	if err != nil {
-		klog.Fatalf("Error creating compute client for project %q: %v", project, err)
-	}
+	framework = utils.NewFramework(kubeconfig, utils.Options{
+		Project: flags.testProjectID,
+		Zone:    flags.zone,
+	})
 
 	m.Run()
 }

diff --git a/test/utils.go b/test/utils.go
@@ -15,151 +15,20 @@
 package test
 
 import (
-	"bytes"
-	"context"
 	"fmt"
 	"math/rand"
 	"os"
 	"os/exec"
-	"strconv"
-
-	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
-	"golang.org/x/oauth2/google"
-	alpha "google.golang.org/api/compute/v0.alpha"
-	beta "google.golang.org/api/compute/v0.beta"
-	compute "google.golang.org/api/compute/v1"
-	"k8s.io/klog/v2"
 )
 
 func setEnvProject(project string) error {
 	if out, err := exec.Command("gcloud", "config", "set", "project", project).CombinedOutput(); err != nil {
-		return fmt.Errorf("failed to set gcloud project to %s: %s, err: %w", project, out, err)
+		return fmt.Errorf("setEnvProject(%q) failed: %q: %w", project, out, err)
 	}
 
 	return os.Setenv("PROJECT", project)
 }
 
-func setupCluster(location, clusterName string, numOfNodes int) error {
-	params := []string{
-		"container",
-		"clusters",
-		"describe",
-		clusterName,
-		"--zone", location,
-	}
-	out, err := exec.Command("gcloud", params...).CombinedOutput()
-	if err != nil {
-		klog.Infof("Cluster %s does not exist, creating instead.", clusterName)
-		return createCluster(location, clusterName, numOfNodes)
-	}
-
-	pattern := "currentNodeCount: "
-	startIndex := bytes.Index(out, []byte(pattern)) + len(pattern)       // The index immediately after the pattern.
-	endIndex := startIndex + bytes.Index(out[startIndex:], []byte("\n")) // The index after the pattern and before new line.
-	if startIndex == -1 || endIndex == -1 {
-		klog.Infof("Cannot find current node count. Delete and recreate cluster.")
-		return deleteAndCreateCluster(location, clusterName, numOfNodes)
-	}
-
-	gotNumOfNodes, err := strconv.Atoi(string(out[startIndex:endIndex]))
-	if err != nil || gotNumOfNodes != numOfNodes {
-		klog.Infof("Got cluster with %d nodes, expect %d. Delete and recreate cluster %s in %s.", gotNumOfNodes, numOfNodes, clusterName, location)
-		return deleteAndCreateCluster(location, clusterName, numOfNodes)
-	}
-	klog.Infof("Use existing cluster %s in zone %s with %d nodes", clusterName, location, numOfNodes)
-	return nil
-}
-
-func createCluster(location, clusterName string, numOfNodes int) error {
-	klog.Infof("Creating cluster %s in %s, numOfNodes=%d", clusterName, location, numOfNodes)
-	params := []string{
-		"container",
-		"clusters",
-		"create",
-		clusterName,
-		"--zone", location,
-		"--num-nodes", strconv.Itoa(numOfNodes),
-	}
-	if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
-		return fmt.Errorf("failed creating cluster: %s, err: %v", out, err)
-	}
-	return nil
-}
-
-func deleteCluster(location, clusterName string) error {
-	klog.Infof("Deleting cluster %s in %s", clusterName, location)
-	params := []string{
-		"container",
-		"clusters",
-		"delete",
-		clusterName,
-		"--zone",
-		location,
-		"--quiet",
-	}
-	if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
-		return fmt.Errorf("failed deleting cluster: %s, err: %v", out, err)
-	}
-	return nil
-}
-
-func deleteAndCreateCluster(location, clusterName string, numOfNodes int) error {
-	if err := deleteCluster(location, clusterName); err != nil {
-		return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err)
-	}
-	if err := createCluster(location, clusterName, numOfNodes); err != nil {
-		return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err)
-	}
-	return nil
-}
-
-func getCredential(location, clusterName string) error {
-	params := []string{
-		"container",
-		"clusters",
-		"get-credentials",
-		clusterName,
-		"--zone",
-		location,
-	}
-	if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
-		return fmt.Errorf("failed setting kubeconfig: %s, err: %v", out, err)
-	}
-	return nil
-}
-
-func newCloud(project string) (cloud.Cloud, error) {
-	ctx := context.Background()
-	client, err := google.DefaultClient(ctx, compute.ComputeScope)
-	if err != nil {
-		return nil, err
-	}
-
-	alpha, err := alpha.New(client)
-	if err != nil {
-		return nil, err
-	}
-	beta, err := beta.New(client)
-	if err != nil {
-		return nil, err
-	}
-	ga, err := compute.New(client)
-	if err != nil {
-		return nil, err
-	}
-
-	svc := &cloud.Service{
-		GA:            ga,
-		Alpha:         alpha,
-		Beta:          beta,
-		ProjectRouter: &cloud.SingleProjectRouter{ID: project},
-		RateLimiter:   &cloud.NopRateLimiter{},
-	}
-
-	theCloud := cloud.NewGCE(svc)
-	return theCloud, nil
-}
-
 func randSeq(n int) string {
 	letterBytes := "0123456789abcdef"
 	b := make([]byte, n)