diff --git a/Makefile b/Makefile index d6f2bc72..39f6fb6b 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,14 @@ BOSKOS_RESOURCE_TYPE ?= gke-internal-project RUN_IN_PROW ?= false -LOCATION ?= us-central1-c -NUM_NODES ?= 3 TEST_TO_RUN ?= .* +REPO_PATH ?= go/src/github.com/GoogleCloudPlatform/gke-networking-recipes +NETWORK_NAME ?= rcp-network +SUBNET_NAME ?= rcp-subnet +CLUSTER_NAME ?= rcp-cluster +ZONE ?= us-west1-a +NUM_NODES ?= 3 +DELETE_CLUSTER ?= false all: bin/recipes-test @@ -27,12 +32,16 @@ bin/recipes-test: .PHONY: test test: bin/recipes-test bin/recipes-test \ - --run-in-prow=$(RUN_IN_PROW) \ + --repo-path=$(REPO_PATH) \ --boskos-resource-type=$(BOSKOS_RESOURCE_TYPE) \ + --run-in-prow=$(RUN_IN_PROW) \ --test-project-id=$(PROJECT_ID) \ + --network-name=$(NETWORK_NAME) \ + --subnet-name=$(SUBNET_NAME) \ --cluster-name=$(CLUSTER_NAME) \ - --location=$(LOCATION) \ + --zone=$(ZONE) \ --num-nodes=$(NUM_NODES) \ + --delete-cluster=$(DELETE_CLUSTER) \ -test.run=$(TEST_TO_RUN) \ .PHONY: clean diff --git a/test/README.md b/test/README.md index a856e08e..8f601530 100644 --- a/test/README.md +++ b/test/README.md @@ -32,10 +32,9 @@ You will need to set the project to run the test locally: export PROJECT_ID= ``` -To specify the location to create the test cluster, you will need to set the environment varible, it will be `us-central1-c` by default. - +You can change the test setting by exporting the test variables in [Makefile](../Makefile). For example, to specify the zone of the cluster, you can do: ``` -export LOCATION= +export ZONE= ``` Then run to start the test or select a set of tests to run: diff --git a/test/main_test.go b/test/main_test.go index 5ba60ae3..7345d904 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -18,13 +18,13 @@ package test import ( "flag" - "fmt" "os" "os/exec" "path/filepath" "strings" "testing" + "github.com/GoogleCloudPlatform/gke-networking-recipes/test/utils" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" "k8s.io/client-go/tools/clientcmd" backendconfigclient "k8s.io/ingress-gce/pkg/backendconfig/client/clientset/versioned" @@ -36,9 +36,12 @@ import ( var ( flags struct { kubeconfig string + repoPath string testProjectID string testClusterName string - location string + testNetworkName string + testSubnetName string + zone string numOfNodes int // Test infrastructure flags. boskosResourceType string @@ -55,13 +58,16 @@ var ( func init() { flag.StringVar(&flags.kubeconfig, "kubeconfig", "", "path to the .kube config file. This will default to $HOME/.kube/config if unset.") + flag.StringVar(&flags.repoPath, "repo-path", "", "path from $HOME to repo. This will default to go/src/github.com/GoogleCloudPlatform/gke-networking-recipes if unset") flag.StringVar(&flags.boskosResourceType, "boskos-resource-type", "gke-internal-project", "name of the boskos resource type to reserve") flag.BoolVar(&flags.inProw, "run-in-prow", false, "is the test running in PROW") flag.StringVar(&flags.testProjectID, "test-project-id", "", "Project ID of the test cluster") - flag.StringVar(&flags.testClusterName, "cluster-name", "", "Name of the test cluster") - flag.StringVar(&flags.location, "location", "", "Location of the test cluster") - flag.IntVar(&flags.numOfNodes, "num-nodes", 3, "The number of nodes to be created in each of the cluster's zones") - flag.BoolVar(&flags.deleteCluster, "delete-cluster", false, "if the cluster is deleted after test runs") + flag.StringVar(&flags.testNetworkName, "network-name", "", "Name of the test network. This will default to rcp-network if unset.") + flag.StringVar(&flags.testSubnetName, "subnet-name", "", "Name of the test subnet. This will default to rcp-subnet if unset.") + flag.StringVar(&flags.testClusterName, "cluster-name", "", "Name of the test cluster. This will default to rcp-cluster if unset.") + flag.StringVar(&flags.zone, "zone", "", "Compute zone of the test cluster. This will default to us-west1-a if unset.") + flag.IntVar(&flags.numOfNodes, "num-nodes", 3, "The number of nodes to be created in each of the cluster's zones. This will default to 3 if unset.") + flag.BoolVar(&flags.deleteCluster, "delete-cluster", false, "if the cluster is deleted after test runs. This will default to false if unset.") } func TestMain(m *testing.M) { @@ -72,20 +78,9 @@ func TestMain(m *testing.M) { if home := os.Getenv("HOME"); home != "" { flags.kubeconfig = filepath.Join(home, ".kube", "config") } else { - klog.Fatalf("kubeconfig path required but not provided") + klog.Fatalf("kubeconfig path required but not provided.") } } - - if flags.testClusterName == "" { - randSuffix := randSeq(6) - flags.testClusterName = "gke-networking-recipes-" + randSuffix - } - - if flags.location == "" { - fmt.Fprintln(os.Stderr, "--location must be set to run the test") - os.Exit(1) - } - project := flags.testProjectID // If running in Prow, then acquire and set up a project through Boskos. @@ -93,7 +88,7 @@ func TestMain(m *testing.M) { project = setupProwConfig(flags.boskosResourceType) if _, ok := os.LookupEnv("USER"); !ok { if err := os.Setenv("USER", "prow"); err != nil { - klog.Fatalf("failed to set user in prow to prow: %v", err) + klog.Fatalf("Failed to set user in prow to prow: %v.", err) } } } @@ -101,43 +96,52 @@ func TestMain(m *testing.M) { output, _ := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput() oldProject := strings.TrimSpace(string(output)) klog.Infof("Using project %s for testing. Restore to existing project %s after testing.", project, oldProject) - if err := setEnvProject(project); err != nil { - klog.Fatalf("failed to set project environment to %q: %v", project, err) + klog.Fatalf("Failed to set project environment to %q: %v.", project, err) } - // After the test, reset the project + // After the test, reset the project. defer func() { if err := setEnvProject(oldProject); err != nil { klog.Errorf("failed to set project environment to %s: %v", oldProject, err) } }() - klog.Infof("setupCluster(%q, %q, %d)", flags.location, flags.testClusterName, flags.numOfNodes) - if err := setupCluster(flags.location, flags.testClusterName, flags.numOfNodes); err != nil { - klog.Fatalf("setupCluster(%q, %q, %d) = %v", flags.location, flags.testClusterName, flags.numOfNodes, err) + // Set up test network and subnets. + if err := utils.SetupTestNetwork(flags.testNetworkName); err != nil { + klog.Fatalf("Failed to set up network %q: %v", flags.testNetworkName, err) } - klog.Infof("getCredential(%q, %q)", flags.location, flags.testClusterName) - if err := getCredential(flags.location, flags.testClusterName); err != nil { - klog.Fatalf("getCredential(%q, %q) = %v", flags.location, flags.testClusterName, err) + region := flags.zone[:len(flags.zone)-2] // Drop the zone + if err := utils.SetupTestSubnet(flags.testSubnetName, flags.testNetworkName, region); err != nil { + klog.Fatalf("Failed to set up subnet %q: %v", flags.testSubnetName, err) } + + // Set up test cluster and its credential. + if err := utils.SetupCluster(flags.testClusterName, flags.zone, flags.testNetworkName, flags.numOfNodes); err != nil { + klog.Fatalf("Failed to set up cluster %q in %q: %v", flags.testClusterName, flags.zone, err) + } + if err := utils.GetCredential(flags.zone, flags.testClusterName); err != nil { + klog.Fatalf("Failed to get credential for cluster %q: %v", flags.testClusterName, err) + } + if flags.deleteCluster { defer func() { - klog.Infof("deleteCluster(%q, %q)", flags.location, flags.testClusterName) - if err := deleteCluster(flags.location, flags.testClusterName); err != nil { - klog.Errorf("deleteCluster(%q, %q) = %v", flags.location, flags.testClusterName, err) + klog.Infof("DeleteCluster(%q, %q)", flags.zone, flags.testClusterName) + if err := utils.DeleteCluster(flags.testClusterName, flags.zone); err != nil { + klog.Fatalf("Failed to create cluster %q: %v", flags.testClusterName, err) } }() } + // Set up kube config. klog.Infof("Using kubeconfig %q", flags.kubeconfig) kubeconfig, err := clientcmd.BuildConfigFromFlags("", flags.kubeconfig) if err != nil { - klog.Fatalf("Error creating kubernetes clients from %q: %v", flags.kubeconfig, err) + klog.Fatalf("Failed to create kubernetes config from %q: %v", flags.kubeconfig, err) } client, err := ctrlClient.New(kubeconfig, ctrlClient.Options{}) if err != nil { - klog.Errorf("Error when creating client: %v", err) + klog.Errorf("Failed to create kubernetes client: %v", err) } Framework.Client = client Framework.BackendConfigClient = backendconfigclient.NewForConfigOrDie(kubeconfig) @@ -145,8 +149,7 @@ func TestMain(m *testing.M) { Framework.Cloud, err = newCloud(project) if err != nil { - klog.Fatalf("Error creating compute client for project %q: %v", project, err) + klog.Fatalf("Failed to create compute client for project %q: %v", project, err) } - m.Run() } diff --git a/test/utils.go b/test/utils.go index fd83ab83..560c945a 100644 --- a/test/utils.go +++ b/test/utils.go @@ -17,14 +17,11 @@ limitations under the License. package test import ( - "bytes" "context" "encoding/json" "fmt" - "math/rand" "os" "os/exec" - "strconv" "time" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" @@ -95,95 +92,6 @@ func setEnvProject(project string) error { return os.Setenv("PROJECT", project) } -func setupCluster(location, clusterName string, numOfNodes int) error { - params := []string{ - "container", - "clusters", - "describe", - clusterName, - "--zone", location, - } - out, err := exec.Command("gcloud", params...).CombinedOutput() - if err != nil { - klog.Infof("Cluster %s does not exist, creating instead.", clusterName) - return createCluster(location, clusterName, numOfNodes) - } - - pattern := "currentNodeCount: " - startIndex := bytes.Index(out, []byte(pattern)) + len(pattern) // The index immediately after the pattern. - endIndex := startIndex + bytes.Index(out[startIndex:], []byte("\n")) // The index after the pattern and before new line. - if startIndex == -1 || endIndex == -1 { - klog.Infof("Cannot find current node count. Delete and recreate cluster.") - return deleteAndCreateCluster(location, clusterName, numOfNodes) - } - - gotNumOfNodes, err := strconv.Atoi(string(out[startIndex:endIndex])) - if err != nil || gotNumOfNodes != numOfNodes { - klog.Infof("Got cluster with %d nodes, expect %d. Delete and recreate cluster %s in %s.", gotNumOfNodes, numOfNodes, clusterName, location) - return deleteAndCreateCluster(location, clusterName, numOfNodes) - } - klog.Infof("Use existing cluster %s in zone %s with %d nodes", clusterName, location, numOfNodes) - return nil -} - -func createCluster(location, clusterName string, numOfNodes int) error { - klog.Infof("Creating cluster %s in %s, numOfNodes=%d", clusterName, location, numOfNodes) - params := []string{ - "container", - "clusters", - "create", - clusterName, - "--zone", location, - "--num-nodes", strconv.Itoa(numOfNodes), - } - if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { - return fmt.Errorf("failed creating cluster: %s, err: %v", out, err) - } - return nil -} - -func deleteCluster(location, clusterName string) error { - klog.Infof("Deleting cluster %s in %s", clusterName, location) - params := []string{ - "container", - "clusters", - "delete", - clusterName, - "--zone", - location, - "--quiet", - } - if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { - return fmt.Errorf("failed deleting cluster: %s, err: %v", out, err) - } - return nil -} - -func deleteAndCreateCluster(location, clusterName string, numOfNodes int) error { - if err := deleteCluster(location, clusterName); err != nil { - return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err) - } - if err := createCluster(location, clusterName, numOfNodes); err != nil { - return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err) - } - return nil -} - -func getCredential(location, clusterName string) error { - params := []string{ - "container", - "clusters", - "get-credentials", - clusterName, - "--zone", - location, - } - if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { - return fmt.Errorf("failed setting kubeconfig: %s, err: %v", out, err) - } - return nil -} - func newCloud(project string) (cloud.Cloud, error) { ctx := context.Background() client, err := google.DefaultClient(ctx, compute.ComputeScope) @@ -216,15 +124,6 @@ func newCloud(project string) (cloud.Cloud, error) { return theCloud, nil } -func randSeq(n int) string { - letterBytes := "0123456789abcdef" - b := make([]byte, n) - for i := range b { - b[i] = letterBytes[rand.Intn(len(letterBytes))] - } - return string(b) -} - func sendTrafficToIngress(address, instanceName, zone string, curlArgs []string) (map[string]string, error) { klog.Infof("SSH into instance %s in zone %s, and send traffic to address %s", instanceName, zone, address) params := []string{ diff --git a/test/utils/cluster.go b/test/utils/cluster.go index 736696c2..0d75890a 100644 --- a/test/utils/cluster.go +++ b/test/utils/cluster.go @@ -39,6 +39,7 @@ func SetupCluster(clusterName, zone, networkName string, numOfNodes int) error { return createCluster(clusterName, zone, networkName, numOfNodes) } + // Check if the existing cluster has the desired number of nodes. pattern := "currentNodeCount: " val, err := checkField([]byte(pattern), out) if err != nil {