diff --git a/Makefile b/Makefile index 2ad20bf0..9b256285 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ PROJECT_ID ?= $(shell gcloud config get-value project 2>&1 | head -n 1) BOSKOS_RESOURCE_TYPE ?= gke-internal-project RUN_IN_PROW ?= false -LOCATION ?= us-central1-c +ZONE ?= us-central1-c NUM_NODES ?= 3 TEST_TO_RUN ?= .* JOB_NAME ?= gke-networking-recipe-e2e @@ -33,7 +33,7 @@ test: bin/recipes-test --boskos-resource-type=$(BOSKOS_RESOURCE_TYPE) \ --test-project-id=$(PROJECT_ID) \ --cluster-name=$(CLUSTER_NAME) \ - --location=$(LOCATION) \ + --zone=$(ZONE) \ --num-nodes=$(NUM_NODES) \ -test.run=$(TEST_TO_RUN) \ diff --git a/test/main_test.go b/test/main_test.go index f80ac5c0..9135a04b 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -24,13 +24,8 @@ import ( "testing" "github.com/GoogleCloudPlatform/gke-networking-recipes/test/utils" - "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" "k8s.io/client-go/tools/clientcmd" - backendconfigclient "k8s.io/ingress-gce/pkg/backendconfig/client/clientset/versioned" - frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned" "k8s.io/klog/v2" - ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" - ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -38,19 +33,14 @@ var ( kubeconfig string testProjectID string testClusterName string - location string + zone string numOfNodes int // Test infrastructure flags. boskosResourceType string inProw bool deleteCluster bool } - Framework struct { - Client ctrlClient.Client - BackendConfigClient *backendconfigclient.Clientset - FrontendConfigClient *frontendconfigclient.Clientset - Cloud cloud.Cloud - } + framework *utils.Framework ) func init() { @@ -59,7 +49,7 @@ func init() { flag.BoolVar(&flags.inProw, "run-in-prow", false, "is the test running in PROW") flag.StringVar(&flags.testProjectID, "test-project-id", "", "Project ID of the test cluster") flag.StringVar(&flags.testClusterName, "cluster-name", "", "Name of the test cluster") - flag.StringVar(&flags.location, "location", "", "Location of the test cluster") + flag.StringVar(&flags.zone, "zone", "", "Zone of the test cluster") flag.IntVar(&flags.numOfNodes, "num-nodes", 3, "The number of nodes to be created in each of the cluster's zones") flag.BoolVar(&flags.deleteCluster, "delete-cluster", false, "if the cluster is deleted after test runs") } @@ -81,8 +71,8 @@ func TestMain(m *testing.M) { flags.testClusterName = "gke-networking-recipes-" + randSuffix } - if flags.location == "" { - fmt.Fprintln(os.Stderr, "--location must be set to run the test") + if flags.zone == "" { + fmt.Fprintln(os.Stderr, "--zone must be set to run the test") os.Exit(1) } @@ -100,39 +90,48 @@ func TestMain(m *testing.M) { if _, ok := os.LookupEnv("USER"); !ok { if err := os.Setenv("USER", "prow"); err != nil { - klog.Fatalf("failed to set user in prow to prow: %v", err) + klog.Fatalf("failed to set user in prow to prow: %v, want nil", err) } } } - output, _ := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput() + output, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput() + if err != nil { + klog.Fatalf("failed to get gcloud project: %q: %v, want nil", string(output), err) + } oldProject := strings.TrimSpace(string(output)) klog.Infof("Using project %s for testing. Restore to existing project %s after testing.", project, oldProject) if err := setEnvProject(project); err != nil { - klog.Fatalf("failed to set project environment to %q: %v", project, err) + klog.Fatalf("setEnvProject(%q) failed: %v, want nil", project, err) } // After the test, reset the project defer func() { if err := setEnvProject(oldProject); err != nil { - klog.Errorf("failed to set project environment to %s: %v", oldProject, err) + klog.Errorf("setEnvProject(%q) failed: %v, want nil", oldProject, err) } }() - klog.Infof("setupCluster(%q, %q, %d)", flags.location, flags.testClusterName, flags.numOfNodes) - if err := setupCluster(flags.location, flags.testClusterName, flags.numOfNodes); err != nil { - klog.Fatalf("setupCluster(%q, %q, %d) = %v", flags.location, flags.testClusterName, flags.numOfNodes, err) + clusterConfig := utils.ClusterConfig{ + Name: flags.testClusterName, + Zone: flags.zone, + NumOfNodes: flags.numOfNodes, + } + klog.Infof("EnsureCluster(%+v)", clusterConfig) + err = utils.EnsureCluster(clusterConfig) + if err != nil { + klog.Fatalf("EnsureCluster(%+v) = %v, want nil", clusterConfig, err) } - klog.Infof("getCredential(%q, %q)", flags.location, flags.testClusterName) - if err := getCredential(flags.location, flags.testClusterName); err != nil { - klog.Fatalf("getCredential(%q, %q) = %v", flags.location, flags.testClusterName, err) + klog.Infof("GetCredentials(%+v)", clusterConfig) + if err := utils.GetCredentials(clusterConfig); err != nil { + klog.Fatalf("GetCredentials(%+v) = %v, want nil", clusterConfig, err) } if flags.deleteCluster { defer func() { - klog.Infof("deleteCluster(%q, %q)", flags.location, flags.testClusterName) - if err := deleteCluster(flags.location, flags.testClusterName); err != nil { - klog.Errorf("deleteCluster(%q, %q) = %v", flags.location, flags.testClusterName, err) + klog.Infof("DeleteCluster(%+v)", clusterConfig) + if err := utils.DeleteCluster(clusterConfig); err != nil { + klog.Errorf("DeleteCluster(%+v) = %v, want nil", clusterConfig, err) } }() } @@ -140,21 +139,13 @@ func TestMain(m *testing.M) { klog.Infof("Using kubeconfig %q", flags.kubeconfig) kubeconfig, err := clientcmd.BuildConfigFromFlags("", flags.kubeconfig) if err != nil { - klog.Fatalf("Error creating kubernetes clients from %q: %v", flags.kubeconfig, err) + klog.Fatalf("BuildConfigFromFlags(%q) = %v, want nil", flags.kubeconfig, err) } - ctrlLog.SetLogger(klog.NewKlogr()) - client, err := ctrlClient.New(kubeconfig, ctrlClient.Options{}) - if err != nil { - klog.Errorf("Failed to create kubernetes client: %v", err) - } - Framework.Client = client - Framework.BackendConfigClient = backendconfigclient.NewForConfigOrDie(kubeconfig) - Framework.FrontendConfigClient = frontendconfigclient.NewForConfigOrDie(kubeconfig) - Framework.Cloud, err = newCloud(project) - if err != nil { - klog.Fatalf("Error creating compute client for project %q: %v", project, err) - } + framework = utils.NewFramework(kubeconfig, utils.Options{ + Project: flags.testProjectID, + Zone: flags.zone, + }) m.Run() } diff --git a/test/utils.go b/test/utils.go index 6dc83233..807162eb 100644 --- a/test/utils.go +++ b/test/utils.go @@ -15,151 +15,20 @@ package test import ( - "bytes" - "context" "fmt" "math/rand" "os" "os/exec" - "strconv" - - "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" - "golang.org/x/oauth2/google" - alpha "google.golang.org/api/compute/v0.alpha" - beta "google.golang.org/api/compute/v0.beta" - compute "google.golang.org/api/compute/v1" - "k8s.io/klog/v2" ) func setEnvProject(project string) error { if out, err := exec.Command("gcloud", "config", "set", "project", project).CombinedOutput(); err != nil { - return fmt.Errorf("failed to set gcloud project to %s: %s, err: %w", project, out, err) + return fmt.Errorf("setEnvProject(%q) failed: %q: %w", project, out, err) } return os.Setenv("PROJECT", project) } -func setupCluster(location, clusterName string, numOfNodes int) error { - params := []string{ - "container", - "clusters", - "describe", - clusterName, - "--zone", location, - } - out, err := exec.Command("gcloud", params...).CombinedOutput() - if err != nil { - klog.Infof("Cluster %s does not exist, creating instead.", clusterName) - return createCluster(location, clusterName, numOfNodes) - } - - pattern := "currentNodeCount: " - startIndex := bytes.Index(out, []byte(pattern)) + len(pattern) // The index immediately after the pattern. - endIndex := startIndex + bytes.Index(out[startIndex:], []byte("\n")) // The index after the pattern and before new line. - if startIndex == -1 || endIndex == -1 { - klog.Infof("Cannot find current node count. Delete and recreate cluster.") - return deleteAndCreateCluster(location, clusterName, numOfNodes) - } - - gotNumOfNodes, err := strconv.Atoi(string(out[startIndex:endIndex])) - if err != nil || gotNumOfNodes != numOfNodes { - klog.Infof("Got cluster with %d nodes, expect %d. Delete and recreate cluster %s in %s.", gotNumOfNodes, numOfNodes, clusterName, location) - return deleteAndCreateCluster(location, clusterName, numOfNodes) - } - klog.Infof("Use existing cluster %s in zone %s with %d nodes", clusterName, location, numOfNodes) - return nil -} - -func createCluster(location, clusterName string, numOfNodes int) error { - klog.Infof("Creating cluster %s in %s, numOfNodes=%d", clusterName, location, numOfNodes) - params := []string{ - "container", - "clusters", - "create", - clusterName, - "--zone", location, - "--num-nodes", strconv.Itoa(numOfNodes), - } - if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { - return fmt.Errorf("failed creating cluster: %s, err: %v", out, err) - } - return nil -} - -func deleteCluster(location, clusterName string) error { - klog.Infof("Deleting cluster %s in %s", clusterName, location) - params := []string{ - "container", - "clusters", - "delete", - clusterName, - "--zone", - location, - "--quiet", - } - if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { - return fmt.Errorf("failed deleting cluster: %s, err: %v", out, err) - } - return nil -} - -func deleteAndCreateCluster(location, clusterName string, numOfNodes int) error { - if err := deleteCluster(location, clusterName); err != nil { - return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err) - } - if err := createCluster(location, clusterName, numOfNodes); err != nil { - return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err) - } - return nil -} - -func getCredential(location, clusterName string) error { - params := []string{ - "container", - "clusters", - "get-credentials", - clusterName, - "--zone", - location, - } - if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { - return fmt.Errorf("failed setting kubeconfig: %s, err: %v", out, err) - } - return nil -} - -func newCloud(project string) (cloud.Cloud, error) { - ctx := context.Background() - client, err := google.DefaultClient(ctx, compute.ComputeScope) - if err != nil { - return nil, err - } - - alpha, err := alpha.New(client) - if err != nil { - return nil, err - } - beta, err := beta.New(client) - if err != nil { - return nil, err - } - ga, err := compute.New(client) - if err != nil { - return nil, err - } - - svc := &cloud.Service{ - GA: ga, - Alpha: alpha, - Beta: beta, - ProjectRouter: &cloud.SingleProjectRouter{ID: project}, - RateLimiter: &cloud.NopRateLimiter{}, - } - - theCloud := cloud.NewGCE(svc) - return theCloud, nil -} - func randSeq(n int) string { letterBytes := "0123456789abcdef" b := make([]byte, n) diff --git a/test/utils/cluster.go b/test/utils/cluster.go new file mode 100644 index 00000000..f954a4c8 --- /dev/null +++ b/test/utils/cluster.go @@ -0,0 +1,115 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "os/exec" + "strconv" + "strings" + + "k8s.io/klog/v2" +) + +type ClusterConfig struct { + Name string + Zone string + NumOfNodes int +} + +func EnsureCluster(config ClusterConfig) error { + if !isClusterExisting(config) { + klog.Infof("Cluster %s in zone %s does not exist, creating.", config.Name, config.Zone) + return createCluster(config) + } + if err := verifyCluster(config); err != nil { + return fmt.Errorf("verifyCluster(%q, %q) failed: %w", config.Name, config.Zone, err) + } + klog.Infof("Use existing cluster %s in zone %s with %d nodes.", config.Name, config.Zone, config.NumOfNodes) + return nil +} + +func createCluster(config ClusterConfig) error { + params := []string{ + "container", + "clusters", + "create", + config.Name, + "--zone", config.Zone, + "--num-nodes", strconv.Itoa(config.NumOfNodes), + } + if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { + return fmt.Errorf("createCluster(%q, %q, %d) failed: %q: %w", config.Name, config.Zone, config.NumOfNodes, out, err) + } + return nil +} + +func DeleteCluster(config ClusterConfig) error { + params := []string{ + "container", + "clusters", + "delete", + config.Name, + "--zone", config.Zone, + "--quiet", + } + if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { + return fmt.Errorf("DeleteCluster(%q, %q) failed: %q: %w", config.Name, config.Zone, out, err) + } + return nil +} + +// GetCredentials will updates the user's cluster credentials that are saved +// in their ~/.kube/config directory and switch to the provided cluster. +func GetCredentials(config ClusterConfig) error { + params := []string{ + "container", + "clusters", + "get-credentials", + config.Name, + "--zone", config.Zone, + } + if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil { + return fmt.Errorf("GetCredentials(%q, %q) failed: %q: %w", config.Name, config.Zone, out, err) + } + return nil +} + +// isClusterExisting checks if the given cluster exists in the given zone. +func isClusterExisting(config ClusterConfig) bool { + clusters := listClusters(config.Zone) + for _, c := range clusters { + if c == config.Name { + return true + } + } + return false +} + +// listClusters lists cluster names in the given zone with Value format. +func listClusters(zone string) []string { + params := []string{ + "container", + "clusters", + "list", + "--zone", zone, + "--format", "value(NAME)", + } + out, err := exec.Command("gcloud", params...).CombinedOutput() + if err != nil { + klog.Fatalf("listClusters(%q) failed: %q: %v", zone, out, err) + } + return strings.Split(string(out), "\n") +} diff --git a/test/utils/framework.go b/test/utils/framework.go new file mode 100644 index 00000000..6ffe717a --- /dev/null +++ b/test/utils/framework.go @@ -0,0 +1,58 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" + "k8s.io/client-go/rest" + backendconfigclient "k8s.io/ingress-gce/pkg/backendconfig/client/clientset/versioned" + frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned" + "k8s.io/klog/v2" + ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" + ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Options for the test framework. +type Options struct { + Project string + Zone string +} + +type Framework struct { + Client ctrlClient.Client + BackendConfigClient *backendconfigclient.Clientset + FrontendConfigClient *frontendconfigclient.Clientset + Cloud cloud.Cloud + Zone string +} + +func NewFramework(config *rest.Config, options Options) *Framework { + ctrlLog.SetLogger(klog.NewKlogr()) + client, err := ctrlClient.New(config, ctrlClient.Options{}) + if err != nil { + klog.Fatalf("Failed to create kubernetes client: %v", err) + } + cloud, err := newCloud(options.Project) + if err != nil { + klog.Fatalf("Error creating compute client for project %q: %v", options.Project, err) + } + return &Framework{ + Client: client, + FrontendConfigClient: frontendconfigclient.NewForConfigOrDie(config), + BackendConfigClient: backendconfigclient.NewForConfigOrDie(config), + Zone: options.Zone, + Cloud: cloud, + } +} diff --git a/test/utils/utils.go b/test/utils/utils.go new file mode 100644 index 00000000..179315e8 --- /dev/null +++ b/test/utils/utils.go @@ -0,0 +1,88 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "fmt" + "os/exec" + "strconv" + "strings" + + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" + "golang.org/x/oauth2/google" + alpha "google.golang.org/api/compute/v0.alpha" + beta "google.golang.org/api/compute/v0.beta" + "google.golang.org/api/compute/v1" +) + +func newCloud(project string) (cloud.Cloud, error) { + ctx := context.Background() + client, err := google.DefaultClient(ctx, compute.ComputeScope) + if err != nil { + return nil, err + } + + alpha, err := alpha.New(client) + if err != nil { + return nil, err + } + beta, err := beta.New(client) + if err != nil { + return nil, err + } + ga, err := compute.New(client) + if err != nil { + return nil, err + } + + svc := &cloud.Service{ + GA: ga, + Alpha: alpha, + Beta: beta, + ProjectRouter: &cloud.SingleProjectRouter{ID: project}, + RateLimiter: &cloud.NopRateLimiter{}, + } + + theCloud := cloud.NewGCE(svc) + return theCloud, nil +} + +// verifyCluster checks if the cluster description has the expected configuration. +func verifyCluster(config ClusterConfig) error { + // Verify if the cluster has the correct currentNodeCount. + params := []string{ + "container", + "clusters", + "describe", + config.Name, + "--zone", config.Zone, + "--format", "value(currentNodeCount)", + } + out, err := exec.Command("gcloud", params...).CombinedOutput() + if err != nil { + return fmt.Errorf("cannot describe cluster %q using gcloud: %w", config.Name, err) + } + + outString := strings.TrimSpace(string(out)) + gotNumOfNodes, err := strconv.Atoi(outString) + if err != nil { + return fmt.Errorf("failed to convert currentNodeCount %q to int: %w", outString, err) + } + if gotNumOfNodes != config.NumOfNodes { + return fmt.Errorf("expect cluster %s to have %d nodes, got %d", config.Name, config.NumOfNodes, gotNumOfNodes) + } + return nil +}