Skip to content

Commit

Permalink
Additional test config and setups in main_test.go.
Browse files Browse the repository at this point in the history 
Add options to specify network, subnet name and repo path. In
main_test.go, also set up network and subnets.
  • Loading branch information
@sawsa307
sawsa307 committed Aug 8, 2023
1 parent 8d16629 commit e365740
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 130 deletions.
20 changes: 15 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.

PROJECT_ID ?= $(shell gcloud config get-value project 2>&1 | head -n 1)
BOSKOS_RESOURCE_TYPE ?= gke-internal-project
RUN_IN_PROW ?= false
LOCATION ?= us-central1-c
NUM_NODES ?= 3
TEST_TO_RUN ?= .*
PKG_DIR ?= src/github.com/GoogleCloudPlatform/gke-networking-recipes
NETWORK_NAME ?= rcp-network
SUBNET_NAME ?= rcp-subnet
CLUSTER_NAME ?= rcp-cluster
ZONE ?= us-west1-a
NUM_NODES ?= 3
DELETE_CLUSTER ?= false

all: bin/recipes-test

Expand All @@ -27,14 +33,18 @@ bin/recipes-test:
.PHONY: test
test: bin/recipes-test
bin/recipes-test \
--run-in-prow=$(RUN_IN_PROW) \
--pkg-dir=$(PKG_DIR) \
--boskos-resource-type=$(BOSKOS_RESOURCE_TYPE) \
--run-in-prow=$(RUN_IN_PROW) \
--test-project-id=$(PROJECT_ID) \
--network-name=$(NETWORK_NAME) \
--subnet-name=$(SUBNET_NAME) \
--cluster-name=$(CLUSTER_NAME) \
--location=$(LOCATION) \
--zone=$(ZONE) \
--num-nodes=$(NUM_NODES) \
--delete-cluster=$(DELETE_CLUSTER) \
-test.run=$(TEST_TO_RUN) \

.PHONY: clean
clean:
rm -rf bin/
rm -rf bin/
71 changes: 37 additions & 34 deletions test/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ package test

import (
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"

"github.com/GoogleCloudPlatform/gke-networking-recipes/test/utils"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"k8s.io/client-go/tools/clientcmd"
backendconfigclient "k8s.io/ingress-gce/pkg/backendconfig/client/clientset/versioned"
Expand All @@ -36,9 +36,12 @@ import (
var (
flags struct {
kubeconfig string
pkgDir string
testProjectID string
testClusterName string
location string
testNetworkName string
testSubnetName string
zone string
numOfNodes int
// Test infrastructure flags.
boskosResourceType string
Expand All @@ -55,13 +58,16 @@ var (

func init() {
flag.StringVar(&flags.kubeconfig, "kubeconfig", "", "path to the .kube config file. This will default to $HOME/.kube/config if unset.")
flag.StringVar(&flags.pkgDir, "pkg-dir", "", "path from $GOPATH to repo. This will default to src/github.com/GoogleCloudPlatform/gke-networking-recipes if unset")
flag.StringVar(&flags.boskosResourceType, "boskos-resource-type", "gke-internal-project", "name of the boskos resource type to reserve")
flag.BoolVar(&flags.inProw, "run-in-prow", false, "is the test running in PROW")
flag.StringVar(&flags.testProjectID, "test-project-id", "", "Project ID of the test cluster")
flag.StringVar(&flags.testClusterName, "cluster-name", "", "Name of the test cluster")
flag.StringVar(&flags.location, "location", "", "Location of the test cluster")
flag.IntVar(&flags.numOfNodes, "num-nodes", 3, "The number of nodes to be created in each of the cluster's zones")
flag.BoolVar(&flags.deleteCluster, "delete-cluster", false, "if the cluster is deleted after test runs")
flag.StringVar(&flags.testNetworkName, "network-name", "", "Name of the test network. This will default to rcp-network if unset.")
flag.StringVar(&flags.testSubnetName, "subnet-name", "", "Name of the test subnet. This will default to rcp-subnet if unset.")
flag.StringVar(&flags.testClusterName, "cluster-name", "", "Name of the test cluster. This will default to rcp-cluster if unset.")
flag.StringVar(&flags.zone, "zone", "", "Compute zone of the test cluster. This will default to us-west1-a if unset.")
flag.IntVar(&flags.numOfNodes, "num-nodes", 3, "The number of nodes to be created in each of the cluster's zones. This will default to 3 if unset.")
flag.BoolVar(&flags.deleteCluster, "delete-cluster", false, "if the cluster is deleted after test runs. This will default to false if unset.")
}

func TestMain(m *testing.M) {
Expand All @@ -72,68 +78,66 @@ func TestMain(m *testing.M) {
if home := os.Getenv("HOME"); home != "" {
flags.kubeconfig = filepath.Join(home, ".kube", "config")
} else {
klog.Fatalf("kubeconfig path required but not provided")
klog.Fatalf("kubeconfig path required but not provided.")
}
}

if flags.testClusterName == "" {
randSuffix := randSeq(6)
flags.testClusterName = "gke-networking-recipes-" + randSuffix
}

if flags.location == "" {
fmt.Fprintln(os.Stderr, "--location must be set to run the test")
os.Exit(1)
}

project := flags.testProjectID

// If running in Prow, then acquire and set up a project through Boskos.
if flags.inProw {
project = setupProwConfig(flags.boskosResourceType)
if _, ok := os.LookupEnv("USER"); !ok {
if err := os.Setenv("USER", "prow"); err != nil {
klog.Fatalf("failed to set user in prow to prow: %v", err)
klog.Fatalf("Failed to set user in prow to prow: %v.", err)
}
}
}

output, _ := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput()
oldProject := strings.TrimSpace(string(output))
klog.Infof("Using project %s for testing. Restore to existing project %s after testing.", project, oldProject)

if err := setEnvProject(project); err != nil {
klog.Fatalf("failed to set project environment to %q: %v", project, err)
klog.Fatalf("Failed to set project environment to %q: %v.", project, err)
}

// After the test, reset the project
// After the test, reset the project.
defer func() {
if err := setEnvProject(oldProject); err != nil {
klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
}
}()

klog.Infof("setupCluster(%q, %q, %d)", flags.location, flags.testClusterName, flags.numOfNodes)
if err := setupCluster(flags.location, flags.testClusterName, flags.numOfNodes); err != nil {
klog.Fatalf("setupCluster(%q, %q, %d) = %v", flags.location, flags.testClusterName, flags.numOfNodes, err)
// Set up test network and subnets.
if err := utils.SetupTestNetwork(flags.testNetworkName); err != nil {
klog.Fatalf("Failed to set up network %q: %v", flags.testNetworkName, err)
}
klog.Infof("getCredential(%q, %q)", flags.location, flags.testClusterName)
if err := getCredential(flags.location, flags.testClusterName); err != nil {
klog.Fatalf("getCredential(%q, %q) = %v", flags.location, flags.testClusterName, err)
region := flags.zone[:len(flags.zone)-2] // Drop the zone
if err := utils.SetupTestSubnet(flags.testSubnetName, flags.testNetworkName, region); err != nil {
klog.Fatalf("Failed to set up subnet %q: %v", flags.testSubnetName, err)
}

// Set up test cluster and its credential.
if err := utils.SetupCluster(flags.testClusterName, flags.zone, flags.testNetworkName, flags.numOfNodes); err != nil {
klog.Fatalf("Failed to set up cluster %q in %q: %v", flags.testClusterName, flags.zone, err)
}
if err := utils.GetCredential(flags.zone, flags.testClusterName); err != nil {
klog.Fatalf("Failed to get credential for cluster %q: %v", flags.testClusterName, err)
}

if flags.deleteCluster {
defer func() {
klog.Infof("deleteCluster(%q, %q)", flags.location, flags.testClusterName)
if err := deleteCluster(flags.location, flags.testClusterName); err != nil {
klog.Errorf("deleteCluster(%q, %q) = %v", flags.location, flags.testClusterName, err)
klog.Infof("DeleteCluster(%q, %q)", flags.zone, flags.testClusterName)
if err := utils.DeleteCluster(flags.testClusterName, flags.zone); err != nil {
klog.Fatalf("Failed to create cluster %q: %v", flags.testClusterName, err)
}
}()
}

// Set up kube config.
klog.Infof("Using kubeconfig %q", flags.kubeconfig)
kubeconfig, err := clientcmd.BuildConfigFromFlags("", flags.kubeconfig)
if err != nil {
klog.Fatalf("Error creating kubernetes clients from %q: %v", flags.kubeconfig, err)
klog.Fatalf("Failed to create kubernetes config from %q: %v", flags.kubeconfig, err)
}
client, err := ctrlClient.New(kubeconfig, ctrlClient.Options{})
if err != nil {
Expand All @@ -145,8 +149,7 @@ func TestMain(m *testing.M) {

Framework.Cloud, err = newCloud(project)
if err != nil {
klog.Fatalf("Error creating compute client for project %q: %v", project, err)
klog.Fatalf("Failed to create compute client for project %q: %v", project, err)
}

m.Run()
}
91 changes: 0 additions & 91 deletions test/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,12 @@ limitations under the License.
package test

import (
"bytes"
"context"
"encoding/json"
"fmt"
"math/rand"
"os"
"os/exec"
"strconv"
"time"

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
Expand Down Expand Up @@ -95,95 +93,6 @@ func setEnvProject(project string) error {
return os.Setenv("PROJECT", project)
}

func setupCluster(location, clusterName string, numOfNodes int) error {
params := []string{
"container",
"clusters",
"describe",
clusterName,
"--zone", location,
}
out, err := exec.Command("gcloud", params...).CombinedOutput()
if err != nil {
klog.Infof("Cluster %s does not exist, creating instead.", clusterName)
return createCluster(location, clusterName, numOfNodes)
}

pattern := "currentNodeCount: "
startIndex := bytes.Index(out, []byte(pattern)) + len(pattern) // The index immediately after the pattern.
endIndex := startIndex + bytes.Index(out[startIndex:], []byte("\n")) // The index after the pattern and before new line.
if startIndex == -1 || endIndex == -1 {
klog.Infof("Cannot find current node count. Delete and recreate cluster.")
return deleteAndCreateCluster(location, clusterName, numOfNodes)
}

gotNumOfNodes, err := strconv.Atoi(string(out[startIndex:endIndex]))
if err != nil || gotNumOfNodes != numOfNodes {
klog.Infof("Got cluster with %d nodes, expect %d. Delete and recreate cluster %s in %s.", gotNumOfNodes, numOfNodes, clusterName, location)
return deleteAndCreateCluster(location, clusterName, numOfNodes)
}
klog.Infof("Use existing cluster %s in zone %s with %d nodes", clusterName, location, numOfNodes)
return nil
}

func createCluster(location, clusterName string, numOfNodes int) error {
klog.Infof("Creating cluster %s in %s, numOfNodes=%d", clusterName, location, numOfNodes)
params := []string{
"container",
"clusters",
"create",
clusterName,
"--zone", location,
"--num-nodes", strconv.Itoa(numOfNodes),
}
if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
return fmt.Errorf("failed creating cluster: %s, err: %v", out, err)
}
return nil
}

func deleteCluster(location, clusterName string) error {
klog.Infof("Deleting cluster %s in %s", clusterName, location)
params := []string{
"container",
"clusters",
"delete",
clusterName,
"--zone",
location,
"--quiet",
}
if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
return fmt.Errorf("failed deleting cluster: %s, err: %v", out, err)
}
return nil
}

func deleteAndCreateCluster(location, clusterName string, numOfNodes int) error {
if err := deleteCluster(location, clusterName); err != nil {
return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err)
}
if err := createCluster(location, clusterName, numOfNodes); err != nil {
return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err)
}
return nil
}

func getCredential(location, clusterName string) error {
params := []string{
"container",
"clusters",
"get-credentials",
clusterName,
"--zone",
location,
}
if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
return fmt.Errorf("failed setting kubeconfig: %s, err: %v", out, err)
}
return nil
}

func newCloud(project string) (cloud.Cloud, error) {
ctx := context.Background()
client, err := google.DefaultClient(ctx, compute.ComputeScope)
Expand Down

0 comments on commit e365740

Please sign in to comment.