Create utils functions for e2e tests.

Create util functions for setup/create gcloud resources. Add function to
check ingress IP and validate its status.

test/utils.go

@@ -19,11 +19,11 @@ package test
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"os"
 	"os/exec"
-	"regexp"
 	"strconv"
 	"time"
 
@@ -33,12 +33,10 @@ import (
 	beta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/klog/v2"
 	boskosclient "sigs.k8s.io/boskos/client"
 	"sigs.k8s.io/boskos/common"
-	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var boskos, _ = boskosclient.NewClient(os.Getenv("JOB_NAME"), "http://boskos", "", "")
@@ -218,62 +216,45 @@ func newCloud(project string) (cloud.Cloud, error) {
 	return theCloud, nil
 }
 
-// createK8sResourcesFromYaml will create all kubernetes resources that are
-// present in the given yaml file.
-func createK8sResourcesFromYaml(ctx context.Context, cli ctrlClient.Client, filePath string) (map[string][]ctrlClient.ObjectKey, error) {
-	yaml, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, err
+func randSeq(n int) string {
+	letterBytes := "0123456789abcdef"
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
 	}
+	return string(b)
+}
 
-	objects, err := parseK8sYaml(string(yaml))
-	if err != nil {
-		return nil, err
+func sendTrafficToIngress(address, instanceName, zone string, curlArgs []string) (map[string]string, error) {
+	klog.Infof("SSH into instance %s in zone %s, and send traffic to address %s", instanceName, zone, address)
+	params := []string{
+		"compute",
+		"ssh",
+		instanceName,
+		"--zone", zone,
+		"--ssh-flag=-t", // Allocate pseudo-terminal to get rid of `Pseudo-terminal will not be allocated because stdin is not a terminal.` message in response.
+		"--ssh-flag=-q", // Use quite mode to get rid of `Connection to xx.xxx.xx.xxx closed' message in reponse.
+		"--",
+		"curl", address,
 	}
-
-	resources := make(map[string][]ctrlClient.ObjectKey)
-
-	for _, obj := range objects {
-		gvk := obj.GetObjectKind().GroupVersionKind().String()
-		resources[gvk] = append(resources[gvk], ctrlClient.ObjectKeyFromObject(obj))
-		if err := cli.Create(ctx, obj); err != nil {
-			return nil, fmt.Errorf("failed to create %s in namespace %s: %v", obj.GetName(), obj.GetNamespace(), err)
+	if curlArgs != nil {
+		klog.Infof("Additional curl arguments: %v", curlArgs)
+		for _, arg := range curlArgs {
+			params = append(params, arg)
 		}
 	}
 
-	return resources, nil
-}
-
-// parseK8sYaml converts a valid K8s YAML file to a list of runtime objects
-func parseK8sYaml(yaml string) ([]ctrlClient.Object, error) {
-	sepYamlfiles := regexp.MustCompile("\n---\n").Split(yaml, -1)
-	retVal := make([]ctrlClient.Object, 0, len(sepYamlfiles))
-	for _, f := range sepYamlfiles {
-		if f == "\n" || f == "" {
-			// ignore empty cases
-			continue
-		}
-
-		decode := scheme.Codecs.UniversalDeserializer().Decode
-		runtimeObj, _, err := decode([]byte(f), nil, nil)
-		if err != nil {
-			return nil, fmt.Errorf("failed to decode YAML file: %v, ", err)
-		}
-
-		clientObj, ok := runtimeObj.(ctrlClient.Object)
-		if !ok {
-			return nil, fmt.Errorf("failed to cast runtime.Object as controllerClient.Object: %#v", runtimeObj)
-		}
-		retVal = append(retVal, clientObj)
+	command := exec.Command("gcloud", params...)
+	klog.Infof("Command is: %s", exec.Command("gcloud", params...).String())
+	out, err := command.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("fail to access ingress VIP: %s, err: %v", out, err)
 	}
-	return retVal, nil
-}
-
-func randSeq(n int) string {
-	letterBytes := "0123456789abcdef"
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	klog.Infof("Curl response body is: %s", out)
+	var jsonMap map[string]string
+	err = json.Unmarshal(out, &jsonMap)
+	if err != nil {
+		return nil, fmt.Errorf("fail to parse the response to json: %v", err)
 	}
-	return string(b)
+	return jsonMap, nil
 }

test/utils/cluster.go

@@ -0,0 +1,116 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+	"os/exec"
+	"strconv"
+
+	"k8s.io/klog/v2"
+)
+
+func SetupCluster(clusterName, zone, networkName string, numOfNodes int) error {
+	klog.Infof("Setting up cluster %s in zone %s in network %s with %d node(s).", clusterName, zone, networkName, numOfNodes)
+	params := []string{
+		"container",
+		"clusters",
+		"describe",
+		clusterName,
+		"--zone", zone,
+	}
+	out, err := exec.Command("gcloud", params...).CombinedOutput()
+	if err != nil {
+		klog.Infof("Cluster %s does not exist, creating.", clusterName)
+		return createCluster(clusterName, zone, networkName, numOfNodes)
+	}
+
+	pattern := "currentNodeCount: "
+	val, err := checkField([]byte(pattern), out)
+	if err != nil {
+		klog.Infof("Cannot get the node count within the cluster description. Delete and recreate cluster %s in %s.", clusterName, zone)
+		return deleteAndCreateCluster(clusterName, zone, networkName, numOfNodes)
+	}
+	gotNumOfNodes, err := strconv.Atoi(val)
+	if err != nil || gotNumOfNodes != numOfNodes {
+		klog.Infof("Got cluster with %d nodes, expect %d. Delete and recreate cluster %s in %s.", gotNumOfNodes, numOfNodes, clusterName, zone)
+		return deleteAndCreateCluster(clusterName, zone, networkName, numOfNodes)
+	}
+	klog.Infof("Use existing cluster %s in zone %s with %d nodes.", clusterName, zone, numOfNodes)
+	return nil
+}
+
+func createCluster(clusterName, zone, networkName string, numOfNodes int) error {
+	klog.Infof("Creating cluster %s in %s in network %s with %d node(s).", clusterName, zone, networkName, numOfNodes)
+	params := []string{
+		"container",
+		"clusters",
+		"create",
+		clusterName,
+		"--network", networkName,
+		"--subnetwork", networkName,
+		"--zone", zone,
+		"--num-nodes", strconv.Itoa(numOfNodes),
+	}
+	if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
+		return fmt.Errorf("failed creating cluster: %s, err: %v", out, err)
+	}
+	return nil
+}
+
+func DeleteCluster(clusterName, zone string) error {
+	klog.Infof("Deleting cluster %s in %s", clusterName, zone)
+	params := []string{
+		"container",
+		"clusters",
+		"delete",
+		clusterName,
+		"--zone",
+		zone,
+		"--quiet",
+	}
+	if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
+		return fmt.Errorf("failed deleting cluster: %s, err: %v", out, err)
+	}
+	return nil
+}
+
+func deleteAndCreateCluster(clusterName, zone, networkName string, numOfNodes int) error {
+	if err := DeleteCluster(clusterName, zone); err != nil {
+		return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err)
+	}
+	if err := createCluster(clusterName, zone, networkName, numOfNodes); err != nil {
+		return fmt.Errorf("failed delete and create cluster: %s, err: %v", clusterName, err)
+	}
+	return nil
+}
+
+func GetCredential(zone, clusterName string) error {
+	klog.Infof("Getting credential for cluster %s in zone %s.", clusterName, zone)
+	params := []string{
+		"container",
+		"clusters",
+		"get-credentials",
+		clusterName,
+		"--zone",
+		zone,
+	}
+	if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
+		return fmt.Errorf("failed setting kubeconfig: %s, err: %v", out, err)
+	}
+	return nil
+}

test/utils/firewall.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+	"os/exec"
+
+	"k8s.io/klog/v2"
+)
+
+func SetupAllowProxyConnectionFirewall(firewallName, networkName string) error {
+	klog.Infof("Setting up allow-proxy-connection firewall %s in network %s.", firewallName, networkName)
+	params := []string{
+		"compute",
+		"firewall-rules",
+		"describe",
+		firewallName,
+	}
+	if _, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
+		klog.Infof("allow-proxy-connection firewall %s does not exist, creating.", firewallName)
+		return createAllowProxyConnectionFirewall(firewallName, networkName)
+	}
+	klog.Infof("Use existing allow-proxy-connection firewall %s.", firewallName)
+	return nil
+}
+
+func createAllowProxyConnectionFirewall(firewallName, networkName string) error {
+	klog.Infof("Creating allow-proxy-connection firewall %s in network %s.", firewallName, networkName)
+	params := []string{
+		"compute",
+		"firewall-rules",
+		"create",
+		firewallName,
+		"--allow", "TCP:8080",
+		"--source-ranges", "10.129.0.0/23",
+		"--network", networkName,
+	}
+	if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
+		return fmt.Errorf("failed creating allow-proxy-connection firewall %s: %s, err: %v", firewallName, out, err)
+	}
+	return nil
+}
+
+func SetupAllowSSHFirewall(firewallName, networkName string) error {
+	klog.Infof("Setting up allow-ssh firewall %s in network %s.", firewallName, networkName)
+	params := []string{
+		"compute",
+		"firewall-rules",
+		"describe",
+		firewallName,
+	}
+	if _, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
+		klog.Infof("allow-ssh firewall %s does not exist, creating.", firewallName)
+		return createAllowSSHFirewall(firewallName, networkName)
+	}
+	klog.Infof("Use existing allow-ssh firewall %s.", firewallName)
+	return nil
+}
+
+func createAllowSSHFirewall(firewallName, networkName string) error {
+	klog.Infof("Creating allow-ssh firewall %s in network %s.", firewallName, networkName)
+	params := []string{
+		"compute",
+		"firewall-rules",
+		"create",
+		firewallName,
+		"--network", networkName,
+		"--action", "allow",
+		"--direction", "ingress",
+		"--target-tags", "allow-ssh",
+		"--rules", "tcp:22",
+	}
+	if out, err := exec.Command("gcloud", params...).CombinedOutput(); err != nil {
+		return fmt.Errorf("failed creating allow-ssh firewall %s: %s, err: %v", firewallName, out, err)
+	}
+	return nil
+}

test/utils/ingress.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"context"
+	"time"
+
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog/v2"
+	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	ingressPollInterval = 30 * time.Second
+	ingressPollTimeout  = 45 * time.Minute
+)
+
+func fetchIngress(ctx context.Context, cli ctrlClient.Client, key ctrlClient.ObjectKey) (*networkingv1.Ingress, error) {
+	obj := &networkingv1.Ingress{}
+	return obj, cli.Get(ctx, key, obj)
+}
+
+func WaitForIngress(ctx context.Context, cli ctrlClient.Client, ingressObjectKey ctrlClient.ObjectKey) (string, error) {
+	var ingressIP string
+	err := wait.PollUntilContextTimeout(ctx, ingressPollInterval, ingressPollTimeout, true, func(ctx context.Context) (done bool, err error) {
+		ing, err := fetchIngress(ctx, cli, ingressObjectKey)
+		if err != nil {
+			klog.Infof("fetchIngress(%s)=%v", ingressObjectKey, err)
+			return false, nil
+		}
+		ingressIPs := ing.Status.LoadBalancer.Ingress
+		if len(ingressIPs) < 1 || ingressIPs[0].IP == "" {
+			klog.Infof("Invalid ingress IP %s when fetchIngress(%s)", ingressIPs, ingressObjectKey)
+			return false, nil
+		}
+		ingressIP = ingressIPs[0].IP
+		return true, nil
+	})
+	return ingressIP, err
+}