Skip to content

Commit

Permalink
Improve validation suite
Browse files Browse the repository at this point in the history 
Simplify the tests by moving the validation code into the util package.

util.ValidateRamenHubOperator() and util.ValidateDRClusterOperator()
returns now and error if ramen operator is not in valid state, and nil
if the operator is in good state. They also log information about the
pod.

Separate looking up the ramen operator pod, and checking for its status.
This way when the pod is not running we return an error with more
context (e.g pod name, pod phase).

Tests are using sub tests to have one test per cluster, which makes test
output more clear.

We use the virtual cluster names ("c1", "c2"). This can be improved
later by getting the cluster names from the dr policy, the clusterset,
or the environment file.

Example run - ramen not deployed:

    % go test -run TestSuites/Validate --config ../e2e.yaml
    2024-10-29T19:32:09.522+0200	INFO	e2e/main_test.go:41	Global setting	{"Timeout": 600}
    2024-10-29T19:32:09.523+0200	INFO	e2e/main_test.go:42	Global setting	{"Retry Interval": 30}
    2024-10-29T19:32:09.523+0200	INFO	e2e/main_test.go:57	TestSuites
    --- FAIL: TestSuites (0.06s)
        --- FAIL: TestSuites/Validate (0.06s)
            --- FAIL: TestSuites/Validate/hub (0.03s)
                validation_suite_test.go:17: no pod with labelSelctor "app=ramen-hub" and identifier "ramen-hub-operator" in namespace "ramen-system"
            --- FAIL: TestSuites/Validate/c1 (0.02s)
                validation_suite_test.go:23: no pod with labelSelctor "app=ramen-dr-cluster" and identifier "ramen-dr-cluster-operator" in namespace "ramen-system"
            --- FAIL: TestSuites/Validate/c2 (0.02s)
                validation_suite_test.go:29: no pod with labelSelctor "app=ramen-dr-cluster" and identifier "ramen-dr-cluster-operator" in namespace "ramen-system"
        main_test.go:60: failed to validate the test suite
    FAIL
    exit status 1
    FAIL	github.com/ramendr/ramen/e2e	0.683s

Example run - ramen deployed:

    % go test -run TestSuites/Validate --config ../e2e.yaml
    2024-10-29T19:43:51.899+0200	INFO	e2e/main_test.go:41	Global setting	{"Timeout": 600}
    2024-10-29T19:43:51.900+0200	INFO	e2e/main_test.go:42	Global setting	{"Retry Interval": 30}
    2024-10-29T19:43:51.901+0200	INFO	e2e/main_test.go:57	TestSuites
    2024-10-29T19:43:51.932+0200	INFO	util/validation.go:35	Ramen hub operator is running	{"pod": "ramen-hub-operator-659658dfcf-c2xnj"}
    2024-10-29T19:43:51.951+0200	INFO	util/validation.go:59	Ramen dr cluster operator is running	{"cluster": "c1", "pod": "ramen-dr-cluster-operator-74f89d85d7-nwx54"}
    2024-10-29T19:43:51.970+0200	INFO	util/validation.go:59	Ramen dr cluster operator is running	{"cluster": "c2", "pod": "ramen-dr-cluster-operator-74f89d85d7-6gjhs"}
    PASS
    ok  	github.com/ramendr/ramen/e2e	0.759s

Signed-off-by: Nir Soffer <nsoffer@redhat.com>
  • Loading branch information
@nirs
nirs committed Oct 29, 2024
1 parent 209c378 commit a17056e
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 63 deletions.
55 changes: 41 additions & 14 deletions e2e/util/validation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,32 +8,57 @@ import (
"fmt"
"strings"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

func CheckRamenHubPodRunningStatus(k8sClient *kubernetes.Clientset) (bool, string, error) {
func ValidateRamenHubOperator(k8sClient *kubernetes.Clientset) error {
labelSelector := "app=ramen-hub"
podIdentifier := "ramen-hub-operator"

ramenNameSpace, err := GetRamenNameSpace(k8sClient)
if err != nil {
return false, "", err
return err
}

return CheckPodRunningStatus(k8sClient, ramenNameSpace, labelSelector, podIdentifier)
pod, err := FindPod(k8sClient, ramenNameSpace, labelSelector, podIdentifier)
if err != nil {
return err
}

if pod.Status.Phase != "Running" {
return fmt.Errorf("ramen hub operator pod %q not running (phase %q)",
pod.Name, pod.Status.Phase)
}

Ctx.Log.Info("Ramen hub operator is running", "pod", pod.Name)

return nil
}

func CheckRamenSpokePodRunningStatus(k8sClient *kubernetes.Clientset) (bool, string, error) {
func ValidateRamenDRClusterOperatorPod(k8sClient *kubernetes.Clientset, clusterName string) error {
labelSelector := "app=ramen-dr-cluster"
podIdentifier := "ramen-dr-cluster-operator"

ramenNameSpace, err := GetRamenNameSpace(k8sClient)
if err != nil {
return false, "", err
return err
}

return CheckPodRunningStatus(k8sClient, ramenNameSpace, labelSelector, podIdentifier)
pod, err := FindPod(k8sClient, ramenNameSpace, labelSelector, podIdentifier)
if err != nil {
return err
}

if pod.Status.Phase != "Running" {
return fmt.Errorf("ramen dr cluster operator pod %q not running (phase %q)",
pod.Name, pod.Status.Phase)
}

Ctx.Log.Info("Ramen dr cluster operator is running", "cluster", clusterName, "pod", pod.Name)

return nil
}

func GetRamenNameSpace(k8sClient *kubernetes.Clientset) (string, error) {
Expand Down Expand Up @@ -68,23 +93,25 @@ func IsOpenShiftCluster(k8sClient *kubernetes.Clientset) (bool, error) {
return false, nil
}

// CheckPodRunningStatus checks if there is at least one pod matching the labelSelector
// FindPod checks if there is at least one pod matching the labelSelector
// in the given namespace that is in the "Running" phase and contains the podIdentifier in its name.
func CheckPodRunningStatus(client *kubernetes.Clientset, namespace, labelSelector, podIdentifier string) (
bool, string, error,
func FindPod(client *kubernetes.Clientset, namespace, labelSelector, podIdentifier string) (
*v1.Pod, error,
) {
pods, err := client.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{
LabelSelector: labelSelector,
})
if err != nil {
return false, "", fmt.Errorf("failed to list pods in namespace %s: %v", namespace, err)
return nil, fmt.Errorf("failed to list pods in namespace %s: %v", namespace, err)
}

for _, pod := range pods.Items {
if strings.Contains(pod.Name, podIdentifier) && pod.Status.Phase == "Running" {
return true, pod.Name, nil
for i := range pods.Items {
pod := &pods.Items[i]
if strings.Contains(pod.Name, podIdentifier) {
return pod, nil
}
}

return false, "", nil
return nil, fmt.Errorf("no pod with labelSelctor %q and identifier %q in namespace %q",
labelSelector, podIdentifier, namespace)
}
67 changes: 18 additions & 49 deletions e2e/validation_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,53 +11,22 @@ import (

func Validate(t *testing.T) {
t.Helper()

if !t.Run("CheckRamenHubOperatorStatus", CheckRamenHubOperatorStatus) {
t.Error("CheckRamenHubOperatorStatus failed")
}

if !t.Run("CheckRamenSpokeOperatorStatus", CheckRamenSpokeOperatorStatus) {
t.Error("CheckRamenHubOperatorStatus failed")
}
}

func CheckRamenHubOperatorStatus(t *testing.T) {
util.Ctx.Log.Info("enter CheckRamenHubOperatorStatus")

isRunning, podName, err := util.CheckRamenHubPodRunningStatus(util.Ctx.Hub.K8sClientSet)
if err != nil {
t.Error(err)
}

if isRunning {
util.Ctx.Log.Info("Ramen Hub Operator is running", "pod", podName)
} else {
t.Error("no running Ramen Hub Operator pod found")
}
}

func CheckRamenSpokeOperatorStatus(t *testing.T) {
util.Ctx.Log.Info("enter CheckRamenSpokeOperatorStatus")

isRunning, podName, err := util.CheckRamenSpokePodRunningStatus(util.Ctx.C1.K8sClientSet)
if err != nil {
t.Error(err)
}

if isRunning {
util.Ctx.Log.Info("Ramen Spoke Operator is running on cluster 1", "pod", podName)
} else {
t.Error("no running Ramen Spoke Operator pod on cluster 1")
}

isRunning, podName, err = util.CheckRamenSpokePodRunningStatus(util.Ctx.C2.K8sClientSet)
if err != nil {
t.Error(err)
}

if isRunning {
util.Ctx.Log.Info("Ramen Spoke Operator is running on cluster 2", "pod", podName)
} else {
t.Error("no running Ramen Spoke Operator pod on cluster 2")
}
t.Run("hub", func(t *testing.T) {
err := util.ValidateRamenHubOperator(util.Ctx.Hub.K8sClientSet)
if err != nil {
t.Fatal(err)
}
})
t.Run("c1", func(t *testing.T) {
err := util.ValidateRamenDRClusterOperatorPod(util.Ctx.C1.K8sClientSet, "c1")
if err != nil {
t.Fatal(err)
}
})
t.Run("c2", func(t *testing.T) {
err := util.ValidateRamenDRClusterOperatorPod(util.Ctx.C2.K8sClientSet, "c2")
if err != nil {
t.Fatal(err)
}
})
}

0 comments on commit a17056e

Please sign in to comment.