CSPL-2344 Upgrade Strategy for SHC

.github/workflows/helm-test-workflow.yml

@@ -2,6 +2,7 @@ name: Helm Test WorkFlow
 on:
   push:
     branches:
+      - cspl-2344-shc
       - develop
       - main
 jobs:

.github/workflows/int-test-workflow.yml

@@ -2,6 +2,7 @@ name: Integration Test WorkFlow
 on:
   push:
     branches:
+      - cspl-2344-shc
       - develop
       - main
       - feature**

pkg/splunk/enterprise/clustermanager_test.go

@@ -867,7 +867,7 @@ func TestAppFrameworkApplyClusterManagerShouldNotFail(t *testing.T) {
 	}
 }
 
-func TestApplyCLusterManagerDeletion(t *testing.T) {
+func TestApplyClusterManagerDeletion(t *testing.T) {
 	ctx := context.TODO()
 	cm := enterpriseApi.ClusterManager{
 		ObjectMeta: metav1.ObjectMeta{

pkg/splunk/enterprise/monitoringconsole.go

@@ -149,6 +149,14 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie
 	if cr.Status.Phase == enterpriseApi.PhaseReady {
 		finalResult := handleAppFrameworkActivity(ctx, client, cr, &cr.Status.AppContext, &cr.Spec.AppFrameworkConfig)
 		result = *finalResult
+
+		// TODO: Fix the Change Annotation logic/ find an alternative (eg. state machine); right now the search head deployer
+		// starts terminating with this and few replicas do not come up properly after that
+
+		// err = changeSearchHeadAnnotations(ctx, client, cr)
+		// if err != nil {
+		// 	return result, err
+		// }
 	}
 	// RequeueAfter if greater than 0, tells the Controller to requeue the reconcile key after the Duration.
 	// Implies that Requeue is true, there is no need to set Requeue to true at the same time as RequeueAfter.

pkg/splunk/enterprise/searchheadcluster.go

@@ -32,6 +32,7 @@ import (
 	splutil "github.com/splunk/splunk-operator/pkg/splunk/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/remotecommand"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -160,6 +161,11 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie
 		return result, err
 	}
 
+	continueReconcile, err := isSearchHeadReadyForUpgrade(ctx, client, cr)
+	if err != nil || !continueReconcile {
+		return result, err
+	}
+
 	deployerManager := splctrl.DefaultStatefulSetPodManager{}
 	phase, err := deployerManager.Update(ctx, client, statefulSet, 1)
 	if err != nil {
@@ -179,7 +185,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie
 		return result, err
 	}
 
-	mgr := newSerachHeadClusterPodManager(client, scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient)
+	mgr := newSearchHeadClusterPodManager(client, scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient)
 	phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas)
 	if err != nil {
 		return result, err
@@ -247,7 +253,7 @@ type searchHeadClusterPodManager struct {
 }
 
 // newSerachHeadClusterPodManager function to create pod manager this is added to write unit test case
-var newSerachHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager {
+var newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager {
 	return searchHeadClusterPodManager{
 		log:             log,
 		cr:              cr,
@@ -667,3 +673,69 @@ func getSearchHeadClusterList(ctx context.Context, c splcommon.ControllerClient,
 
 	return objectList, nil
 }
+
+// isSearchHeadReadyForUpgrade checks if SearchHeadCluster can be upgraded if a version upgrade is in-progress
+// No-operation otherwise; returns bool, err accordingly
+func isSearchHeadReadyForUpgrade(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.SearchHeadCluster) (bool, error) {
+	reqLogger := log.FromContext(ctx)
+	scopedLog := reqLogger.WithName("isSearchHeadReadyForUpgrade").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace())
+	eventPublisher, _ := newK8EventPublisher(c, cr)
+
+	// check if a MonitoringConsole is attached to the instance
+	monitoringConsoleRef := cr.Spec.MonitoringConsoleRef
+	if monitoringConsoleRef.Name == "" {
+		return true, nil
+	}
+
+	namespacedName := types.NamespacedName{
+		Namespace: cr.GetNamespace(),
+		Name:      GetSplunkStatefulsetName(SplunkSearchHead, cr.GetName()),
+	}
+
+	// check if the stateful set is created at this instance
+	statefulSet := &appsv1.StatefulSet{}
+	err := c.Get(ctx, namespacedName, statefulSet)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return true, nil
+		}
+		eventPublisher.Warning(ctx, "isSearchHeadReadyForUpgrade", fmt.Sprintf("Could not find the Search Head stateful set. Reason %v", err))
+		scopedLog.Error(err, "Unable to get Stateful Set")
+		return false, err
+	}
+
+	namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: monitoringConsoleRef.Name}
+	monitoringConsole := &enterpriseApi.MonitoringConsole{}
+
+	// get the monitoring console referred in search head cluster
+	err = c.Get(ctx, namespacedName, monitoringConsole)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return true, nil
+		}
+		eventPublisher.Warning(ctx, "isSearchHeadReadyForUpgrade", fmt.Sprintf("Could not find the Monitoring Console. Reason %v", err))
+		scopedLog.Error(err, "Unable to get Monitoring Console")
+		return false, err
+	}
+
+	mcImage, err := getCurrentImage(ctx, c, monitoringConsole, SplunkMonitoringConsole)
+	if err != nil {
+		eventPublisher.Warning(ctx, "isSearchHeadReadyForUpgrade", fmt.Sprintf("Could not get the Monitoring Console Image. Reason %v", err))
+		scopedLog.Error(err, "Unable to get Monitoring Console current image")
+		return false, err
+	}
+
+	shcImage, err := getCurrentImage(ctx, c, cr, SplunkSearchHead)
+	if err != nil {
+		eventPublisher.Warning(ctx, "isSearchHeadReadyForUpgrade", fmt.Sprintf("Could not get the Search Head Image. Reason %v", err))
+		scopedLog.Error(err, "Unable to get Search Head current image")
+		return false, err
+	}
+
+	// check if an image upgrade is happening and whether the SearchHeadCluster is ready for the upgrade
+	if (cr.Spec.Image != shcImage) && (monitoringConsole.Status.Phase != enterpriseApi.PhaseReady || mcImage != cr.Spec.Image) {
+		return false, nil
+	}
+
+	return true, nil
+}

pkg/splunk/enterprise/searchheadcluster_test.go

@@ -1436,7 +1436,7 @@ func TestSearchHeadClusterWithReadyState(t *testing.T) {
 	}
 
 	// mock new search pod manager
-	newSerachHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager {
+	newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager {
 		return searchHeadClusterPodManager{
 			log:     log,
 			cr:      cr,
@@ -1873,3 +1873,91 @@ func TestSearchHeadClusterWithReadyState(t *testing.T) {
 		debug.PrintStack()
 	}
 }
+
+func TestIsSearchHeadReadyForUpgrade(t *testing.T) {
+	ctx := context.TODO()
+
+	builder := fake.NewClientBuilder()
+	client := builder.Build()
+	utilruntime.Must(enterpriseApi.AddToScheme(clientgoscheme.Scheme))
+
+	// Create License Manager
+	mc := enterpriseApi.MonitoringConsole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "test",
+		},
+		Spec: enterpriseApi.MonitoringConsoleSpec{
+			CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{
+				Spec: enterpriseApi.Spec{
+					ImagePullPolicy: "Always",
+					Image:           "splunk/splunk:latest",
+				},
+				Volumes: []corev1.Volume{},
+			},
+		},
+	}
+
+	err := client.Create(ctx, &mc)
+	_, err = ApplyMonitoringConsole(ctx, client, &mc)
+	if err != nil {
+		t.Errorf("applyMonitoringConsole should not have returned error; err=%v", err)
+	}
+	mc.Status.Phase = enterpriseApi.PhaseReady
+	err = client.Status().Update(ctx, &mc)
+	if err != nil {
+		t.Errorf("Unexpected status update  %v", err)
+		debug.PrintStack()
+	}
+
+	// Create Search Head Cluster
+	shc := enterpriseApi.SearchHeadCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "test",
+		},
+		Spec: enterpriseApi.SearchHeadClusterSpec{
+			CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{
+				Spec: enterpriseApi.Spec{
+					ImagePullPolicy: "Always",
+					Image:           "splunk/splunk:latest",
+				},
+				Volumes: []corev1.Volume{},
+				MonitoringConsoleRef: corev1.ObjectReference{
+					Name: "test",
+				},
+			},
+			Replicas: int32(3),
+		},
+	}
+
+	err = client.Create(ctx, &shc)
+	_, err = ApplySearchHeadCluster(ctx, client, &shc)
+	if err != nil {
+		t.Errorf("applySearchHeadCluster should not have returned error; err=%v", err)
+	}
+
+	mc.Spec.Image = "splunk2"
+	shc.Spec.Image = "splunk2"
+	_, err = ApplyMonitoringConsole(ctx, client, &mc)
+
+	searchHeadCluster := &enterpriseApi.SearchHeadCluster{}
+	namespacedName := types.NamespacedName{
+		Name:      shc.Name,
+		Namespace: shc.Namespace,
+	}
+	err = client.Get(ctx, namespacedName, searchHeadCluster)
+	if err != nil {
+		t.Errorf("Get Search Head Cluster should not have returned error=%v", err)
+	}
+
+	check, err := isSearchHeadReadyForUpgrade(ctx, client, searchHeadCluster)
+
+	if err != nil {
+		t.Errorf("Unexpected upgradeScenario error %v", err)
+	}
+
+	if !check {
+		t.Errorf("isSearchHeadReadyForUpgrade: SHC should be ready for upgrade")
+	}
+}