Merge pull request #39 from open-traffic-generator/dev-startup-probe

Add startup probe for ixia-c components
open-traffic-generator · Feb 16, 2024 · 6c4ac83 · 6c4ac83
2 parents 86b82ad + f75cd6c
commit 6c4ac83
Show file tree

Hide file tree

Showing 8 changed files with 166 additions and 23 deletions.
diff --git a/controllers/ixiatg_controller.go b/controllers/ixiatg_controller.go
@@ -121,6 +121,9 @@ const (
 	LIVENESS_PERIOD  int32 = 10
 	LIVENESS_FAILURE int32 = 6
 
+	STARTUP_PERIOD  int32 = 3
+	STARTUP_FAILURE int32 = 20
+
 	MIN_MEM_CONTROLLER string = "25Mi"
 	MIN_MEM_GNMI       string = "15Mi"
 	MIN_CPU_PROTOCOL   string = "200m"
@@ -157,6 +160,7 @@ type componentRel struct {
 	Name            string                 `json:"name"`
 	Path            string                 `json:"path"`
 	Port            int32
+	StartUpEnable   *bool  `json:"startup-enable,omitempty"`
 	Tag             string `json:"tag"`
 	VolMntName      string
 	VolMntPath      string
@@ -437,9 +441,9 @@ func (r *IxiaTGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			var contStatus []corev1.ContainerStatus
 			if found.Status.Phase != corev1.PodRunning {
 				requeue = true
-				for _, s := range found.Status.ContainerStatuses {
-					contStatus = append(contStatus, s)
-				}
+			}
+			for _, s := range found.Status.ContainerStatuses {
+				contStatus = append(contStatus, s)
 			}
 			for _, podEntry := range ixia.Status.Interfaces {
 				err = r.Get(ctx, types.NamespacedName{Name: podEntry.PodName, Namespace: ixia.Namespace}, found)
@@ -452,9 +456,9 @@ func (r *IxiaTGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 				}
 				if found.Status.Phase != corev1.PodRunning {
 					requeue = true
-					for _, s := range found.Status.ContainerStatuses {
-						contStatus = append(contStatus, s)
-					}
+				}
+				for _, s := range found.Status.ContainerStatuses {
+					contStatus = append(contStatus, s)
 				}
 			}
 
@@ -467,6 +471,9 @@ func (r *IxiaTGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 							break
 						}
 					}
+					if !c.Ready {
+						requeue = true
+					}
 				}
 			}
 		}
@@ -1258,6 +1265,14 @@ func (r *IxiaTGReconciler) containersForController(ctx context.Context, ixia *ne
 			}
 			container.LivenessProbe = &probe
 		}
+		if pbHdlr != nil && (comp.StartUpEnable == nil || *comp.StartUpEnable) {
+			probe := corev1.Probe{
+				ProbeHandler:     *pbHdlr,
+				PeriodSeconds:    STARTUP_PERIOD,
+				FailureThreshold: STARTUP_FAILURE,
+			}
+			container.StartupProbe = &probe
+		}
 
 		updateControllerContainer(&container, comp, newGNMI)
 		// License server related handling
@@ -1334,6 +1349,7 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string,
 	}
 	for cName, comp := range componentDep[versionToDeploy].Ixia.Containers {
 		var tcpSock corev1.TCPSocketAction
+		var pbHdlr *corev1.ProbeHandler = nil
 		if strings.HasPrefix(comp.Name, INIT_CONT_NAME_PREFIX) {
 			continue
 		}
@@ -1362,6 +1378,7 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string,
 		if cName == IMAGE_PROTOCOL_ENG {
 			compCopy.DefEnv["INTF_LIST"] = strings.Join(intfList, ",")
 			tcpSock = corev1.TCPSocketAction{Port: intstr.IntOrString{IntVal: PROTOCOL_ENG_PORT}}
+			pbHdlr = &corev1.ProbeHandler{TCPSocket: &tcpSock}
 			if _, ok := resRequest["cpu"]; !ok {
 				resRequest["cpu"] = resource.MustParse(MIN_CPU_PROTOCOL)
 			}
@@ -1375,6 +1392,7 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string,
 		} else {
 			compCopy.DefEnv["ARG_IFACE_LIST"] = argIntfList
 			tcpSock = corev1.TCPSocketAction{Port: intstr.IntOrString{IntVal: TRAFFIC_ENG_PORT}}
+			pbHdlr = &corev1.ProbeHandler{TCPSocket: &tcpSock}
 			if _, ok := resRequest["cpu"]; !ok {
 				resRequest["cpu"] = resource.MustParse(MIN_CPU_TRAFFIC)
 			}
@@ -1385,17 +1403,24 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string,
 		}
 		container.Resources.Requests = resRequest
 
-		if compCopy.LiveNessEnable == nil || *compCopy.LiveNessEnable {
-			pbHdlr := corev1.ProbeHandler{TCPSocket: &tcpSock}
+		if pbHdlr != nil && (compCopy.LiveNessEnable == nil || *compCopy.LiveNessEnable) {
 			probe := corev1.Probe{
-				ProbeHandler:                  pbHdlr,
+				ProbeHandler:                  *pbHdlr,
 				InitialDelaySeconds:           compCopy.LiveNessDelay,
 				PeriodSeconds:                 compCopy.LiveNessPeriod,
 				FailureThreshold:              compCopy.LiveNessFailure,
 				TerminationGracePeriodSeconds: pointer.Int64(1),
 			}
 			container.LivenessProbe = &probe
 		}
+		if pbHdlr != nil && (compCopy.StartUpEnable == nil || *compCopy.StartUpEnable) {
+			probe := corev1.Probe{
+				ProbeHandler:     *pbHdlr,
+				PeriodSeconds:    STARTUP_PERIOD,
+				FailureThreshold: STARTUP_FAILURE,
+			}
+			container.StartupProbe = &probe
+		}
 		updateControllerContainer(&container, compCopy, false)
 		log.Infof("Adding to pod: %s, container: %s, Image: %s, Args: %v, Cmd: %v, Env: %v",
 			podName, name, image, container.Args, container.Command, container.Env)

diff --git a/tests/py/liveness/test_liveness_custom_config.py b/tests/py/liveness/test_liveness_custom_config.py
@@ -32,10 +32,10 @@ def test_liveness_custom_config():
         utils.load_liveness_configmap(probe_params)
         utils.create_kne_config(namespace1_config, namespace1)
         utils.ixia_c_pods_ok(namespace1, expected_pods)
-        utils.check_liveness_data('ixia-c', expected_pods[0], namespace1, True, 1, 10, 3)
-        utils.check_liveness_data('gnmi', expected_pods[0], namespace1, False)
-        utils.check_liveness_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, True, 12, 10, 6)
-        utils.check_liveness_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, True, 1, 5, 6)
+        utils.check_probe_data('ixia-c', expected_pods[0], namespace1, True, True, 1, 10, 3)
+        utils.check_probe_data('gnmi', expected_pods[0], namespace1, True, False)
+        utils.check_probe_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, True, True, 12, 10, 6)
+        utils.check_probe_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, True, True, 1, 5, 6)
         op_rscount = utils.ixia_c_operator_ok(op_rscount)
 
         print("[Namespace:{}]Deleting KNE topology".format(

diff --git a/tests/py/liveness/test_liveness_default_config.py b/tests/py/liveness/test_liveness_default_config.py
@@ -35,11 +35,11 @@ def test_liveness_default_config():
             ))
             utils.create_kne_config(namespace1_config, namespace1)
             utils.ixia_c_pods_ok(namespace1, expected_pods)
-            utils.check_liveness_data('ixia-c', expected_pods[0], namespace1, True, 1, 10, 6)
-            utils.check_liveness_data('gnmi', expected_pods[0], namespace1, True, 1, 10, 6)
-            utils.check_liveness_data('license-server', expected_pods[0], namespace1, True, 1, 10, 6)
-            utils.check_liveness_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, True, 1, 10, 6)
-            utils.check_liveness_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, True, 1, 10, 6)
+            utils.check_probe_data('ixia-c', expected_pods[0], namespace1, True, True, 1, 10, 6)
+            utils.check_probe_data('gnmi', expected_pods[0], namespace1, True, True, 1, 10, 6)
+            utils.check_probe_data('license-server', expected_pods[0], namespace1, True, True, 1, 10, 6)
+            utils.check_probe_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, True, True, 1, 10, 6)
+            utils.check_probe_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, True, True, 1, 10, 6)
             op_rscount = utils.ixia_c_operator_ok(op_rscount)
 
             print("[Namespace:{}]Deleting KNE topology".format(

diff --git a/tests/py/liveness/test_liveness_disabled_config.py b/tests/py/liveness/test_liveness_disabled_config.py
@@ -10,7 +10,7 @@ def test_liveness_disabled_config():
     Delete pd kne topology,
     - namespace - 1: ixia-c
     Validate,
-    - disabled liveness parameters for protocol engines
+    - disabled liveness parameters for traffic engines
     """
     namespace1 = 'ixia-c'
     namespace1_config = 'ixia_c_pd_topology.yaml'
@@ -29,7 +29,7 @@ def test_liveness_disabled_config():
         utils.load_liveness_configmap(probe_params)
         utils.create_kne_config(namespace1_config, namespace1)
         utils.ixia_c_pods_ok(namespace1, expected_pods)
-        utils.check_liveness_data(expected_pods[1]+container_extension, expected_pods[1], namespace1, False)
+        utils.check_probe_data(expected_pods[1]+container_extension, expected_pods[1], namespace1, True, False)
         op_rscount = utils.ixia_c_operator_ok(op_rscount)
 
         print("[Namespace:{}]Deleting KNE topology".format(

diff --git a/tests/py/liveness/test_startup_default_config.py b/tests/py/liveness/test_startup_default_config.py
@@ -0,0 +1,61 @@
+import pytest
+import utils
+import time
+
+@pytest.mark.liveness
+def test_startup_default_config():
+    """
+    Deploy pd kne topology with default version,
+    - namespace - 1: ixia-c
+    Delete pd kne topology,
+    - namespace - 1: ixia-c
+    Validate,
+    - default startup probe parameters for all ixia-c components
+    """
+    namespace1 = 'ixia-c'
+    namespace1_config = 'ixia_c_pd_topology.yaml'
+    expected_pods = [
+        'otg-controller',
+        'otg-port-eth1',
+        'arista1'
+    ]
+    container_extensions = [
+        '-protocol-engine',
+        '-traffic-engine'
+    ]
+    try:
+        op_rscount = utils.get_operator_restart_count()
+        print("[Namespace:{}]Deploying KNE topology".format(
+            namespace1
+        ))
+        utils.create_kne_config(namespace1_config, namespace1)
+        utils.ixia_c_pods_ok(namespace1, expected_pods)
+        utils.check_probe_data('ixia-c', expected_pods[0], namespace1, False, True, 0, 3, 20)
+        utils.check_probe_data('gnmi', expected_pods[0], namespace1, False, True, 0, 3, 20)
+        utils.check_probe_data('license-server', expected_pods[0], namespace1, False, True, 0, 3, 20)
+        utils.check_probe_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, False, True, 0, 3, 20)
+        utils.check_probe_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, False, True, 0, 3, 20)
+        op_rscount = utils.ixia_c_operator_ok(op_rscount)
+
+        print("[Namespace:{}]Deleting KNE topology".format(
+            namespace1
+        ))
+        utils.delete_kne_config(namespace1_config, namespace1)
+        utils.ixia_c_pods_ok(namespace1, [])
+        utils.wait_for(
+            lambda: utils.topology_deleted(namespace1),
+            'topology deleted',
+            timeout_seconds=30
+        )
+        op_rscount = utils.ixia_c_operator_ok(op_rscount)
+
+    finally:
+        utils.delete_kne_config(namespace1_config, namespace1)
+        utils.ixia_c_pods_ok(namespace1, [])
+
+        utils.wait_for(
+            lambda: utils.topology_deleted(namespace1),
+            'topology deleted',
+            timeout_seconds=30
+        )
+        time.sleep(5)
diff --git a/tests/py/liveness/test_startup_disabled_config.py b/tests/py/liveness/test_startup_disabled_config.py
@@ -0,0 +1,53 @@
+import pytest
+import utils
+import time
+
+@pytest.mark.liveness
+def test_liveness_disabled_config():
+    """
+    Deploy pd kne topology with default version,
+    - namespace - 1: ixia-c
+    Delete pd kne topology,
+    - namespace - 1: ixia-c
+    Validate,
+    - disabled startup probe parameters for protocol engines
+    """
+    namespace1 = 'ixia-c'
+    namespace1_config = 'ixia_c_pd_topology.yaml'
+    expected_pods = [
+        'otg-controller',
+        'otg-port-eth1',
+        'arista1'
+    ]
+    container_extension = '-protocol-engine'
+    probe_params = {'protocol-engine':{'startup-enable': False}}
+    try:
+        op_rscount = utils.get_operator_restart_count()
+        print("[Namespace:{}]Deploying KNE topology".format(
+            namespace1
+        ))
+        utils.load_liveness_configmap(probe_params)
+        utils.create_kne_config(namespace1_config, namespace1)
+        utils.ixia_c_pods_ok(namespace1, expected_pods)
+        utils.check_probe_data(expected_pods[1]+container_extension, expected_pods[1], namespace1, False, False)
+        op_rscount = utils.ixia_c_operator_ok(op_rscount)
+
+        print("[Namespace:{}]Deleting KNE topology".format(
+            namespace1
+        ))
+        utils.delete_kne_config(namespace1_config, namespace1)
+        utils.ixia_c_pods_ok(namespace1, [])
+        op_rscount = utils.ixia_c_operator_ok(op_rscount)
+        utils.reset_configmap()
+
+    finally:
+        utils.delete_kne_config(namespace1_config, namespace1)
+        utils.ixia_c_pods_ok(namespace1, [])
+        utils.reset_configmap()
+
+        utils.wait_for(
+            lambda: utils.topology_deleted(namespace1),
+            'topology deleted',
+            timeout_seconds=30
+        )
+        time.sleep(5)
diff --git a/tests/py/utils/common.py b/tests/py/utils/common.py
@@ -561,8 +561,12 @@ def ixia_c_operator_ok(prev_op_rscount):
     return op_rscount
 
 
-def check_liveness_data(cont, pod, namespace, enabled=True, delay=0, period=0, failure=0):
-    base_cmd = "'jsonpath={.spec.containers[?(@.name==\"" + cont + "\")].livenessProbe"
+def check_probe_data(cont, pod, namespace, liveness=True, enabled=True, delay=0, period=0, failure=0):
+    base_cmd = "'jsonpath={.spec.containers[?(@.name==\"" + cont + "\")]."
+    if liveness:
+        base_cmd = base_cmd + "livenessProbe"
+    else:
+        base_cmd = base_cmd + "startupProbe"
     base_cmd = "kubectl get pod/{} -n {} -o ".format(pod, namespace) + base_cmd
     cmd = base_cmd + "}'"
     out, _ = exec_shell(cmd, True, True)

diff --git a/version b/version
@@ -1 +1 @@
-0.3.25
+0.3.26