From f75cd6ca5c217eb6354424226f357e63dbffe6f2 Mon Sep 17 00:00:00 2001 From: anjan-keysight Date: Tue, 13 Feb 2024 09:17:56 +0000 Subject: [PATCH] Add startup probe for ixia-c components --- controllers/ixiatg_controller.go | 43 ++++++++++--- .../liveness/test_liveness_custom_config.py | 8 +-- .../liveness/test_liveness_default_config.py | 10 +-- .../liveness/test_liveness_disabled_config.py | 4 +- .../liveness/test_startup_default_config.py | 61 +++++++++++++++++++ .../liveness/test_startup_disabled_config.py | 53 ++++++++++++++++ tests/py/utils/common.py | 8 ++- version | 2 +- 8 files changed, 166 insertions(+), 23 deletions(-) create mode 100644 tests/py/liveness/test_startup_default_config.py create mode 100644 tests/py/liveness/test_startup_disabled_config.py diff --git a/controllers/ixiatg_controller.go b/controllers/ixiatg_controller.go index 7e27a9c..7c64f83 100644 --- a/controllers/ixiatg_controller.go +++ b/controllers/ixiatg_controller.go @@ -121,6 +121,9 @@ const ( LIVENESS_PERIOD int32 = 10 LIVENESS_FAILURE int32 = 6 + STARTUP_PERIOD int32 = 3 + STARTUP_FAILURE int32 = 20 + MIN_MEM_CONTROLLER string = "25Mi" MIN_MEM_GNMI string = "15Mi" MIN_CPU_PROTOCOL string = "200m" @@ -157,6 +160,7 @@ type componentRel struct { Name string `json:"name"` Path string `json:"path"` Port int32 + StartUpEnable *bool `json:"startup-enable,omitempty"` Tag string `json:"tag"` VolMntName string VolMntPath string @@ -437,9 +441,9 @@ func (r *IxiaTGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr var contStatus []corev1.ContainerStatus if found.Status.Phase != corev1.PodRunning { requeue = true - for _, s := range found.Status.ContainerStatuses { - contStatus = append(contStatus, s) - } + } + for _, s := range found.Status.ContainerStatuses { + contStatus = append(contStatus, s) } for _, podEntry := range ixia.Status.Interfaces { err = r.Get(ctx, types.NamespacedName{Name: podEntry.PodName, Namespace: ixia.Namespace}, found) @@ -452,9 +456,9 @@ func (r *IxiaTGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } if found.Status.Phase != corev1.PodRunning { requeue = true - for _, s := range found.Status.ContainerStatuses { - contStatus = append(contStatus, s) - } + } + for _, s := range found.Status.ContainerStatuses { + contStatus = append(contStatus, s) } } @@ -467,6 +471,9 @@ func (r *IxiaTGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr break } } + if !c.Ready { + requeue = true + } } } } @@ -1258,6 +1265,14 @@ func (r *IxiaTGReconciler) containersForController(ctx context.Context, ixia *ne } container.LivenessProbe = &probe } + if pbHdlr != nil && (comp.StartUpEnable == nil || *comp.StartUpEnable) { + probe := corev1.Probe{ + ProbeHandler: *pbHdlr, + PeriodSeconds: STARTUP_PERIOD, + FailureThreshold: STARTUP_FAILURE, + } + container.StartupProbe = &probe + } updateControllerContainer(&container, comp, newGNMI) // License server related handling @@ -1334,6 +1349,7 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string, } for cName, comp := range componentDep[versionToDeploy].Ixia.Containers { var tcpSock corev1.TCPSocketAction + var pbHdlr *corev1.ProbeHandler = nil if strings.HasPrefix(comp.Name, INIT_CONT_NAME_PREFIX) { continue } @@ -1362,6 +1378,7 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string, if cName == IMAGE_PROTOCOL_ENG { compCopy.DefEnv["INTF_LIST"] = strings.Join(intfList, ",") tcpSock = corev1.TCPSocketAction{Port: intstr.IntOrString{IntVal: PROTOCOL_ENG_PORT}} + pbHdlr = &corev1.ProbeHandler{TCPSocket: &tcpSock} if _, ok := resRequest["cpu"]; !ok { resRequest["cpu"] = resource.MustParse(MIN_CPU_PROTOCOL) } @@ -1375,6 +1392,7 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string, } else { compCopy.DefEnv["ARG_IFACE_LIST"] = argIntfList tcpSock = corev1.TCPSocketAction{Port: intstr.IntOrString{IntVal: TRAFFIC_ENG_PORT}} + pbHdlr = &corev1.ProbeHandler{TCPSocket: &tcpSock} if _, ok := resRequest["cpu"]; !ok { resRequest["cpu"] = resource.MustParse(MIN_CPU_TRAFFIC) } @@ -1385,10 +1403,9 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string, } container.Resources.Requests = resRequest - if compCopy.LiveNessEnable == nil || *compCopy.LiveNessEnable { - pbHdlr := corev1.ProbeHandler{TCPSocket: &tcpSock} + if pbHdlr != nil && (compCopy.LiveNessEnable == nil || *compCopy.LiveNessEnable) { probe := corev1.Probe{ - ProbeHandler: pbHdlr, + ProbeHandler: *pbHdlr, InitialDelaySeconds: compCopy.LiveNessDelay, PeriodSeconds: compCopy.LiveNessPeriod, FailureThreshold: compCopy.LiveNessFailure, @@ -1396,6 +1413,14 @@ func (r *IxiaTGReconciler) containersForIxia(podName string, intfList []string, } container.LivenessProbe = &probe } + if pbHdlr != nil && (compCopy.StartUpEnable == nil || *compCopy.StartUpEnable) { + probe := corev1.Probe{ + ProbeHandler: *pbHdlr, + PeriodSeconds: STARTUP_PERIOD, + FailureThreshold: STARTUP_FAILURE, + } + container.StartupProbe = &probe + } updateControllerContainer(&container, compCopy, false) log.Infof("Adding to pod: %s, container: %s, Image: %s, Args: %v, Cmd: %v, Env: %v", podName, name, image, container.Args, container.Command, container.Env) diff --git a/tests/py/liveness/test_liveness_custom_config.py b/tests/py/liveness/test_liveness_custom_config.py index 4092b2b..8641ce3 100644 --- a/tests/py/liveness/test_liveness_custom_config.py +++ b/tests/py/liveness/test_liveness_custom_config.py @@ -32,10 +32,10 @@ def test_liveness_custom_config(): utils.load_liveness_configmap(probe_params) utils.create_kne_config(namespace1_config, namespace1) utils.ixia_c_pods_ok(namespace1, expected_pods) - utils.check_liveness_data('ixia-c', expected_pods[0], namespace1, True, 1, 10, 3) - utils.check_liveness_data('gnmi', expected_pods[0], namespace1, False) - utils.check_liveness_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, True, 12, 10, 6) - utils.check_liveness_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, True, 1, 5, 6) + utils.check_probe_data('ixia-c', expected_pods[0], namespace1, True, True, 1, 10, 3) + utils.check_probe_data('gnmi', expected_pods[0], namespace1, True, False) + utils.check_probe_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, True, True, 12, 10, 6) + utils.check_probe_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, True, True, 1, 5, 6) op_rscount = utils.ixia_c_operator_ok(op_rscount) print("[Namespace:{}]Deleting KNE topology".format( diff --git a/tests/py/liveness/test_liveness_default_config.py b/tests/py/liveness/test_liveness_default_config.py index 16ca7c8..f88defe 100644 --- a/tests/py/liveness/test_liveness_default_config.py +++ b/tests/py/liveness/test_liveness_default_config.py @@ -35,11 +35,11 @@ def test_liveness_default_config(): )) utils.create_kne_config(namespace1_config, namespace1) utils.ixia_c_pods_ok(namespace1, expected_pods) - utils.check_liveness_data('ixia-c', expected_pods[0], namespace1, True, 1, 10, 6) - utils.check_liveness_data('gnmi', expected_pods[0], namespace1, True, 1, 10, 6) - utils.check_liveness_data('license-server', expected_pods[0], namespace1, True, 1, 10, 6) - utils.check_liveness_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, True, 1, 10, 6) - utils.check_liveness_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, True, 1, 10, 6) + utils.check_probe_data('ixia-c', expected_pods[0], namespace1, True, True, 1, 10, 6) + utils.check_probe_data('gnmi', expected_pods[0], namespace1, True, True, 1, 10, 6) + utils.check_probe_data('license-server', expected_pods[0], namespace1, True, True, 1, 10, 6) + utils.check_probe_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, True, True, 1, 10, 6) + utils.check_probe_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, True, True, 1, 10, 6) op_rscount = utils.ixia_c_operator_ok(op_rscount) print("[Namespace:{}]Deleting KNE topology".format( diff --git a/tests/py/liveness/test_liveness_disabled_config.py b/tests/py/liveness/test_liveness_disabled_config.py index 05bd8ac..484d700 100644 --- a/tests/py/liveness/test_liveness_disabled_config.py +++ b/tests/py/liveness/test_liveness_disabled_config.py @@ -10,7 +10,7 @@ def test_liveness_disabled_config(): Delete pd kne topology, - namespace - 1: ixia-c Validate, - - disabled liveness parameters for protocol engines + - disabled liveness parameters for traffic engines """ namespace1 = 'ixia-c' namespace1_config = 'ixia_c_pd_topology.yaml' @@ -29,7 +29,7 @@ def test_liveness_disabled_config(): utils.load_liveness_configmap(probe_params) utils.create_kne_config(namespace1_config, namespace1) utils.ixia_c_pods_ok(namespace1, expected_pods) - utils.check_liveness_data(expected_pods[1]+container_extension, expected_pods[1], namespace1, False) + utils.check_probe_data(expected_pods[1]+container_extension, expected_pods[1], namespace1, True, False) op_rscount = utils.ixia_c_operator_ok(op_rscount) print("[Namespace:{}]Deleting KNE topology".format( diff --git a/tests/py/liveness/test_startup_default_config.py b/tests/py/liveness/test_startup_default_config.py new file mode 100644 index 0000000..758e06a --- /dev/null +++ b/tests/py/liveness/test_startup_default_config.py @@ -0,0 +1,61 @@ +import pytest +import utils +import time + +@pytest.mark.liveness +def test_startup_default_config(): + """ + Deploy pd kne topology with default version, + - namespace - 1: ixia-c + Delete pd kne topology, + - namespace - 1: ixia-c + Validate, + - default startup probe parameters for all ixia-c components + """ + namespace1 = 'ixia-c' + namespace1_config = 'ixia_c_pd_topology.yaml' + expected_pods = [ + 'otg-controller', + 'otg-port-eth1', + 'arista1' + ] + container_extensions = [ + '-protocol-engine', + '-traffic-engine' + ] + try: + op_rscount = utils.get_operator_restart_count() + print("[Namespace:{}]Deploying KNE topology".format( + namespace1 + )) + utils.create_kne_config(namespace1_config, namespace1) + utils.ixia_c_pods_ok(namespace1, expected_pods) + utils.check_probe_data('ixia-c', expected_pods[0], namespace1, False, True, 0, 3, 20) + utils.check_probe_data('gnmi', expected_pods[0], namespace1, False, True, 0, 3, 20) + utils.check_probe_data('license-server', expected_pods[0], namespace1, False, True, 0, 3, 20) + utils.check_probe_data(expected_pods[1]+container_extensions[0], expected_pods[1], namespace1, False, True, 0, 3, 20) + utils.check_probe_data(expected_pods[1]+container_extensions[1], expected_pods[1], namespace1, False, True, 0, 3, 20) + op_rscount = utils.ixia_c_operator_ok(op_rscount) + + print("[Namespace:{}]Deleting KNE topology".format( + namespace1 + )) + utils.delete_kne_config(namespace1_config, namespace1) + utils.ixia_c_pods_ok(namespace1, []) + utils.wait_for( + lambda: utils.topology_deleted(namespace1), + 'topology deleted', + timeout_seconds=30 + ) + op_rscount = utils.ixia_c_operator_ok(op_rscount) + + finally: + utils.delete_kne_config(namespace1_config, namespace1) + utils.ixia_c_pods_ok(namespace1, []) + + utils.wait_for( + lambda: utils.topology_deleted(namespace1), + 'topology deleted', + timeout_seconds=30 + ) + time.sleep(5) diff --git a/tests/py/liveness/test_startup_disabled_config.py b/tests/py/liveness/test_startup_disabled_config.py new file mode 100644 index 0000000..b80859f --- /dev/null +++ b/tests/py/liveness/test_startup_disabled_config.py @@ -0,0 +1,53 @@ +import pytest +import utils +import time + +@pytest.mark.liveness +def test_liveness_disabled_config(): + """ + Deploy pd kne topology with default version, + - namespace - 1: ixia-c + Delete pd kne topology, + - namespace - 1: ixia-c + Validate, + - disabled startup probe parameters for protocol engines + """ + namespace1 = 'ixia-c' + namespace1_config = 'ixia_c_pd_topology.yaml' + expected_pods = [ + 'otg-controller', + 'otg-port-eth1', + 'arista1' + ] + container_extension = '-protocol-engine' + probe_params = {'protocol-engine':{'startup-enable': False}} + try: + op_rscount = utils.get_operator_restart_count() + print("[Namespace:{}]Deploying KNE topology".format( + namespace1 + )) + utils.load_liveness_configmap(probe_params) + utils.create_kne_config(namespace1_config, namespace1) + utils.ixia_c_pods_ok(namespace1, expected_pods) + utils.check_probe_data(expected_pods[1]+container_extension, expected_pods[1], namespace1, False, False) + op_rscount = utils.ixia_c_operator_ok(op_rscount) + + print("[Namespace:{}]Deleting KNE topology".format( + namespace1 + )) + utils.delete_kne_config(namespace1_config, namespace1) + utils.ixia_c_pods_ok(namespace1, []) + op_rscount = utils.ixia_c_operator_ok(op_rscount) + utils.reset_configmap() + + finally: + utils.delete_kne_config(namespace1_config, namespace1) + utils.ixia_c_pods_ok(namespace1, []) + utils.reset_configmap() + + utils.wait_for( + lambda: utils.topology_deleted(namespace1), + 'topology deleted', + timeout_seconds=30 + ) + time.sleep(5) \ No newline at end of file diff --git a/tests/py/utils/common.py b/tests/py/utils/common.py index 4f88143..aed1ed9 100644 --- a/tests/py/utils/common.py +++ b/tests/py/utils/common.py @@ -561,8 +561,12 @@ def ixia_c_operator_ok(prev_op_rscount): return op_rscount -def check_liveness_data(cont, pod, namespace, enabled=True, delay=0, period=0, failure=0): - base_cmd = "'jsonpath={.spec.containers[?(@.name==\"" + cont + "\")].livenessProbe" +def check_probe_data(cont, pod, namespace, liveness=True, enabled=True, delay=0, period=0, failure=0): + base_cmd = "'jsonpath={.spec.containers[?(@.name==\"" + cont + "\")]." + if liveness: + base_cmd = base_cmd + "livenessProbe" + else: + base_cmd = base_cmd + "startupProbe" base_cmd = "kubectl get pod/{} -n {} -o ".format(pod, namespace) + base_cmd cmd = base_cmd + "}'" out, _ = exec_shell(cmd, True, True) diff --git a/version b/version index 0da054d..e23fb32 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.3.25 \ No newline at end of file +0.3.26