From fe0566492c4900fe1cfc8338bb84d09531028c2f Mon Sep 17 00:00:00 2001 From: PJHsieh <49477291+PJHsieh@users.noreply.github.com> Date: Mon, 19 Jun 2023 10:14:36 +0800 Subject: [PATCH 1/5] JIRA-JRTC-17881: [Healthd]Add test for fan direction alarm and health led * test_fan_direction_alarm Add test case to test the abiliby for system-health's detection fan direction * test_system_health_led Add test case to test the led color is as expected as defined in system_health configuration file. --- .../test_system_health_fan_direction.py | 278 ++++++++++++++++++ tests/system_health/test_system_health_led.py | 71 +++++ 2 files changed, 349 insertions(+) create mode 100644 tests/system_health/test_system_health_fan_direction.py create mode 100644 tests/system_health/test_system_health_led.py diff --git a/tests/system_health/test_system_health_fan_direction.py b/tests/system_health/test_system_health_fan_direction.py new file mode 100644 index 00000000000..4c597369548 --- /dev/null +++ b/tests/system_health/test_system_health_fan_direction.py @@ -0,0 +1,278 @@ +import logging +import re +import time +import random +import pytest +from datetime import timedelta + +from tests.common.platform.daemon_utils import check_pmon_daemon_enable_status +from tests.common.utilities import wait_until + +from tests.system_health.test_system_health import get_system_health_config + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('any'), + pytest.mark.disable_loganalyzer +] + +daemon_name = "thermalctld" + +daemon_running_str = "RUNNING" +daemon_stopped_str = "STOPPED" +daemon_exited_str = "EXITED" + +FAN_DIR_ERROR_MSG_REX = ".*direction .* is not aligned with .* direction.*" +STATUS_LED_COLOR_MSG_REX = r'System status LED (.*)' +NOT_OK_REX = r'\bNot OK\b' + +TIMEOUT = 60*2 +BOOT_TIME_IN_SECONDS = 300 + + +def is_booting_time_expired(duthost): + uptime = duthost.get_uptime() + logging.info('uptime={}'.format(uptime)) + booting_time = timedelta(seconds=float(BOOT_TIME_IN_SECONDS)) + + return uptime > booting_time + + +def check_thermalctld_pid(duthost, daemon_pid): + result_pid = duthost.shell("docker exec pmon bash -c 'ps -aux| grep thermalctld | head -n -2' | awk '{print $2}' ")['stdout_lines'] + return len(result_pid) == 1 and int(result_pid[0]) == daemon_pid + + +def check_expected_daemon_status(duthost, expected_daemon_status): + daemon_status, _ = duthost.get_pmon_daemon_status(daemon_name) + logging.info("daemon_status = {}, expected_daemon_status = {}".format(daemon_status, expected_daemon_status)) + # check whether the daemon_status got from 'get_pmon_daemon_status()' matches the expected status + if daemon_status != expected_daemon_status: + return False + else: + # if the expected status matches, check extra conditions under different expected daemon status + if daemon_status == daemon_running_str: + result_pid = duthost.shell("docker exec pmon bash -c 'ps -aux| grep thermalctld | head -n -2' | awk '{print $2}' ")['stdout_lines'] + return (len(result_pid) >= 2) + + return True + + +@pytest.fixture(scope="function") +def setup(duthosts, rand_one_dut_hostname): + duthost = duthosts[rand_one_dut_hostname] + daemon_en_status = check_pmon_daemon_enable_status(duthost, daemon_name) + if daemon_en_status is False: + pytest.skip("{} is not enabled in {}".format(daemon_name, duthost.facts['platform'])) + data = collect_data(duthost) + + if len(data['keys']) == 0: + pytest.skip("(ECMSG-feature) Fan direction not support on this dut") + + select_key = random.sample(data['keys'], 1) + select_direction = data['data'][select_key[0]] + + # check all fan direction is the same + for direction in data['data'].items(): + assert direction[1] == select_direction, "Not all fans spin the same way before testing." + + daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) + logger.info("{} daemon is {} with pid {}".format(daemon_name, daemon_status, daemon_pid)) + stop_thermalctld_child_processes(duthost, daemon_pid) + + led_dict = get_system_health_config(duthost, "led_color", None) + + sys_led_color = get_led_color_by_system_health_summary(duthost) + + return data, led_dict, sys_led_color + + +@pytest.fixture(scope="function", autouse=True) +def teardown_function(duthosts, rand_one_dut_hostname, setup): + _, _, sys_led_color = setup + logging.info("sys_led_color = {}".format(sys_led_color)) + duthost = duthosts[rand_one_dut_hostname] + wait_until(300, 5, 0, is_booting_time_expired, duthost) + + yield + + duthost.stop_pmon_daemon(daemon_name) + assert(wait_until(50, 10, 0, check_expected_daemon_status, duthost, daemon_stopped_str)), "The daemon thermalctld is not successfully stopped" + + duthost.start_pmon_daemon(daemon_name) + logger.info("To make sure thermalctld two process are running.") + assert(wait_until(50, 10, 0, check_expected_daemon_status, duthost, daemon_running_str)), "The daemon thermalctld is not successfully restarted" + + if sys_led_color is not None: + assert wait_until(TIMEOUT, 10, 0, verify_sys_led_status, + duthost, sys_led_color), \ + "The color {} of the System Status LED is not successfully restored.".format(sys_led_color) + + +def stop_thermalctld_child_processes(duthost, daemon_pid): + """ + Stop thermalctld child processes. + + Args: + duthost(obj): dut object + daemon_pid(int): thermalctld main process pid. + """ + logging.info("Stopping thermalctld child processes for PID: {}".format(daemon_pid)) + + child_list = duthost.shell("docker exec pmon bash -c 'pgrep -P {}'".format(daemon_pid))['stdout_lines'] + + cmd = "docker exec pmon bash -c 'kill -9 {}'" + for pid_str in child_list: + pid = int(pid_str) + logging.info(cmd.format(pid)) + try: + duthost.shell(cmd.format(pid)) + except Exception as e: + logging.error("Failed to stop process with PID {}: {}".format(pid, str(e))) + + assert(wait_until(30, 3, 0, check_thermalctld_pid, duthost, daemon_pid)), "Can't stop thermalctld child processes within expceted time." + + +def collect_data(duthost): + """ + Collect fan direction data through STATE_DB. + + Return + dev_keys : list of device name as keys in state db. + dev_data : dict of fan direction (intake/exhaust) + """ + keys = duthost.shell('sonic-db-cli STATE_DB KEYS "FAN_INFO|*"')['stdout_lines'] + + dev_keys = [] + dev_data = {} + for k in keys: + data = duthost.shell('sonic-db-cli STATE_DB HGET "{}" "direction"'.format(k))['stdout_lines'] + if 'N/A' not in data: + # skip fan direction is 'N/A'. + dev_data[k] = data + dev_keys.append(k) + + return {'keys': dev_keys, 'data': dev_data} + + +def get_system_health_summary_output(duthost): + """ + Retrieves the system health summary output. + + Args: + duthost: The host on which the command is executed. + + Returns: + msg_list(list): A list of strings representing the system health summary messages. + """ + + msg_list = duthost.shell("show system-health summary")["stdout_lines"] + return msg_list + + +def parse_system_health_summary(duthost, msg_regex): + """ + Parse the system health summary output + + Args: + duthost: The host on which the command is executed. + msg_regex: The regular expression to match against the error messages. + + Return: + True if a matching string is found in the messages, otherwise False. + """ + message = get_system_health_summary_output(duthost) + logging.info("message = {}".format(message)) + + for string in message: + if re.search(msg_regex, string): + return True + + return False + + +def get_led_color_by_system_health_summary(duthost): + """ + Retrieves the LED color from the system health summary output. + + Args: + duthost: The host on which the command is executed. + + Returns: + color(string): The LED color message if found, otherwise None. + """ + logging.info("Enter API {}".format(get_led_color_by_system_health_summary.__name__)) + color = None + msg_list = get_system_health_summary_output(duthost) + + for msg in msg_list: + match = re.search(STATUS_LED_COLOR_MSG_REX, msg) + if match: + color = match.group(1) + logging.info("color = {}".format(color)) + return color + + return color + + +def verify_sys_led_status(duthost, expected_value): + ''' + Verify the system LED status. + + Parameters: + led_status (str): The current LED status obtained from the system. + expected_value (str): The expected LED status for verification. + + Returns: + bool: True if the expected LED status is found in the current LED status; False otherwise. + ''' + + + led_status = get_led_color_by_system_health_summary(duthost) + logging.info("led_status{}, expected{}".format(led_status, expected_value)) + return expected_value in led_status + + +def test_fan_direction_alarm(duthosts, rand_one_dut_hostname, setup): + """ + Test Scenario: + S1.Select one running fan. + S2.Set the fan direction to another one to State DB. + S3.Check the system-health output as expected or not. + S4.Restore the fan direction. + S5.Check the error message disappear or not. + """ + + duthost = duthosts[rand_one_dut_hostname] + data, led_dict, sys_led_color = setup + logging.info("data{}, led_dict{}, color{}".format(data, led_dict, sys_led_color )) + + #S1.Select one running fan. + select_key = random.sample(data['keys'], 1) + select_direction = data['data'][select_key[0]] + logging.info("select_direction{}".format(select_direction)) + + + #S2.Set the fan direction to another one to State DB. + change_direction = 'intake' if 'exhaust' in select_direction else 'exhaust' + duthost.shell('sonic-db-cli STATE_DB HSET "{}" "direction" "{}"'.format(select_key[0], change_direction)) + + + + #S3.Check the system-health output as expected or not. + # Contain the string match pattern FAN_DIR_ERROR_MSG_REX + # Contain the string match led_color["fault"] defined in system_health_monitoring_config.json + assert(parse_system_health_summary(duthost, FAN_DIR_ERROR_MSG_REX)), "Missing error message about fans spin in different way" + + if led_dict.get("fault", None) is not None: + assert wait_until(TIMEOUT, 10, 0, verify_sys_led_status, + duthost, led_dict["fault"]), \ + "The color {} of the System Status LED is not shown.".format(led_dict["fault"]) + + #S4.Restore the fan direction. + duthost.shell('sonic-db-cli STATE_DB HSET "{}" "direction" "{}"'.format(select_key[0], select_direction[0])) + + #S5.Check the system-health output error message disappear or not. + assert(not parse_system_health_summary(duthost, FAN_DIR_ERROR_MSG_REX)), "Shouldn't have error message." + diff --git a/tests/system_health/test_system_health_led.py b/tests/system_health/test_system_health_led.py new file mode 100644 index 00000000000..5aaf365027c --- /dev/null +++ b/tests/system_health/test_system_health_led.py @@ -0,0 +1,71 @@ +import logging +import re +import pytest + +from tests.common.platform.daemon_utils import check_pmon_daemon_enable_status +from tests.common.utilities import wait_until + +from tests.system_health.test_system_health import get_system_health_config + +from tests.system_health.test_system_health_fan_direction import get_led_color_by_system_health_summary +from tests.system_health.test_system_health_fan_direction import verify_sys_led_status, is_booting_time_expired + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('any'), + pytest.mark.disable_loganalyzer +] + + +STATUS_LED_COLOR_MSG_REX = r'System status LED (.*)' +NOT_OK_REX = r'\bNot OK\b' +TIMEOUT = 60*2 + + +@pytest.fixture(scope="function") +def setup(duthosts, rand_one_dut_hostname): + duthost = duthosts[rand_one_dut_hostname] + led_dict = get_system_health_config(duthost, "led_color", None) + sys_led_color = get_led_color_by_system_health_summary(duthost) + + return led_dict, sys_led_color + + +@pytest.fixture(scope="function", autouse=True) +def teardown_function(duthosts, rand_one_dut_hostname, setup): + led_dict, sys_led_color = setup + logging.info("sys_led_color = {}".format(sys_led_color)) + duthost = duthosts[rand_one_dut_hostname] + wait_until(300, 5, 0, is_booting_time_expired, duthost) + + yield + + logger.info("Restart pmon container") + stdout = duthost.shell("docker start pmon")['stdout_lines'] + + if sys_led_color is None: + return + + assert wait_until(TIMEOUT, 10, 0, verify_sys_led_status, + duthost, sys_led_color), \ + "The color {} of the System Status LED is not successfully restored.".format(sys_led_color) + + +def test_system_health_led(duthosts, enum_rand_one_per_hwsku_hostname, setup): + """ + Test Scenario: + S1. pmon container + S2. Check System Status Led Status from show system-health summary output + """ + + #S1. Stop pmon container + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + led_dict, sys_led_color = setup + stdout = duthost.shell("docker stop pmon")['stdout_lines'] + + #S2. Check System Status Led Status from show system-health summary output + if led_dict.get("fault", None) is not None: + assert wait_until(TIMEOUT, 10, 0, verify_sys_led_status, + duthost, led_dict["fault"]), \ + "The color {} of the System Status LED is not shown.".format(led_dict["fault"]) From 18b747645fa9163bb5dd35541bfd5b3879e7145c Mon Sep 17 00:00:00 2001 From: ecsonic <49477291+PJHsieh@users.noreply.github.com> Date: Wed, 25 Dec 2024 01:12:05 +0000 Subject: [PATCH 2/5] Address pre-commit errors --- .../test_system_health_fan_direction.py | 101 +++++++++--------- tests/system_health/test_system_health_led.py | 33 +++--- 2 files changed, 66 insertions(+), 68 deletions(-) diff --git a/tests/system_health/test_system_health_fan_direction.py b/tests/system_health/test_system_health_fan_direction.py index 4c597369548..389506075a9 100644 --- a/tests/system_health/test_system_health_fan_direction.py +++ b/tests/system_health/test_system_health_fan_direction.py @@ -1,6 +1,5 @@ import logging import re -import time import random import pytest from datetime import timedelta @@ -12,10 +11,7 @@ logger = logging.getLogger(__name__) -pytestmark = [ - pytest.mark.topology('any'), - pytest.mark.disable_loganalyzer -] +pytestmark = [pytest.mark.topology("any"), pytest.mark.disable_loganalyzer] daemon_name = "thermalctld" @@ -24,23 +20,25 @@ daemon_exited_str = "EXITED" FAN_DIR_ERROR_MSG_REX = ".*direction .* is not aligned with .* direction.*" -STATUS_LED_COLOR_MSG_REX = r'System status LED (.*)' -NOT_OK_REX = r'\bNot OK\b' +STATUS_LED_COLOR_MSG_REX = r"System status LED (.*)" +NOT_OK_REX = r"\bNot OK\b" -TIMEOUT = 60*2 +TIMEOUT = 60 * 2 BOOT_TIME_IN_SECONDS = 300 def is_booting_time_expired(duthost): uptime = duthost.get_uptime() - logging.info('uptime={}'.format(uptime)) + logging.info("uptime={}".format(uptime)) booting_time = timedelta(seconds=float(BOOT_TIME_IN_SECONDS)) return uptime > booting_time def check_thermalctld_pid(duthost, daemon_pid): - result_pid = duthost.shell("docker exec pmon bash -c 'ps -aux| grep thermalctld | head -n -2' | awk '{print $2}' ")['stdout_lines'] + result_pid = duthost.shell("docker exec pmon bash -c 'ps -aux| grep thermalctld | head -n -2' | awk '{print $2}' ")[ + "stdout_lines" + ] return len(result_pid) == 1 and int(result_pid[0]) == daemon_pid @@ -53,8 +51,10 @@ def check_expected_daemon_status(duthost, expected_daemon_status): else: # if the expected status matches, check extra conditions under different expected daemon status if daemon_status == daemon_running_str: - result_pid = duthost.shell("docker exec pmon bash -c 'ps -aux| grep thermalctld | head -n -2' | awk '{print $2}' ")['stdout_lines'] - return (len(result_pid) >= 2) + result_pid = duthost.shell( + "docker exec pmon bash -c 'ps -aux| grep thermalctld | head -n -2' | awk '{print $2}' " + )["stdout_lines"] + return len(result_pid) >= 2 return True @@ -64,17 +64,17 @@ def setup(duthosts, rand_one_dut_hostname): duthost = duthosts[rand_one_dut_hostname] daemon_en_status = check_pmon_daemon_enable_status(duthost, daemon_name) if daemon_en_status is False: - pytest.skip("{} is not enabled in {}".format(daemon_name, duthost.facts['platform'])) + pytest.skip("{} is not enabled in {}".format(daemon_name, duthost.facts["platform"])) data = collect_data(duthost) - if len(data['keys']) == 0: + if len(data["keys"]) == 0: pytest.skip("(ECMSG-feature) Fan direction not support on this dut") - select_key = random.sample(data['keys'], 1) - select_direction = data['data'][select_key[0]] + select_key = random.sample(data["keys"], 1) + select_direction = data["data"][select_key[0]] # check all fan direction is the same - for direction in data['data'].items(): + for direction in data["data"].items(): assert direction[1] == select_direction, "Not all fans spin the same way before testing." daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) @@ -98,16 +98,20 @@ def teardown_function(duthosts, rand_one_dut_hostname, setup): yield duthost.stop_pmon_daemon(daemon_name) - assert(wait_until(50, 10, 0, check_expected_daemon_status, duthost, daemon_stopped_str)), "The daemon thermalctld is not successfully stopped" + assert wait_until( + 50, 10, 0, check_expected_daemon_status, duthost, daemon_stopped_str + ), "The daemon thermalctld is not successfully stopped" duthost.start_pmon_daemon(daemon_name) logger.info("To make sure thermalctld two process are running.") - assert(wait_until(50, 10, 0, check_expected_daemon_status, duthost, daemon_running_str)), "The daemon thermalctld is not successfully restarted" + assert wait_until( + 50, 10, 0, check_expected_daemon_status, duthost, daemon_running_str + ), "The daemon thermalctld is not successfully restarted" if sys_led_color is not None: - assert wait_until(TIMEOUT, 10, 0, verify_sys_led_status, - duthost, sys_led_color), \ - "The color {} of the System Status LED is not successfully restored.".format(sys_led_color) + assert wait_until( + TIMEOUT, 10, 0, verify_sys_led_status, duthost, sys_led_color + ), "The color {} of the System Status LED is not successfully restored.".format(sys_led_color) def stop_thermalctld_child_processes(duthost, daemon_pid): @@ -120,7 +124,7 @@ def stop_thermalctld_child_processes(duthost, daemon_pid): """ logging.info("Stopping thermalctld child processes for PID: {}".format(daemon_pid)) - child_list = duthost.shell("docker exec pmon bash -c 'pgrep -P {}'".format(daemon_pid))['stdout_lines'] + child_list = duthost.shell("docker exec pmon bash -c 'pgrep -P {}'".format(daemon_pid))["stdout_lines"] cmd = "docker exec pmon bash -c 'kill -9 {}'" for pid_str in child_list: @@ -131,7 +135,9 @@ def stop_thermalctld_child_processes(duthost, daemon_pid): except Exception as e: logging.error("Failed to stop process with PID {}: {}".format(pid, str(e))) - assert(wait_until(30, 3, 0, check_thermalctld_pid, duthost, daemon_pid)), "Can't stop thermalctld child processes within expceted time." + assert wait_until( + 30, 3, 0, check_thermalctld_pid, duthost, daemon_pid + ), "Can't stop thermalctld child processes within expceted time." def collect_data(duthost): @@ -142,18 +148,18 @@ def collect_data(duthost): dev_keys : list of device name as keys in state db. dev_data : dict of fan direction (intake/exhaust) """ - keys = duthost.shell('sonic-db-cli STATE_DB KEYS "FAN_INFO|*"')['stdout_lines'] + keys = duthost.shell('sonic-db-cli STATE_DB KEYS "FAN_INFO|*"')["stdout_lines"] dev_keys = [] dev_data = {} for k in keys: - data = duthost.shell('sonic-db-cli STATE_DB HGET "{}" "direction"'.format(k))['stdout_lines'] - if 'N/A' not in data: + data = duthost.shell('sonic-db-cli STATE_DB HGET "{}" "direction"'.format(k))["stdout_lines"] + if "N/A" not in data: # skip fan direction is 'N/A'. dev_data[k] = data dev_keys.append(k) - return {'keys': dev_keys, 'data': dev_data} + return {"keys": dev_keys, "data": dev_data} def get_system_health_summary_output(duthost): @@ -217,7 +223,7 @@ def get_led_color_by_system_health_summary(duthost): def verify_sys_led_status(duthost, expected_value): - ''' + """ Verify the system LED status. Parameters: @@ -226,8 +232,7 @@ def verify_sys_led_status(duthost, expected_value): Returns: bool: True if the expected LED status is found in the current LED status; False otherwise. - ''' - + """ led_status = get_led_color_by_system_health_summary(duthost) logging.info("led_status{}, expected{}".format(led_status, expected_value)) @@ -246,33 +251,31 @@ def test_fan_direction_alarm(duthosts, rand_one_dut_hostname, setup): duthost = duthosts[rand_one_dut_hostname] data, led_dict, sys_led_color = setup - logging.info("data{}, led_dict{}, color{}".format(data, led_dict, sys_led_color )) + logging.info("data{}, led_dict{}, color{}".format(data, led_dict, sys_led_color)) - #S1.Select one running fan. - select_key = random.sample(data['keys'], 1) - select_direction = data['data'][select_key[0]] + # S1.Select one running fan. + select_key = random.sample(data["keys"], 1) + select_direction = data["data"][select_key[0]] logging.info("select_direction{}".format(select_direction)) - - #S2.Set the fan direction to another one to State DB. - change_direction = 'intake' if 'exhaust' in select_direction else 'exhaust' + # S2.Set the fan direction to another one to State DB. + change_direction = "intake" if "exhaust" in select_direction else "exhaust" duthost.shell('sonic-db-cli STATE_DB HSET "{}" "direction" "{}"'.format(select_key[0], change_direction)) - - - #S3.Check the system-health output as expected or not. + # S3.Check the system-health output as expected or not. # Contain the string match pattern FAN_DIR_ERROR_MSG_REX # Contain the string match led_color["fault"] defined in system_health_monitoring_config.json - assert(parse_system_health_summary(duthost, FAN_DIR_ERROR_MSG_REX)), "Missing error message about fans spin in different way" + assert parse_system_health_summary( + duthost, FAN_DIR_ERROR_MSG_REX + ), "Missing error message about fans spin in different way" if led_dict.get("fault", None) is not None: - assert wait_until(TIMEOUT, 10, 0, verify_sys_led_status, - duthost, led_dict["fault"]), \ - "The color {} of the System Status LED is not shown.".format(led_dict["fault"]) + assert wait_until( + TIMEOUT, 10, 0, verify_sys_led_status, duthost, led_dict["fault"] + ), "The color {} of the System Status LED is not shown.".format(led_dict["fault"]) - #S4.Restore the fan direction. + # S4.Restore the fan direction. duthost.shell('sonic-db-cli STATE_DB HSET "{}" "direction" "{}"'.format(select_key[0], select_direction[0])) - #S5.Check the system-health output error message disappear or not. - assert(not parse_system_health_summary(duthost, FAN_DIR_ERROR_MSG_REX)), "Shouldn't have error message." - + # S5.Check the system-health output error message disappear or not. + assert not parse_system_health_summary(duthost, FAN_DIR_ERROR_MSG_REX), "Shouldn't have error message." diff --git a/tests/system_health/test_system_health_led.py b/tests/system_health/test_system_health_led.py index 5aaf365027c..40b9906303f 100644 --- a/tests/system_health/test_system_health_led.py +++ b/tests/system_health/test_system_health_led.py @@ -1,8 +1,6 @@ import logging -import re import pytest -from tests.common.platform.daemon_utils import check_pmon_daemon_enable_status from tests.common.utilities import wait_until from tests.system_health.test_system_health import get_system_health_config @@ -12,15 +10,12 @@ logger = logging.getLogger(__name__) -pytestmark = [ - pytest.mark.topology('any'), - pytest.mark.disable_loganalyzer -] +pytestmark = [pytest.mark.topology("any"), pytest.mark.disable_loganalyzer] -STATUS_LED_COLOR_MSG_REX = r'System status LED (.*)' -NOT_OK_REX = r'\bNot OK\b' -TIMEOUT = 60*2 +STATUS_LED_COLOR_MSG_REX = r"System status LED (.*)" +NOT_OK_REX = r"\bNot OK\b" +TIMEOUT = 60 * 2 @pytest.fixture(scope="function") @@ -42,14 +37,14 @@ def teardown_function(duthosts, rand_one_dut_hostname, setup): yield logger.info("Restart pmon container") - stdout = duthost.shell("docker start pmon")['stdout_lines'] + duthost.shell("docker start pmon")["stdout_lines"] if sys_led_color is None: return - assert wait_until(TIMEOUT, 10, 0, verify_sys_led_status, - duthost, sys_led_color), \ - "The color {} of the System Status LED is not successfully restored.".format(sys_led_color) + assert wait_until( + TIMEOUT, 10, 0, verify_sys_led_status, duthost, sys_led_color + ), "The color {} of the System Status LED is not successfully restored.".format(sys_led_color) def test_system_health_led(duthosts, enum_rand_one_per_hwsku_hostname, setup): @@ -59,13 +54,13 @@ def test_system_health_led(duthosts, enum_rand_one_per_hwsku_hostname, setup): S2. Check System Status Led Status from show system-health summary output """ - #S1. Stop pmon container + # S1. Stop pmon container duthost = duthosts[enum_rand_one_per_hwsku_hostname] led_dict, sys_led_color = setup - stdout = duthost.shell("docker stop pmon")['stdout_lines'] + duthost.shell("docker stop pmon")["stdout_lines"] - #S2. Check System Status Led Status from show system-health summary output + # S2. Check System Status Led Status from show system-health summary output if led_dict.get("fault", None) is not None: - assert wait_until(TIMEOUT, 10, 0, verify_sys_led_status, - duthost, led_dict["fault"]), \ - "The color {} of the System Status LED is not shown.".format(led_dict["fault"]) + assert wait_until( + TIMEOUT, 10, 0, verify_sys_led_status, duthost, led_dict["fault"] + ), "The color {} of the System Status LED is not shown.".format(led_dict["fault"]) From 8ef324f933c2604e71bc9f30fb71a6bb412e47f1 Mon Sep 17 00:00:00 2001 From: ecsonic <49477291+PJHsieh@users.noreply.github.com> Date: Wed, 25 Dec 2024 08:34:22 +0000 Subject: [PATCH 3/5] Skip tests on VS platform --- .../conditional_mark/tests_mark_conditions.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index abd185a9a96..23501c90641 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -2192,6 +2192,18 @@ system_health/test_system_health.py::test_service_checker_with_process_exit: - "branch in ['internal-202012']" - "build_version.split('.')[1].isdigit() and int(build_version.split('.')[1]) <= 44" +system_health/test_system_health_fan_direction.py: + skip: + reason: "There is no table SYSTEM_HEALTH_INFO in STATE_DB on kvm testbed, skip in PR testing" + conditions: + - "asic_type in ['vs']" + +system_health/test_system_health_led.py: + skip: + reason: "There is no table SYSTEM_HEALTH_INFO in STATE_DB on kvm testbed, skip in PR testing" + conditions: + - "asic_type in ['vs']" + ####################################### ##### tacacs ##### ####################################### From f96b186670bff590bc2bd4a5824b1910af5e221c Mon Sep 17 00:00:00 2001 From: ecsonic <49477291+PJHsieh@users.noreply.github.com> Date: Wed, 25 Dec 2024 08:43:44 +0000 Subject: [PATCH 4/5] Add test cases into onboarding-t0 and onboarding-t1 PR checker and let the pipeline to check. --- .azure-pipelines/pr_test_scripts.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.azure-pipelines/pr_test_scripts.yaml b/.azure-pipelines/pr_test_scripts.yaml index 1874727ba7b..312cf189418 100644 --- a/.azure-pipelines/pr_test_scripts.yaml +++ b/.azure-pipelines/pr_test_scripts.yaml @@ -481,12 +481,16 @@ onboarding_t0: # Flaky, we will triage and fix it later, move to onboarding to unblock pr check - dhcp_relay/test_dhcp_relay_stress.py - pc/test_lag_member_forwarding.py + - system_health/test_system_health_fan_direction.py + - system_health/test_system_health_led.py onboarding_t1: - pc/test_lag_member_forwarding.py - pfcwd/test_pfcwd_all_port_storm.py - pfcwd/test_pfcwd_function.py - pfcwd/test_pfcwd_timer_accuracy.py + - system_health/test_system_health_fan_direction.py + - system_health/test_system_health_led.py specific_param: t0-sonic: From 8434451e22225017985f038eb3f348c527234dd5 Mon Sep 17 00:00:00 2001 From: ecsonic <49477291+PJHsieh@users.noreply.github.com> Date: Wed, 25 Dec 2024 09:04:22 +0000 Subject: [PATCH 5/5] Revise skip reason --- .../plugins/conditional_mark/tests_mark_conditions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index 23501c90641..d956c5ceabe 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -2194,13 +2194,13 @@ system_health/test_system_health.py::test_service_checker_with_process_exit: system_health/test_system_health_fan_direction.py: skip: - reason: "There is no table SYSTEM_HEALTH_INFO in STATE_DB on kvm testbed, skip in PR testing" + reason: "Not support on kvm testbed, skip in PR testing" conditions: - "asic_type in ['vs']" system_health/test_system_health_led.py: skip: - reason: "There is no table SYSTEM_HEALTH_INFO in STATE_DB on kvm testbed, skip in PR testing" + reason: "Not support on kvm testbed, skip in PR testing" conditions: - "asic_type in ['vs']"