From 89c5b5564778c3ddaf31448d806e2d35881f0d91 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 19 Oct 2023 12:29:33 -0700 Subject: [PATCH 1/5] Removes requiring an *-only option --- CIME/Tools/bless_test_results | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index cb6bd2f972a..82c92c6cfad 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -104,7 +104,7 @@ OR def create_bless_options(parser): bless_group = parser.add_argument_group("Bless options") - mutual_bless_group = bless_group.add_mutually_exclusive_group(required=True) + mutual_bless_group = bless_group.add_mutually_exclusive_group() mutual_bless_group.add_argument( "-n", "--namelists-only", action="store_true", help="Only analyze namelists." From 01b7547481d5d9738703a6dd20a5296d3d45beb4 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 19 Oct 2023 17:59:03 -0700 Subject: [PATCH 2/5] Fixes default bless behavior --- CIME/bless_test_results.py | 104 +++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 0f09b91eb0b..181d449e59f 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -183,7 +183,6 @@ def bless_history(test_name, case, baseline_name, baseline_root, report_only, fo return True, None -############################################################################### def bless_test_results( baseline_name, baseline_root, @@ -203,7 +202,8 @@ def bless_test_results( new_test_id=None, **_, # Capture all for extra ): - ############################################################################### + bless_all = not (namelists_only | hist_only | tput_only | mem_only) + test_status_files = get_test_status_files(test_root, compiler, test_id=test_id) # auto-adjust test-id if multiple rounds of tests were matched @@ -255,18 +255,15 @@ def bless_test_results( if bless_tests in [[], None] or CIME.utils.match_any( test_name, bless_tests_counts ): - ts_kwargs = dict(ignore_namelists=True, ignore_memleak=True) - - if tput_only: - ts_kwargs["check_throughput"] = True - - if mem_only: - ts_kwargs["check_memory"] = True - - overall_result, phase = ts.get_overall_test_status(**ts_kwargs) + overall_result, phase = ts.get_overall_test_status( + ignore_namelists=True, + ignore_memleak=True, + check_throughput=True, + check_memory=True, + ) # See if we need to bless namelist - if not hist_only: + if namelists_only or bless_all: if no_skip_pass: nl_bless = True else: @@ -274,41 +271,26 @@ def bless_test_results( else: nl_bless = False - # See if we need to bless baselines - if not namelists_only and not build_only: - run_result = ts.get_status(RUN_PHASE) - if run_result is None: - broken_blesses.append((test_name, "no run phase")) - logger.warning( - "Test '{}' did not make it to run phase".format(test_name) - ) - hist_bless = False - elif run_result != TEST_PASS_STATUS: - broken_blesses.append((test_name, "run phase did not pass")) - logger.warning( - "Test '{}' run phase did not pass, not safe to bless, test status = {}".format( - test_name, ts.phase_statuses_dump() - ) - ) - hist_bless = False - elif overall_result == TEST_FAIL_STATUS: - broken_blesses.append((test_name, "test did not pass")) - logger.warning( - "Test '{}' did not pass due to phase {}, not safe to bless, test status = {}".format( - test_name, phase, ts.phase_statuses_dump() - ) - ) - hist_bless = False + hist_bless, tput_bless, mem_bless = [False] * 3 - elif no_skip_pass: - hist_bless = True - else: - hist_bless = ts.get_status(BASELINE_PHASE) != TEST_PASS_STATUS - else: - hist_bless = False + # Skip if test is build only i.e. testopts contains "B" + if not build_only: + allowed = bless_allowed( + test_name, ts, broken_blesses, overall_result, no_skip_pass, phase + ) + + # See if we need to bless baselines + if hist_only or bless_all: + hist_bless = allowed + + if tput_only or bless_all: + tput_bless = allowed + + if mem_only or bless_all: + mem_bless = allowed # Now, do the bless - if not nl_bless and not hist_bless and not tput_only and not mem_only: + if not nl_bless and not hist_bless and not tput_bless and not mem_bless: logger.info( "Nothing to bless for test: {}, overall status: {}".format( test_name, overall_result @@ -441,3 +423,37 @@ def bless_test_results( success = False return success + + +def bless_allowed(test_name, ts, broken_blesses, overall_result, no_skip_pass, phase): + allow_bless = False + + run_result = ts.get_status(RUN_PHASE) + + if run_result is None: + broken_blesses.append((test_name, "no run phase")) + logger.warning("Test '{}' did not make it to run phase".format(test_name)) + allow_bless = False + elif run_result != TEST_PASS_STATUS: + broken_blesses.append((test_name, "run phase did not pass")) + logger.warning( + "Test '{}' run phase did not pass, not safe to bless, test status = {}".format( + test_name, ts.phase_statuses_dump() + ) + ) + allow_bless = False + elif overall_result == TEST_FAIL_STATUS: + broken_blesses.append((test_name, "test did not pass")) + logger.warning( + "Test '{}' did not pass due to phase {}, not safe to bless, test status = {}".format( + test_name, phase, ts.phase_statuses_dump() + ) + ) + allow_bless = False + + elif no_skip_pass: + allow_bless = True + else: + allow_bless = ts.get_status(BASELINE_PHASE) != TEST_PASS_STATUS + + return allow_bless From df9c7202089b23dc21b30f46b46f9487e7fe8d07 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 19 Oct 2023 18:52:06 -0700 Subject: [PATCH 3/5] Adds feature to exclude tests --- CIME/Tools/bless_test_results | 2 + CIME/bless_test_results.py | 267 ++++++++++++++++++---------------- 2 files changed, 141 insertions(+), 128 deletions(-) diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index 82c92c6cfad..d2bc707d3c2 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -90,6 +90,8 @@ OR "\ncan follow either the config_pes.xml or the env_mach_pes.xml format.", ) + parser.add_argument("--exclude", nargs="*", help="Exclude tests") + parser.add_argument( "bless_tests", nargs="*", diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 181d449e59f..a22e7ed9519 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -5,6 +5,7 @@ get_scripts_root, EnvironmentContext, parse_test_name, + match_any, ) from CIME.config import Config from CIME.test_status import * @@ -200,6 +201,7 @@ def bless_test_results( no_skip_pass=False, new_test_root=None, new_test_id=None, + exclude=None, **_, # Capture all for extra ): bless_all = not (namelists_only | hist_only | tput_only | mem_only) @@ -220,10 +222,14 @@ def bless_test_results( most_recent = sorted(timestamps)[-1] logger.info("Matched test batch is {}".format(most_recent)) - bless_tests_counts = None + bless_tests_counts = [] if bless_tests: bless_tests_counts = dict([(bless_test, 0) for bless_test in bless_tests]) + # compile excludes into single regex + if exclude is not None: + exclude = re.compile("|".join([f"({x})" for x in exclude])) + broken_blesses = [] for test_status_file in test_status_files: if not most_recent in test_status_file: @@ -239,7 +245,7 @@ def bless_test_results( if test_name is None: case_dir = os.path.basename(test_dir) test_name = CIME.utils.normalize_case_id(case_dir) - if not bless_tests or CIME.utils.match_any(test_name, bless_tests_counts): + if not bless_tests or match_any(test_name, bless_tests_counts): broken_blesses.append( ( "unknown", @@ -252,154 +258,159 @@ def bless_test_results( else: continue - if bless_tests in [[], None] or CIME.utils.match_any( - test_name, bless_tests_counts - ): - overall_result, phase = ts.get_overall_test_status( - ignore_namelists=True, - ignore_memleak=True, - check_throughput=True, - check_memory=True, - ) + # Must pass tests to continue + has_no_tests = bless_tests in [[], None] + match_test_name = match_any(test_name, bless_tests_counts) + excluded = exclude.match(test_name) - # See if we need to bless namelist - if namelists_only or bless_all: - if no_skip_pass: - nl_bless = True - else: - nl_bless = ts.get_status(NAMELIST_PHASE) != TEST_PASS_STATUS + if (not has_no_tests and not match_test_name) or excluded: + logger.info("Skipping {!r}".format(test_name)) + + continue + + overall_result, phase = ts.get_overall_test_status( + ignore_namelists=True, + ignore_memleak=True, + check_throughput=True, + check_memory=True, + ) + + # See if we need to bless namelist + if namelists_only or bless_all: + if no_skip_pass: + nl_bless = True else: - nl_bless = False + nl_bless = ts.get_status(NAMELIST_PHASE) != TEST_PASS_STATUS + else: + nl_bless = False - hist_bless, tput_bless, mem_bless = [False] * 3 + hist_bless, tput_bless, mem_bless = [False] * 3 - # Skip if test is build only i.e. testopts contains "B" - if not build_only: - allowed = bless_allowed( - test_name, ts, broken_blesses, overall_result, no_skip_pass, phase - ) + # Skip if test is build only i.e. testopts contains "B" + if not build_only: + allowed = bless_allowed( + test_name, ts, broken_blesses, overall_result, no_skip_pass, phase + ) - # See if we need to bless baselines - if hist_only or bless_all: - hist_bless = allowed + # See if we need to bless baselines + if hist_only or bless_all: + hist_bless = allowed - if tput_only or bless_all: - tput_bless = allowed + if tput_only or bless_all: + tput_bless = allowed - if mem_only or bless_all: - mem_bless = allowed + if mem_only or bless_all: + mem_bless = allowed - # Now, do the bless - if not nl_bless and not hist_bless and not tput_bless and not mem_bless: - logger.info( - "Nothing to bless for test: {}, overall status: {}".format( - test_name, overall_result - ) + # Now, do the bless + if not nl_bless and not hist_bless and not tput_bless and not mem_bless: + logger.info( + "Nothing to bless for test: {}, overall status: {}".format( + test_name, overall_result ) - else: - logger.info( - "###############################################################################" - ) - logger.info( - "Blessing results for test: {}, most recent result: {}".format( - test_name, overall_result - ) - ) - logger.info("Case dir: {}".format(test_dir)) - logger.info( - "###############################################################################" + ) + else: + logger.info( + "###############################################################################" + ) + logger.info( + "Blessing results for test: {}, most recent result: {}".format( + test_name, overall_result ) - if not force: - time.sleep(2) - - with Case(test_dir) as case: - # Resolve baseline_name and baseline_root - if baseline_name is None: - baseline_name_resolved = case.get_value("BASELINE_NAME_CMP") - if not baseline_name_resolved: - baseline_name_resolved = CIME.utils.get_current_branch( - repo=CIME.utils.get_cime_root() - ) - else: - baseline_name_resolved = baseline_name - - if baseline_root is None: - baseline_root_resolved = case.get_value("BASELINE_ROOT") - else: - baseline_root_resolved = baseline_root - - if baseline_name_resolved is None: - broken_blesses.append( - (test_name, "Could not determine baseline name") + ) + logger.info("Case dir: {}".format(test_dir)) + logger.info( + "###############################################################################" + ) + if not force: + time.sleep(2) + + with Case(test_dir) as case: + # Resolve baseline_name and baseline_root + if baseline_name is None: + baseline_name_resolved = case.get_value("BASELINE_NAME_CMP") + if not baseline_name_resolved: + baseline_name_resolved = CIME.utils.get_current_branch( + repo=CIME.utils.get_cime_root() ) - continue + else: + baseline_name_resolved = baseline_name - if baseline_root_resolved is None: - broken_blesses.append( - (test_name, "Could not determine baseline root") - ) - continue + if baseline_root is None: + baseline_root_resolved = case.get_value("BASELINE_ROOT") + else: + baseline_root_resolved = baseline_root - # Bless namelists - if nl_bless: - success, reason = bless_namelists( + if baseline_name_resolved is None: + broken_blesses.append( + (test_name, "Could not determine baseline name") + ) + continue + + if baseline_root_resolved is None: + broken_blesses.append( + (test_name, "Could not determine baseline root") + ) + continue + + # Bless namelists + if nl_bless: + success, reason = bless_namelists( + test_name, + report_only, + force, + pesfile, + baseline_name_resolved, + baseline_root_resolved, + new_test_root=new_test_root, + new_test_id=new_test_id, + ) + if not success: + broken_blesses.append((test_name, reason)) + + # Bless hist files + if hist_bless: + if "HOMME" in test_name: + success = False + reason = "HOMME tests cannot be blessed with bless_for_tests" + else: + success, reason = bless_history( test_name, - report_only, - force, - pesfile, - baseline_name_resolved, - baseline_root_resolved, - new_test_root=new_test_root, - new_test_id=new_test_id, - ) - if not success: - broken_blesses.append((test_name, reason)) - - # Bless hist files - if hist_bless: - if "HOMME" in test_name: - success = False - reason = ( - "HOMME tests cannot be blessed with bless_for_tests" - ) - else: - success, reason = bless_history( - test_name, - case, - baseline_name_resolved, - baseline_root_resolved, - report_only, - force, - ) - - if not success: - broken_blesses.append((test_name, reason)) - - if tput_only: - success, reason = bless_throughput( case, - test_name, - baseline_root_resolved, baseline_name_resolved, + baseline_root_resolved, report_only, force, ) - if not success: - broken_blesses.append((test_name, reason)) + if not success: + broken_blesses.append((test_name, reason)) + + if tput_only: + success, reason = bless_throughput( + case, + test_name, + baseline_root_resolved, + baseline_name_resolved, + report_only, + force, + ) - if mem_only: - success, reason = bless_memory( - case, - test_name, - baseline_root_resolved, - baseline_name_resolved, - report_only, - force, - ) + if not success: + broken_blesses.append((test_name, reason)) + + if mem_only: + success, reason = bless_memory( + case, + test_name, + baseline_root_resolved, + baseline_name_resolved, + report_only, + force, + ) - if not success: - broken_blesses.append((test_name, reason)) + if not success: + broken_blesses.append((test_name, reason)) # Emit a warning if items in bless_tests did not match anything if bless_tests: From 78f15ddf3bb522ee39b34216ff027443d72f910d Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 23 Oct 2023 09:36:26 -0700 Subject: [PATCH 4/5] Fixes none value for exclude regex --- CIME/bless_test_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index a22e7ed9519..396a9a0e1c6 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -261,7 +261,7 @@ def bless_test_results( # Must pass tests to continue has_no_tests = bless_tests in [[], None] match_test_name = match_any(test_name, bless_tests_counts) - excluded = exclude.match(test_name) + excluded = exclude.match(test_name) if exclude else False if (not has_no_tests and not match_test_name) or excluded: logger.info("Skipping {!r}".format(test_name)) From 9ffded57f351d1570c6dd5721d9fa904c0198c1e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 23 Oct 2023 09:45:34 -0700 Subject: [PATCH 5/5] Fixes function name --- CIME/bless_test_results.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 396a9a0e1c6..59488fc9450 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -288,19 +288,19 @@ def bless_test_results( # Skip if test is build only i.e. testopts contains "B" if not build_only: - allowed = bless_allowed( + bless_needed = is_bless_needed( test_name, ts, broken_blesses, overall_result, no_skip_pass, phase ) # See if we need to bless baselines if hist_only or bless_all: - hist_bless = allowed + hist_bless = bless_needed if tput_only or bless_all: - tput_bless = allowed + tput_bless = bless_needed if mem_only or bless_all: - mem_bless = allowed + mem_bless = bless_needed # Now, do the bless if not nl_bless and not hist_bless and not tput_bless and not mem_bless: @@ -436,15 +436,15 @@ def bless_test_results( return success -def bless_allowed(test_name, ts, broken_blesses, overall_result, no_skip_pass, phase): - allow_bless = False +def is_bless_needed(test_name, ts, broken_blesses, overall_result, no_skip_pass, phase): + needed = False run_result = ts.get_status(RUN_PHASE) if run_result is None: broken_blesses.append((test_name, "no run phase")) logger.warning("Test '{}' did not make it to run phase".format(test_name)) - allow_bless = False + needed = False elif run_result != TEST_PASS_STATUS: broken_blesses.append((test_name, "run phase did not pass")) logger.warning( @@ -452,7 +452,7 @@ def bless_allowed(test_name, ts, broken_blesses, overall_result, no_skip_pass, p test_name, ts.phase_statuses_dump() ) ) - allow_bless = False + needed = False elif overall_result == TEST_FAIL_STATUS: broken_blesses.append((test_name, "test did not pass")) logger.warning( @@ -460,11 +460,11 @@ def bless_allowed(test_name, ts, broken_blesses, overall_result, no_skip_pass, p test_name, phase, ts.phase_statuses_dump() ) ) - allow_bless = False + needed = False elif no_skip_pass: - allow_bless = True + needed = True else: - allow_bless = ts.get_status(BASELINE_PHASE) != TEST_PASS_STATUS + needed = ts.get_status(BASELINE_PHASE) != TEST_PASS_STATUS - return allow_bless + return needed