From bef6909b2ce5f68a19babe9746acd8f8daa6aeb3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 9 Sep 2024 10:47:14 -0600 Subject: [PATCH 1/4] make the hidden status of batch jobs xml dependent - currently hardcoded for case.st_archive only --- CIME/Tools/xmlchange | 1 - CIME/XML/env_batch.py | 13 ++++++++----- CIME/data/config/xml_schemas/config_workflow.xsd | 2 ++ CIME/utils.py | 7 +++++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 46debd906a2..4101f9b5dba 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -55,7 +55,6 @@ from standard_script_setup import * from CIME.utils import ( expect, convert_to_type, - get_batch_script_for_job, Timeout, ) from CIME.status import append_case_status diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index b444a29333a..266f3d1e150 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -32,6 +32,7 @@ def __init__(self, case_root=None, infile="env_batch.xml", read_only=False): initialize an object interface to file env_batch.xml in the case directory """ self._batchtype = None + self._hidden_batch_script = {} # This arbitrary setting should always be overwritten self._default_walltime = "00:20:00" schema = os.path.join(utils.get_schema_path(), "env_batch.xsd") @@ -257,7 +258,9 @@ def make_batch_script(self, input_template, job, case, outfile=None): subgroup=job, overrides=overrides, ) - output_name = get_batch_script_for_job(job) if outfile is None else outfile + env_workflow = case.get_env("workflow") + self._hidden_batch_script[job] = env_workflow.get_value("hidden", subgroup=job) + output_name = get_batch_script_for_job(job, hidden=self._hidden_batch_script[job]) if outfile is None else outfile logger.info("Creating file {}".format(output_name)) with open(output_name, "w") as fd: fd.write(output_text) @@ -745,7 +748,7 @@ def submit_jobs( alljobs = [ j for j in alljobs - if os.path.isfile(os.path.join(self._caseroot, get_batch_script_for_job(j))) + if os.path.isfile(os.path.join(self._caseroot, get_batch_script_for_job(j, hidden=self._hidden_batch_script[j]))) ] startindex = 0 @@ -1071,7 +1074,7 @@ def _submit_single_job( batchsubmit, submitargs, batchredirect, - get_batch_script_for_job(job), + get_batch_script_for_job(job, hidden=self._hidden_batch_script[job]), ) elif batch_env_flag: sequence = ( @@ -1079,14 +1082,14 @@ def _submit_single_job( submitargs, run_args, batchredirect, - os.path.join(self._caseroot, get_batch_script_for_job(job)), + os.path.join(self._caseroot, get_batch_script_for_job(job, hidden=self._hidden_batch_script[job])), ) else: sequence = ( batchsubmit, submitargs, batchredirect, - os.path.join(self._caseroot, get_batch_script_for_job(job)), + os.path.join(self._caseroot, get_batch_script_for_job(job, hidden=self._hidden_batch_script[job])), run_args, ) diff --git a/CIME/data/config/xml_schemas/config_workflow.xsd b/CIME/data/config/xml_schemas/config_workflow.xsd index 14a82586c08..6a10b167347 100644 --- a/CIME/data/config/xml_schemas/config_workflow.xsd +++ b/CIME/data/config/xml_schemas/config_workflow.xsd @@ -13,6 +13,7 @@ + @@ -57,6 +58,7 @@ + diff --git a/CIME/utils.py b/CIME/utils.py index 0973647823b..1b7337d1669 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -2530,8 +2530,11 @@ def run_bld_cmd_ensure_logging(cmd, arg_logger, from_dir=None, timeout=None): expect(stat == 0, filter_unicode(errput)) -def get_batch_script_for_job(job): - return job if "st_archive" in job else "." + job +def get_batch_script_for_job(job, hidden="True"): + # this if statement is for backward compatibility + if job == "case.st_archive" and not hidden: + hidden = "False" + return "." + job if not hidden or hidden == "True" else job def string_in_list(_string, _list): From 47ea873d8db28a479a039175c703a533cdab0ff9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 9 Sep 2024 13:35:21 -0600 Subject: [PATCH 2/4] fix underscore issue --- CIME/XML/env_batch.py | 58 +++++++++++++++++++++++++++++--- CIME/tests/test_sys_cime_case.py | 3 +- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index 266f3d1e150..655f33ddee2 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -260,7 +260,18 @@ def make_batch_script(self, input_template, job, case, outfile=None): ) env_workflow = case.get_env("workflow") self._hidden_batch_script[job] = env_workflow.get_value("hidden", subgroup=job) - output_name = get_batch_script_for_job(job, hidden=self._hidden_batch_script[job]) if outfile is None else outfile + output_name = ( + get_batch_script_for_job( + job, + hidden=( + self._hidden_batch_script[job] + if job in self._hidden_batch_script + else None + ), + ) + if outfile is None + else outfile + ) logger.info("Creating file {}".format(output_name)) with open(output_name, "w") as fd: fd.write(output_text) @@ -748,7 +759,17 @@ def submit_jobs( alljobs = [ j for j in alljobs - if os.path.isfile(os.path.join(self._caseroot, get_batch_script_for_job(j, hidden=self._hidden_batch_script[j]))) + if os.path.isfile( + os.path.join( + self._caseroot, + get_batch_script_for_job( + j, + hidden=self._hidden_batch_script[j] + if j in self._hidden_batch_script + else None, + ), + ) + ) ] startindex = 0 @@ -1074,7 +1095,14 @@ def _submit_single_job( batchsubmit, submitargs, batchredirect, - get_batch_script_for_job(job, hidden=self._hidden_batch_script[job]), + get_batch_script_for_job( + job, + hidden=( + self._hidden_batch_script[job] + if job in self._hidden_batch_script + else None + ), + ), ) elif batch_env_flag: sequence = ( @@ -1082,14 +1110,34 @@ def _submit_single_job( submitargs, run_args, batchredirect, - os.path.join(self._caseroot, get_batch_script_for_job(job, hidden=self._hidden_batch_script[job])), + os.path.join( + self._caseroot, + get_batch_script_for_job( + job, + hidden=( + self._hidden_batch_script[job] + if job in self._hidden_batch_script + else None + ), + ), + ), ) else: sequence = ( batchsubmit, submitargs, batchredirect, - os.path.join(self._caseroot, get_batch_script_for_job(job, hidden=self._hidden_batch_script[job])), + os.path.join( + self._caseroot, + get_batch_script_for_job( + job, + hidden=( + self._hidden_batch_script[job] + if job in self._hidden_batch_script + else None + ), + ), + ), run_args, ) diff --git a/CIME/tests/test_sys_cime_case.py b/CIME/tests/test_sys_cime_case.py index 4b226ff3b46..d07456eb570 100644 --- a/CIME/tests/test_sys_cime_case.py +++ b/CIME/tests/test_sys_cime_case.py @@ -731,7 +731,8 @@ def test_self_build_cprnc(self): ) self.run_cmd_assert_result( - "./xmlchange CCSM_CPRNC=this_is_a_broken_cprnc", from_dir=casedir + "./xmlchange CCSM_CPRNC=this_is_a_broken_cprnc --file env_test.xml", + from_dir=casedir, ) self.run_cmd_assert_result("./case.build", from_dir=casedir) self.run_cmd_assert_result("./case.submit", from_dir=casedir) From ad8d32fd45be787af35629ee1dc0781b59f0c80a Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 10 Sep 2024 11:25:02 -0600 Subject: [PATCH 3/4] make hidden a logical --- CIME/XML/env_batch.py | 8 +++++++- CIME/data/config/xml_schemas/config_workflow.xsd | 2 +- CIME/tests/test_sys_create_newcase.py | 2 +- CIME/utils.py | 8 ++++---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index 655f33ddee2..69a6560ae49 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -259,7 +259,13 @@ def make_batch_script(self, input_template, job, case, outfile=None): overrides=overrides, ) env_workflow = case.get_env("workflow") - self._hidden_batch_script[job] = env_workflow.get_value("hidden", subgroup=job) + + hidden = env_workflow.get_value("hidden", subgroup=job) + if hidden is None or hidden == "True" or hidden == "true": + self._hidden_batch_script[job] = True + else: + self._hidden_batch_script[job] = False + output_name = ( get_batch_script_for_job( job, diff --git a/CIME/data/config/xml_schemas/config_workflow.xsd b/CIME/data/config/xml_schemas/config_workflow.xsd index 6a10b167347..5b09913a4b6 100644 --- a/CIME/data/config/xml_schemas/config_workflow.xsd +++ b/CIME/data/config/xml_schemas/config_workflow.xsd @@ -13,7 +13,7 @@ - + diff --git a/CIME/tests/test_sys_create_newcase.py b/CIME/tests/test_sys_create_newcase.py index b99ca4f10c4..1be636aff36 100644 --- a/CIME/tests/test_sys_create_newcase.py +++ b/CIME/tests/test_sys_create_newcase.py @@ -74,7 +74,7 @@ def test_a_createnewcase(self): # on systems (like github workflow) that do not have batch, set this for the next test if batch_system == "none": self.run_cmd_assert_result( - './xmlchange --subgroup case.run BATCH_COMMAND_FLAGS="-q \$JOB_QUEUE"', + r'./xmlchange --subgroup case.run BATCH_COMMAND_FLAGS="-q \$JOB_QUEUE"', from_dir=testdir, ) diff --git a/CIME/utils.py b/CIME/utils.py index 1b7337d1669..85fa30017b0 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -2530,11 +2530,11 @@ def run_bld_cmd_ensure_logging(cmd, arg_logger, from_dir=None, timeout=None): expect(stat == 0, filter_unicode(errput)) -def get_batch_script_for_job(job, hidden="True"): +def get_batch_script_for_job(job, hidden=None): # this if statement is for backward compatibility - if job == "case.st_archive" and not hidden: - hidden = "False" - return "." + job if not hidden or hidden == "True" else job + if hidden is None: + hidden = job != "case.st_archive" + return "." + job if hidden else job def string_in_list(_string, _list): From a86c8f3e90d84799cc4bd37e8903f3e2dee5c226 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 10 Sep 2024 13:30:11 -0600 Subject: [PATCH 4/4] case.st_archive is special for backward compatibility --- CIME/XML/env_batch.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index 69a6560ae49..fddb7dbc16f 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -261,7 +261,12 @@ def make_batch_script(self, input_template, job, case, outfile=None): env_workflow = case.get_env("workflow") hidden = env_workflow.get_value("hidden", subgroup=job) - if hidden is None or hidden == "True" or hidden == "true": + # case.st_archive is not hidden for backward compatibility + if ( + (job != "case.st_archive" and hidden is None) + or hidden == "True" + or hidden == "true" + ): self._hidden_batch_script[job] = True else: self._hidden_batch_script[job] = False