Merge pull request #4652 from jedwards4b/maint-5.6_vista_port

Maint 5.6 vista port
ESMCI · Aug 21, 2024 · 01194c2 · 01194c2
2 parents 1d99a89 + a9fcdad
commit 01194c2
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 11 deletions.
diff --git a/.github/workflows/srt.yml b/.github/workflows/srt.yml
@@ -94,6 +94,6 @@ jobs:
 
 #     the following can be used by developers to login to the github server in case of errors
 #     see https://github.com/marketplace/actions/debugging-with-tmate for further details
-      - name: Setup tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
+#      - name: Setup tmate session
+#        if: ${{ failure() }}
+#        uses: mxschmitt/action-tmate@v3
diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml
@@ -706,6 +706,20 @@
     </queues>
   </batch_system>
 
+  <batch_system MACH="vista" type="slurm" >
+    <batch_submit>ssh login1.vista.tacc.utexas.edu sbatch  </batch_submit>
+    <jobid_pattern>Submitted batch job (\d+)$</jobid_pattern>
+    <submit_args>
+      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
+      <arg flag="-p" name="$JOB_QUEUE"/>
+      <arg flag="--account" name="$PROJECT"/>
+    </submit_args>
+    <queues>
+      <queue walltimemax="2:00:00" nodemin="1" nodemax="32" default="true">gg</queue>
+      <queue walltimemax="2:00:00" nodemin="1" nodemax="29">gg-4k</queue>
+    </queues>
+  </batch_system>
+
   <batch_system MACH="zeus" type="lsf">
     <batch_env>-env</batch_env>
     <submit_args>

diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml
@@ -311,7 +311,7 @@ using a fortran linker.
   <SFC> nagfor </SFC>
 </compiler>
 
-<compiler COMPILER="pgi">
+<compiler COMPILER="nvhpc">
   <CFLAGS>
     <base> -gopt  -time </base>
     <append compile_threaded="true"> -mp </append>
@@ -615,19 +615,23 @@ using a fortran linker.
   </CFLAGS>
 </compiler>
 
-<compiler MACH="bluewaters" COMPILER="pgi">
+<compiler MACH="vista" COMPILER="nvhpc">
   <CFLAGS>
     <append DEBUG="FALSE"> -O2 </append>
     <append> -nofma </append>
   </CFLAGS>
   <CXX_LIBS>
-    <base> -lmpichf90_pgi $ENV{PGI_PATH}/linux86-64/$ENV{PGI_VERSION}/lib/f90main.o </base>
+    <base>  </base>
   </CXX_LIBS>
   <FFLAGS>
     <append DEBUG="FALSE"> -O2 </append>
     <append> -nofma </append>
   </FFLAGS>
   <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
+  <SLIBS>
+    <append> -Wl,-rpath,$(NETCDF_PATH)/lib -L$ENV{NETCDF_PATH}/lib -lnetcdff -lnetcdf </append>
+    <append MPILIB="openmpi"> -Wl,-rpath,$(PNETCDF_PATH)/lib -L$ENV{PNETCDF_PATH}/lib -lpnetcdf</append>
+  </SLIBS>
 </compiler>
 
 

diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml
@@ -2890,6 +2890,55 @@ This allows using a different mpirun command to launch unit tests
     </module_system>
   </machine>
 
+  <machine MACH="vista">
+    <DESC>Grace Hopper ARM TACC , batch system is SLURM</DESC>
+    <NODENAME_REGEX>.*.vista.tacc.utexas.edu</NODENAME_REGEX>
+    <OS>LINUX</OS>
+    <COMPILERS>nvhpc,gnu</COMPILERS>
+    <MPILIBS>openmpi</MPILIBS>
+    <CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>/work2/02503/edwardsj/CESM/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>/work2/02503/edwardsj/CESM/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>/work2/02503/edwardsj/CESM/cesm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>/work2/02503/edwardsj/CESM/cime/tools/cprnc/cprnc.vista</CCSM_CPRNC>
+    <GMAKE_J>4</GMAKE_J>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
+    <SUPPORTED_BY></SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>144</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>144</MAX_MPITASKS_PER_NODE>
+    <mpirun mpilib="openmpi">
+      <executable>mpirun</executable>
+      <arguments>
+	<arg name="num_tasks"> -np {{ total_tasks }}</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module">
+      <init_path lang="perl">/opt/apps/lmod/lmod/init/perl</init_path>
+      <init_path lang="python">/opt/apps/lmod/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="sh">/opt/apps/lmod/lmod/init/sh</init_path>
+      <init_path lang="csh">/opt/apps/lmod/lmod/init/csh</init_path>
+      <cmd_path lang="perl">/opt/apps/lmod/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">/opt/apps/lmod/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+	<command name="purge"></command>
+	<command name="load">TACC</command>
+	<command name="load">nvidia</command>
+      </modules>
+      <modules mpilib="openmpi">
+	<command name="load">openmpi/5.0.3</command>
+      </modules>
+    </module_system>
+    <environment_variables>
+      <env name="OMP_STACKSIZE">256M</env>
+      <env name="NETCDF_PATH">/scratch/00422/cazes/vista_netcdf_install/netcdf_4.9.2_nvhpc/</env>
+      <env name="PNETCDF_PATH">/scratch/00422/cazes/vista_netcdf_install/pnetcdf_1.12.3_nvhpc/</env>
+    </environment_variables>
+  </machine>
+
+
   <machine MACH="zeus">
     <DESC> CMCC Lenovo ThinkSystem SD530, os is Linux, 36 pes/node, batch system is LSF</DESC>
     <NODENAME_REGEX>(login[1,2]-ib|n[0-9][0-9][0-9]-ib)</NODENAME_REGEX>

diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py
@@ -603,7 +603,7 @@ def _submit_single_job(self, case, job, dep_jobs=None, allow_fail=False,
         if batch_system is None or batch_system == "none" or no_batch:
             logger.info("Starting job script {}".format(job))
             function_name = job.replace(".", "_")
-            job_name = "."+job
+            job_name = os.path.join(self._caseroot,job)
             if not dry_run:
                 args = self._build_run_args(job, True, skip_pnl=skip_pnl, set_continue_run=resubmit_immediate,
                                             submit_resubmits=not resubmit_immediate)
@@ -688,16 +688,16 @@ def _submit_single_job(self, case, job, dep_jobs=None, allow_fail=False,
         if batch_system == 'lsf' and batch_env_flag == 'none':
             sequence = (run_args, batchsubmit, submitargs, batchredirect, get_batch_script_for_job(job))
         elif batch_env_flag:
-            sequence = (batchsubmit, submitargs, run_args, batchredirect, get_batch_script_for_job(job))
+            sequence = (batchsubmit, submitargs, run_args, batchredirect, os.path.join(self._caseroot,get_batch_script_for_job(job)))
         else:
-            sequence = (batchsubmit, submitargs, batchredirect, get_batch_script_for_job(job), run_args)
+            sequence = (batchsubmit, submitargs, batchredirect, os.path.join(self._caseroot,get_batch_script_for_job(job)), run_args)
 
         submitcmd = " ".join(s.strip() for s in sequence if s is not None)
         if dry_run:
             return submitcmd
         else:
             logger.info("Submitting job script {}".format(submitcmd))
-            output = run_cmd_no_fail(submitcmd, combine_output=True)
+            s, output, _ = run_cmd(submitcmd, combine_output=True)
             jobid = self.get_job_id(output)
             logger.info("Submitted job id is {}".format(jobid))
             return jobid

diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py
@@ -83,7 +83,7 @@ def _save_build_provenance_cesm(case, lid): # pylint: disable=unused-argument
     manic = os.path.join("manage_externals","checkout_externals")
     manic_full_path = os.path.join(srcroot, manic)
     out = None
-    if os.path.exists(manic_full_path):
+    if shutil.which('svn') and os.path.exists(manic_full_path):
         args = " --status --verbose --no-logging"
         stat, out, err = run_cmd(manic_full_path + args, from_dir=srcroot)
         errmsg = """Error gathering provenance information from manage_externals.