LUMI setup

PTsolvers · Nov 24, 2023 · 8f6cc96 · 8f6cc96
1 parent 22618c4
commit 8f6cc96
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 16 deletions.
diff --git a/scripts_future_API/run_stokes3D.sh b/scripts_future_API/run_stokes3D.sh
@@ -5,6 +5,5 @@ module load partition/G
 module load rocm/5.3.3
 
 export MPICH_GPU_SUPPORT_ENABLED=1
-export LD_PRELOAD=${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so
 
 julia --project -O3 tm_stokes_mpi_wip.jl
diff --git a/scripts_future_API/submit_stokes3D.sh b/scripts_future_API/submit_stokes3D.sh
@@ -12,7 +12,4 @@
 # export ROCR_VISIBLE_DEVICES=0,2,4,6
 # srun --cpu-bind=map_cpu:49,17,1,33 ./run_stokes3D.sh
 
-export MPICH_GPU_SUPPORT_ENABLED=1
-export LD_PRELOAD=${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so
-
 srun --cpu-bind=map_cpu:49,57,17,25,1,9,33,41 ./run_stokes3D.sh
diff --git a/scripts_future_API/tm_stokes_mpi_wip.jl b/scripts_future_API/tm_stokes_mpi_wip.jl
@@ -14,7 +14,7 @@ const SBC = BoundaryCondition{Slip}
 
 using LinearAlgebra, Printf
 using KernelAbstractions
-# using AMDGPU
+using AMDGPU
 
 using FastIce.Distributed
 using MPI
@@ -35,28 +35,29 @@ max_abs_g(A) = (max_l = maximum(abs.(interior(A))); MPI.Allreduce(max_l, MPI.MAX
 function main(; do_visu=false, do_save=false)
     MPI.Init()
 
-    backend = CPU()
+    backend = ROCBackend()
+    dims = (1, 1, 1)
     # dims = (4, 2, 2)
-    dims = (2, 1, 1)
+    # dims = (2, 1, 1)
     topo = CartesianTopology(dims)
     arch = Architecture(backend, topo)
     set_device!(arch)
 
     comm = cartesian_communicator(topo)
 
-    size_l = (62, 62, 62)
+    size_l = (254, 254, 254)
     size_g = global_grid_size(topo, size_l)
 
-    b_width = (16, 8, 4) #(128, 32, 4)#
+    outer_width = (16, 16, 16) #(128, 32, 4)#
 
     if global_rank(topo) == 0
         @show dimensions(topo)
         @show size_g
     end
 
     grid_g = CartesianGrid(; origin=(-2.0, -1.0, 0.0),
-                             extent=(4.0, 2.0, 2.0),
-                             size=size_g)
+                           extent=(4.0, 2.0, 2.0),
+                           size=size_g)
 
     grid_l = local_grid(grid_g, topo)
 
@@ -99,7 +100,7 @@ function main(; do_visu=false, do_save=false)
                                       physics,
                                       gravity,
                                       boundary_conditions,
-                                      outer_width=b_width,
+                                      outer_width,
                                       iter_params,
                                       other_fields)
 
@@ -214,10 +215,10 @@ function main(; do_visu=false, do_save=false)
 
     if (global_rank(topo) == 0) && do_visu
         fig = Figure()
-        axs = (Pr = Axis(fig[1, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Pr"),
-            Vx = Axis(fig[1, 2][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vx"),
-            Vy = Axis(fig[2, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vy"),
-            Vz = Axis(fig[2, 2][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vz"))
+        axs = (Pr=Axis(fig[1, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Pr"),
+               Vx=Axis(fig[1, 2][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vx"),
+               Vy=Axis(fig[2, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vy"),
+               Vz=Axis(fig[2, 2][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vz"))
         plt = (Pr = heatmap!(axs.Pr, xcenters(grid_g), zcenters(grid_g), Pr_g[:, size(grid_g, 2)÷2+1, :]; colormap=:turbo),
                Vx = heatmap!(axs.Vx, xvertices(grid_g), zcenters(grid_g), Vx_g[:, size(grid_g, 2)÷2+1, :]; colormap=:turbo),
                Vy = heatmap!(axs.Vy, xcenters(grid_g), zcenters(grid_g), Vy_g[:, size(grid_g, 2)÷2+1, :]; colormap=:turbo),