From 8f6cc96d0126678ab3fae4c8ceaf8f1839286e9c Mon Sep 17 00:00:00 2001 From: Ivan Utkin Date: Fri, 24 Nov 2023 02:41:03 +0200 Subject: [PATCH] LUMI setup --- scripts_future_API/run_stokes3D.sh | 1 - scripts_future_API/submit_stokes3D.sh | 3 --- scripts_future_API/tm_stokes_mpi_wip.jl | 25 +++++++++++++------------ 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/scripts_future_API/run_stokes3D.sh b/scripts_future_API/run_stokes3D.sh index 2655f81b..d7ae7850 100755 --- a/scripts_future_API/run_stokes3D.sh +++ b/scripts_future_API/run_stokes3D.sh @@ -5,6 +5,5 @@ module load partition/G module load rocm/5.3.3 export MPICH_GPU_SUPPORT_ENABLED=1 -export LD_PRELOAD=${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so julia --project -O3 tm_stokes_mpi_wip.jl diff --git a/scripts_future_API/submit_stokes3D.sh b/scripts_future_API/submit_stokes3D.sh index a48061bb..04615068 100755 --- a/scripts_future_API/submit_stokes3D.sh +++ b/scripts_future_API/submit_stokes3D.sh @@ -12,7 +12,4 @@ # export ROCR_VISIBLE_DEVICES=0,2,4,6 # srun --cpu-bind=map_cpu:49,17,1,33 ./run_stokes3D.sh -export MPICH_GPU_SUPPORT_ENABLED=1 -export LD_PRELOAD=${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so - srun --cpu-bind=map_cpu:49,57,17,25,1,9,33,41 ./run_stokes3D.sh diff --git a/scripts_future_API/tm_stokes_mpi_wip.jl b/scripts_future_API/tm_stokes_mpi_wip.jl index 319303c0..3ba53a9b 100644 --- a/scripts_future_API/tm_stokes_mpi_wip.jl +++ b/scripts_future_API/tm_stokes_mpi_wip.jl @@ -14,7 +14,7 @@ const SBC = BoundaryCondition{Slip} using LinearAlgebra, Printf using KernelAbstractions -# using AMDGPU +using AMDGPU using FastIce.Distributed using MPI @@ -35,19 +35,20 @@ max_abs_g(A) = (max_l = maximum(abs.(interior(A))); MPI.Allreduce(max_l, MPI.MAX function main(; do_visu=false, do_save=false) MPI.Init() - backend = CPU() + backend = ROCBackend() + dims = (1, 1, 1) # dims = (4, 2, 2) - dims = (2, 1, 1) + # dims = (2, 1, 1) topo = CartesianTopology(dims) arch = Architecture(backend, topo) set_device!(arch) comm = cartesian_communicator(topo) - size_l = (62, 62, 62) + size_l = (254, 254, 254) size_g = global_grid_size(topo, size_l) - b_width = (16, 8, 4) #(128, 32, 4)# + outer_width = (16, 16, 16) #(128, 32, 4)# if global_rank(topo) == 0 @show dimensions(topo) @@ -55,8 +56,8 @@ function main(; do_visu=false, do_save=false) end grid_g = CartesianGrid(; origin=(-2.0, -1.0, 0.0), - extent=(4.0, 2.0, 2.0), - size=size_g) + extent=(4.0, 2.0, 2.0), + size=size_g) grid_l = local_grid(grid_g, topo) @@ -99,7 +100,7 @@ function main(; do_visu=false, do_save=false) physics, gravity, boundary_conditions, - outer_width=b_width, + outer_width, iter_params, other_fields) @@ -214,10 +215,10 @@ function main(; do_visu=false, do_save=false) if (global_rank(topo) == 0) && do_visu fig = Figure() - axs = (Pr = Axis(fig[1, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Pr"), - Vx = Axis(fig[1, 2][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vx"), - Vy = Axis(fig[2, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vy"), - Vz = Axis(fig[2, 2][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vz")) + axs = (Pr=Axis(fig[1, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Pr"), + Vx=Axis(fig[1, 2][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vx"), + Vy=Axis(fig[2, 1][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vy"), + Vz=Axis(fig[2, 2][1, 1]; aspect=DataAspect(), xlabel="x", ylabel="z", title="Vz")) plt = (Pr = heatmap!(axs.Pr, xcenters(grid_g), zcenters(grid_g), Pr_g[:, size(grid_g, 2)÷2+1, :]; colormap=:turbo), Vx = heatmap!(axs.Vx, xvertices(grid_g), zcenters(grid_g), Vx_g[:, size(grid_g, 2)÷2+1, :]; colormap=:turbo), Vy = heatmap!(axs.Vy, xcenters(grid_g), zcenters(grid_g), Vy_g[:, size(grid_g, 2)÷2+1, :]; colormap=:turbo),