From cefb1c7d17db2ef760d29730a7f2f549adf19cfb Mon Sep 17 00:00:00 2001
From: Ludovic Raess <ludovic.rass@gmail.com>
Date: Thu, 14 Sep 2023 16:27:12 +0300
Subject: [PATCH] test exch on lumi

---
 scripts_future_API/bench3d.jl    |  4 +++-
 scripts_future_API/exchanger2.jl | 11 +++++------
 scripts_future_API/profileme.sh  |  4 ++--
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/scripts_future_API/bench3d.jl b/scripts_future_API/bench3d.jl
index 2557d3fd..b1f1e530 100644
--- a/scripts_future_API/bench3d.jl
+++ b/scripts_future_API/bench3d.jl
@@ -80,7 +80,9 @@ function main(backend=CPU(), T::DataType=Float64, dims=(0, 0, 0))
 
     # actions
     for it = 1:nt
-        copyto!(A, A_new)
+        # copyto!(A, A_new)
+        # KernelAbstractions.synchronize(backend)
+
         hide_comm( diffusion_kernel!(backend, 256), neighbors, ranges, A_new, A, h, _dx, _dy, _dz )
     end
 
diff --git a/scripts_future_API/exchanger2.jl b/scripts_future_API/exchanger2.jl
index c6895b37..775dec21 100644
--- a/scripts_future_API/exchanger2.jl
+++ b/scripts_future_API/exchanger2.jl
@@ -1,10 +1,10 @@
 using KernelAbstractions
 using MPI
 
-# using AMDGPU
+using AMDGPU
 
-using CUDA
-using NVTX
+# using CUDA
+# using NVTX
 
 include("mpi_utils.jl")
 include("mpi_utils2.jl")
@@ -51,7 +51,7 @@ function main(backend = CPU(), T::DataType = Float64, dims = (0, 0, 0))
             range  = ranges[2*(dim-1) + side]
             offset, ndrange = first(range), size(range)
             start_exchange(exchangers[dim][side], comm, rank, halo, border) do compute_bc
-                NVTX.@range "borders" do_work!(backend, 256)(A, me, offset; ndrange)
+                do_work!(backend, 256)(A, me, offset; ndrange)
                 if compute_bc
                     # apply_bcs!(Val(dim), fields, bcs.velocity)
                 end
@@ -60,7 +60,6 @@ function main(backend = CPU(), T::DataType = Float64, dims = (0, 0, 0))
         end
         wait.(exchangers[dim])
     end
-
     KernelAbstractions.synchronize(backend)
 
     # for dim in eachindex(neighbors)
@@ -75,7 +74,7 @@ function main(backend = CPU(), T::DataType = Float64, dims = (0, 0, 0))
     return
 end
 
-backend = CUDABackend()
+backend = ROCBackend()
 T::DataType = Int
 dims = (0, 0, 1)
 
diff --git a/scripts_future_API/profileme.sh b/scripts_future_API/profileme.sh
index f89f9adb..87b13d15 100755
--- a/scripts_future_API/profileme.sh
+++ b/scripts_future_API/profileme.sh
@@ -6,10 +6,10 @@ source /users/lurass/scratch/setenv_lumi.sh
 
 # srun --cpu-bind=map_cpu:49,57,17,25,1,9,33,41 -N1 -n8 --gpus-per-node=8 profileme.sh
 
-julia --project bench3d.jl
+# julia --project bench3d.jl
 # julia --project exchanger2.jl
 # julia --project rocmaware.jl
 
 # ENABLE_JITPROFILING=1 ../../myrocprof --hsa-trace --hip-trace julia --project ./exchanger2.jl
 
-# ENABLE_JITPROFILING=1 rocprof --hip-trace --hsa-trace -d ./prof_out${SLURM_PROCID} -o ./prof_out${SLURM_PROCID}/results${SLURM_PROCID}.csv julia --project bench3d.jl
\ No newline at end of file
+ENABLE_JITPROFILING=1 rocprof --hip-trace --hsa-trace -d ./prof_out${SLURM_PROCID} -o ./prof_out${SLURM_PROCID}/results${SLURM_PROCID}.csv julia --project bench3d.jl
\ No newline at end of file