Merge pull request #450 from STEllAR-GROUP/adapt-for-multigpu

Add MultiGPU Support

.gitignore

@@ -3,6 +3,7 @@
 newgit
 cmake-build-*
 cmake-cuda-*
+spack-build*
 .spack-env/*
 .clangd/*
 .cache

.jenkins/lsu/hip-tests-entry.sh

@@ -9,11 +9,11 @@
 set -eux
 
 # Tests with griddim = 8
-# we need the gcc module for the silo fortran compilation...
-srun -p mi100 -N 1 -n 1 -t 04:00:00 bash -lc "module load gcc/12 rocm/5 hwloc && module list && hipcc --version && rocminfo && ./build-all.sh Release with-CC-clang without-cuda without-mpi without-papi without-apex with-kokkos with-simd without-hpx-backend-multipole without-hpx-backend-monopole with-hpx-cuda-polling without-otf2 boost jemalloc hdf5 silo vc hpx kokkos cppuddle octotiger && cd build/octotiger/build && ctest --output-on-failure " 
+# we need the llvm module for the silo fortran compilation...
+srun -p mi100 -N 1 -n 1 -t 04:00:00 bash -lc "module load rocm/5.4 hwloc llvm/14 && module list && hipcc --version && rocminfo && ./build-all.sh Release with-CC-clang without-cuda without-mpi without-papi without-apex with-kokkos with-simd without-hpx-backend-multipole without-hpx-backend-monopole with-hpx-cuda-polling without-otf2 boost jemalloc hdf5 silo vc hpx kokkos cppuddle octotiger && cd build/octotiger/build && ctest --output-on-failure " 
 
 # Tests with griddim = 16
-sed -i 's/GRIDDIM=8/GRIDDIM=16/' build-octotiger.sh
-srun -p mi100 -N 1 -n 1 -t 04:00:00 bash -lc "module load gcc/12 rocm/5 hwloc && module list && hipcc --version && rocminfo && ./build-all.sh Release with-CC-clang without-cuda without-mpi without-papi without-apex with-kokkos with-simd without-hpx-backend-multipole without-hpx-backend-monopole with-hpx-cuda-polling without-otf2 octotiger && cd build/octotiger/build && ctest --output-on-failure -E legacy " 
-sed -i 's/GRIDDIM=16/GRIDDIM=8/' build-octotiger.sh
+# sed -i 's/GRIDDIM=8/GRIDDIM=16/' build-octotiger.sh
+# srun -p mi100 -N 1 -n 1 -t 04:00:00 bash -lc "module load rocm/5.4 hwloc llvm/14 && module list && hipcc --version && rocminfo && ./build-all.sh Release with-CC-clang without-cuda without-mpi without-papi without-apex with-kokkos with-simd without-hpx-backend-multipole without-hpx-backend-monopole with-hpx-cuda-polling without-otf2 octotiger && cd build/octotiger/build && ctest --output-on-failure -E legacy " 
+# sed -i 's/GRIDDIM=16/GRIDDIM=8/' build-octotiger.sh
 

CMakeLists.txt

@@ -59,7 +59,6 @@ option(OCTOTIGER_WITH_AVX512 "" OFF)
 
 option(OCTOTIGER_WITH_TESTS "Enable tests" ON)
 set(OCTOTIGER_WITH_GRIDDIM "8" CACHE STRING "Grid size")
-set(OCTOTIGER_WITH_GRIDDIM "8" CACHE STRING "Grid size")
 set(OCTOTIGER_WITH_MAX_NUMBER_FIELDS "15" CACHE STRING "Maximum allowed nf_ (number fields)")
 set(OCTOTIGER_THETA_MINIMUM "0.34" CACHE STRING "Minimal allowed theta value - important for optimizations")
 option(OCTOTIGER_WITH_DOCU "Enable the target to build the documentation" OFF)
@@ -211,6 +210,7 @@ set(source_files
     src/taylor.cpp
     src/util.cpp
     src/common_kernel/interactions_iterators.cpp
+    src/common_kernel/gravity_performance_counters.cpp
     src/cuda_util/cuda_scheduler.cpp
     src/monopole_interactions/legacy/cuda_monopole_interaction_interface.cpp
     src/monopole_interactions/legacy/monopole_cuda_kernel.cpp
@@ -287,6 +287,7 @@ set(header_files
     octotiger/common_kernel/struct_of_array_data.hpp
     octotiger/common_kernel/kokkos_util.hpp
     octotiger/common_kernel/std_simd.hpp
+    octotiger/common_kernel/gravity_performance_counters.hpp
     octotiger/compute_factor.hpp
     octotiger/cuda_util/cuda_global_def.hpp
     octotiger/cuda_util/cuda_helper.hpp
@@ -362,6 +363,7 @@ set(hydro_header_files
     octotiger/unitiger/hydro_impl/hydro.hpp
     octotiger/unitiger/hydro_impl/reconstruct.hpp
     octotiger/unitiger/hydro_impl/flux.hpp
+    octotiger/unitiger/hydro_impl/hydro_performance_counters.hpp
     octotiger/unitiger/radiation/radiation_physics.hpp
     octotiger/unitiger/radiation/radiation_physics_impl.hpp
     octotiger/test_problems/exact_sod.hpp
@@ -374,6 +376,7 @@ set(hydro_source_files
    src/unitiger/hydro_impl/reconstruct_cuda_kernel.cpp
    src/unitiger/hydro_impl/hydro_kernel_interface.cpp
    src/unitiger/hydro_impl/hydro_cuda_interface.cpp
+   src/unitiger/hydro_impl/hydro_performance_counters.cpp
    src/test_problems/sod/exact_sod.cpp)
 
 set(cu_source_files

frontend/frontend-helper.cpp

@@ -39,6 +39,9 @@
 #include "octotiger/multipole_interactions/legacy/multipole_interaction_interface.hpp"
 #include "octotiger/multipole_interactions/util/calculate_stencil.hpp"
 
+#include "octotiger/unitiger/hydro_impl/hydro_performance_counters.hpp"
+#include "octotiger/common_kernel/gravity_performance_counters.hpp"
+
 #include <hpx/collectives/broadcast_direct.hpp>
 #include <hpx/hpx_init.hpp>
 #include <hpx/include/actions.hpp>
@@ -216,6 +219,8 @@ void start_octotiger(int argc, char* argv[]) {
 
 void register_hpx_functions(void) {
     hpx::register_startup_function(&node_server::register_counters);
+    hpx::register_startup_function(&octotiger::hydro::register_performance_counters);
+    hpx::register_startup_function(&octotiger::fmm::register_performance_counters);
     hpx::register_pre_shutdown_function([]() { options::all_localities.clear(); });
 }
 #endif

frontend/frontend-helper.hpp

@@ -12,6 +12,7 @@
 
 void start_octotiger(int argc, char* argv[]);
 void register_hpx_functions();
+void register_cppuddle_allocator_counters(void);
 
 // Get called once per locality
 void init_executors();