Merge pull request #314 from PDoakORNL/frontier_build

Frontier build

build-aux/frontier_build.sh

@@ -1,2 +1,30 @@
 
+#export CXXFLAGS="-I/opt/cray/pe/papi/6.0.0.17/include"
+#export LDFLAGS="-L/opt/cray/pe/papi/6.0.0.17/lib -lpapi"
+#-DDCA_PROFILER=PAPI
+
+FFTW_PATH=/sw/frontier/spack-envs/base/opt/cray-sles15-zen3/gcc-11.2.0/fftw-3.3.10-njtwalw5xctv6e3fshucadlgu32jb4k6
+
+MAGMA_ROOT=/lustre/orion/world-shared/cph102/epd/gcc-11.2.0/magma-2.7.2-c5m5kzaz7irix5hk5zzf3mrwwlij43is
+
+OPENBLAS_ROOT=/lustre/orion/world-shared/cph102/epd/gcc-11.2.0/openblas-0.3.25-scaywvuh5zsm5u7smg54plj2oyf7nekv
+
+cmake -DDCA_WITH_CUDA=off -DDCA_WITH_HIP=ON \
+      -DFFTW_ROOT=$FFTW_PATH \
+      -DDCA_FIX_BROKEN_MPICH=ON \
+      -DROCM_ROOT=${OLCF_ROCM_ROOT} \
+      -DMAGMA_ROOT=${MAGMA_ROOT} \
+      -DLAPACK_ROOT=${OPENBLAS_ROOT} \
+      -DBLAS_ROOT=${OPENBLAS_ROOT} \
+      -DDCA_WITH_TESTS_FAST=ON \
+      -DTEST_RUNNER="srun" \
+      -DGPU_TARGETS=gfx90a \
+      -DAMDGPU_TARGETS=gfx90a \
+      -DCMAKE_C_COMPILER=mpicc \
+      -DCMAKE_CXX_COMPILER=mpic++ \
+      -DCMAKE_INSTALL_PREFIX=$INST \
+      -DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH \
+      -GNinja \
+      ..
+=======
 cmake -DDCA_WITH_CUDA=off -DDCA_WITH_HIP=ON      -DFFTW_INCLUDE_DIR=${OLCF_FFTW_ROOT}/include       -DFFTW_LIBRARY=${OLCF_FFTW_ROOT}/lib/libfftw3.a       -DDCA_FIX_BROKEN_MPICH=ON       -GNinja       -DROCM_ROOT=${ROCM_PATH}       -DDCA_WITH_TESTS_FAST=ON -DTEST_RUNNER="srun"       -DGPU_TARGETS=gfx908       -DCMAKE_C_COMPILER=mpicc       -DCMAKE_CXX_COMPILER=mpic++ ..

build-aux/frontier_load_modules.sh

@@ -1,19 +1,25 @@
 #!/bin/bash
 #
-# Loads all modules that are required to build DCA++ on ORNL's Spock cluster.
-# A reset is done at the beginning to restore to the default programming environment on Spock.
+# Loads all modules that are required to build DCA++ on ORNL's Frontier.
+# A reset is done at the beginning to restore to the default programming environment on Frontier.
 # This is for development only at this point.
 #
-# Usage: source spock_load_modules.sh
+# Usage: source frontier_load_modules.sh
 
 module reset
 module load gcc/11.2.0
 module load openblas
 module load hdf5
-module load rocm
+module load rocm/5.4.0
 module load fftw
 module load cmake
 module load ninja
 
+# After 2 weeks of digging through opaque linking and runtime errors,
+# I have concluded that cray-libsci causes such a mess
+# that it's much easier to compile your own openblas
+# and magma rather than fuss with it.  I did the latter in 1 day.
+module unload cray-libsci
+
 export CC=mpicc
 export CXX=mpicxx

cmake/FindMAGMA.cmake

@@ -30,7 +30,7 @@ if(MAGMA_FOUND)
       INTERFACE_COMPILE_DEFINITION "${MAGMA_DEFINITIONS}"
       IMPORTED_LINK_INTERFACE_LANGUAGES "C;CXX"
       IMPORTED_LOCATION "${MAGMA_LIBRARY}")
-    target_link_libraries(magma::magma INTERFACE BLAS::BLAS LAPACK::LAPACK)
+    target_link_libraries(magma::magma INTERFACE BLAS::BLAS LAPACK::LAPACK ${DCA_GPU_LIBS})
     message("Added magma::magma target")
   endif()
   if(NOT TARGET magma::sparse)
@@ -40,6 +40,7 @@ if(MAGMA_FOUND)
       INTERFACE_COMPILE_DEFINITION "${MAGMA_DEFINITIONS}"
       IMPORTED_LINK_INTERFACE_LANGUAGES "C;CXX"
       IMPORTED_LOCATION "${MAGMA_sparse_LIBRARY}")
+    target_link_libraries(magma::sparse INTERFACE ${DCA_GPU_LIBS})
     message("Added magma::sparse target")
   endif()  
 endif()

cmake/dca_external_libs.cmake

@@ -33,14 +33,20 @@ if (NOT DCA_HAVE_BLAS)
   find_package(BLAS REQUIRED)
 endif()
 
-mark_as_advanced(LAPACK_LIBRARIES)
-message("LAPACK_LIBRARIES: ${LAPACK_FOUND} ${LAPACK_LINKER_FLAGS} ${LAPACK_LIBRARIES} ${LAPACK95_LIBRARIES}")
-list(APPEND DCA_EXTERNAL_LIBS ${LAPACK_LIBRARIES})
+mark_as_advanced(BLAS_LIBRARIES)
+message("BLAS_LIBRARIES: ${BLAS_FOUND} ${BLAS_LINKER_FLAGS} ${BLAS_LIBRARIES} ${BLAS_LIBRARIES}")
+list(APPEND DCA_EXTERNAL_LIBS ${BLAS_LIBRARIES})
 
 ################################################################################
 # HDF5
 
-find_package(HDF5 REQUIRED COMPONENTS C CXX)
+if (NOT HDF5_LIBRARIES)
+  message("Trying to find HDF5 library")
+  set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE true)
+  find_package(HDF5 REQUIRED COMPONENTS C CXX)
+  message("HDF5: ${HDF5_FOUND} ${HDF5_LIBRARIES}")
+  mark_as_advanced(HDF5_LIBRARIES)
+endif()
 
 ################################################################################
 # ADIOS2

cmake/dca_hip.cmake

@@ -65,7 +65,7 @@ if (CMAKE_HIP_COMPILER)
   dca_add_haves_define(DCA_HAVE_HIP)
   dca_add_haves_define(DCA_HAVE_GPU)
   dca_add_haves_define(__HIP_PLATFORM_AMD__)
-  list(APPEND DCA_GPU_LIBS hip::host roc::hipblas)
+  list(APPEND DCA_GPU_LIBS hip::host roc::hipblas roc::hipsparse)
   set(DCA_HIP_PROPERTIES "CMAKE_HIP_ARCHITECTURES gfx906,gfx908")
   set(CMAKE_HIP_STANDARD 17)
   list(APPEND HIP_HIPCC_FLAGS "-fPIC")

include/dca/parallel/mpi_concurrency/mpi_collective_sum.hpp

@@ -624,8 +624,23 @@ inline void MPICollectiveSum::resolveSums() {
     return resolveSumsImplementation<double>();
   else if (current_type_ == MPI_FLOAT)
     return resolveSumsImplementation<float>();
+  else if (current_type_ == MPI_CHAR)
+    return resolveSumsImplementation<char>();
+  //else if (current_type_ == MPI_BYTE)
+  else if (current_type_ == MPI_UNSIGNED_CHAR)
+    return resolveSumsImplementation<std::uint8_t>();
+  else if (current_type_ == MPI_SHORT)
+    return resolveSumsImplementation<short>();
+  else if (current_type_ == MPI_INT)
+    return resolveSumsImplementation<int>();
+  else if (current_type_ == MPI_UNSIGNED)
+    return resolveSumsImplementation<unsigned int>();
+  else if (current_type_ == MPI_LONG)
+    return resolveSumsImplementation<long int>();
   else if (current_type_ == MPI_UNSIGNED_LONG)
     return resolveSumsImplementation<unsigned long int>();
+  else if (current_type_ == MPI_LONG_LONG_INT)
+    return resolveSumsImplementation<long long int>();
   else
     throw(std::logic_error("Type not supported."));
 }

include/dca/parallel/mpi_concurrency/mpi_type_map.hpp

@@ -54,6 +54,13 @@ struct MPITypeMap<std::uint8_t> {
   }
 };
 
+template <>
+struct MPITypeMap<short> {
+  static MPI_Datatype value() {
+    return MPI_SHORT;
+  }
+};
+
 template <>
 struct MPITypeMap<int> {
   static MPI_Datatype value() {
@@ -69,7 +76,14 @@ struct MPITypeMap<unsigned int> {
 };
 
 template <>
-struct MPITypeMap<std::size_t> {
+struct MPITypeMap<long int> {
+  static MPI_Datatype value() {
+    return MPI_LONG;
+  }
+};
+
+template <>
+struct MPITypeMap<unsigned long> {
   static MPI_Datatype value() {
     return MPI_UNSIGNED_LONG;
   }

include/dca/platform/dca_gpu_blas.h

@@ -20,7 +20,7 @@
 #include "dca/linalg/util/error_cuda.hpp"
 #elif defined(DCA_HAVE_HIP)
 #include <hip/hip_runtime.h>
-#include <hipblas.h>
+#include <hipblas/hipblas.h>
 #include <hip/hip_complex.h>
 #include "dca/util/cuda2hip.h"
 #include "dca/linalg/util/error_hip.hpp"

src/linalg/util/CMakeLists.txt

@@ -5,6 +5,6 @@ if(DCA_HAVE_GPU)
   add_library(gpu_utils STATIC ${srcs})
   dca_gpu_runtime_link(gpu_utils)
   dca_gpu_blas_link(gpu_utils)
-  target_link_libraries(gpu_utils PUBLIC magma::magma)
+  target_link_libraries(gpu_utils PUBLIC magma::magma) #"$<TARGET_PROPERTY:magma::magma,INTERFACE_LINK_LIBRARIES>")
 endif()
 

src/phys/dca_algorithms/CMakeLists.txt

@@ -7,3 +7,4 @@
 
 add_library(dca_algorithms STATIC compute_band_structure.cpp)
 dca_gpu_runtime_link(dca_algorithms)
+dca_gpu_blas_link(dca_algorithms)

src/profiling/CMakeLists.txt

@@ -2,9 +2,13 @@
 
 add_library(profiling STATIC events/time.cpp)
 
-find_library(PAPI_LIB papi)
-if(PAPI_LIB AND DCA_PROFILER EQUAL "PAPI")
+if(DCA_PROFILER STREQUAL "PAPI")
+	find_library(PAPI_LIB papi)
     add_library(papi_profiling STATIC events/papi_and_time_event.cpp)
-    target_link_libraries(papi_profiling PUBLIC papi)
+	if(PAPI_LIB)
+		target_link_libraries(papi_profiling PUBLIC papi)
+	else()
+		message(WARNING "Unable to find -lpapi. Be sure to include manually")
+	endif()
     target_link_libraries(profiling PUBLIC papi_profiling)
 endif()