diff --git a/build-aux/frontier_build.sh b/build-aux/frontier_build.sh index ab03def64..3f8f89f3e 100644 --- a/build-aux/frontier_build.sh +++ b/build-aux/frontier_build.sh @@ -1,16 +1,15 @@ -#export CXXFLAGS="-I/opt/cray/pe/papi/6.0.0.17/include" -#export LDFLAGS="-L/opt/cray/pe/papi/6.0.0.17/lib -lpapi" -#-DDCA_PROFILER=PAPI +export FFTW_PATH=/lustre/orion/world-shared/cph102/epd/spack/opt/spack/linux-sles15-zen3/gcc-12.2.0/fftw-3.3.10-tajdtzkealhold4bmpuq7wiwzurnclr4 +export MAGMA_ROOT=/lustre/orion/world-shared/cph102/epd/spack/opt/spack/linux-sles15-zen3/gcc-12.2.0/magma-2.7.2-gbjcrprqdw7y5uplm5upmqbi65zqwubb +export OPENBLAS_ROOT=/lustre/orion/world-shared/cph102/epd/spack/opt/spack/linux-sles15-zen3/gcc-12.2.0/openblas-0.3.25-t62dxdtaqba6lzrwoy4uddswlprgma6n +export HDF5_ROOT=/lustre/orion/world-shared/cph102/epd/spack/opt/spack/linux-sles15-zen3/gcc-12.2.0/hdf5-1.14.3-3so3g5x2roywum3edvjun7jbhwisei6p +export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/opt/rocm-5.7.0/hip/bin:${HDF5_ROOT} +export PATH=/sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/cmake-3.23.2-4r4mpiba7cwdw2hlakh5i7tchi64s3qd/bin:${PATH} -FFTW_PATH=/sw/frontier/spack-envs/base/opt/cray-sles15-zen3/gcc-11.2.0/fftw-3.3.10-njtwalw5xctv6e3fshucadlgu32jb4k6 -MAGMA_ROOT=/lustre/orion/world-shared/cph102/epd/gcc-11.2.0/magma-2.7.2-c5m5kzaz7irix5hk5zzf3mrwwlij43is -OPENBLAS_ROOT=/lustre/orion/world-shared/cph102/epd/gcc-11.2.0/openblas-0.3.25-scaywvuh5zsm5u7smg54plj2oyf7nekv -HDF5_ROOT=/lustre/orion/world-shared/cph102/epd/gcc-11.2.0/hdf5-1.14.3-mjg4476nl2cfrmk5qmcrmvo7ryg2bdwd cmake -DDCA_WITH_CUDA=off -DDCA_WITH_HIP=ON \ -DFFTW_ROOT=$FFTW_PATH \ -DDCA_FIX_BROKEN_MPICH=ON \ - -DROCM_ROOT=${OLCF_ROCM_ROOT} \ + -DROCM_ROOT=${ROCM_PATH} \ -DMAGMA_ROOT=${MAGMA_ROOT} \ -DLAPACK_ROOT=${OPENBLAS_ROOT} \ -DBLAS_ROOT=${OPENBLAS_ROOT} \ @@ -20,7 +19,8 @@ cmake -DDCA_WITH_CUDA=off -DDCA_WITH_HIP=ON \ -DAMDGPU_TARGETS=gfx90a \ -DCMAKE_C_COMPILER=mpicc \ -DCMAKE_CXX_COMPILER=mpic++ \ + -DCMAKE_HIP_COMPILER=/opt/rocm-5.7.0/llvm/bin/clang++ \ -DCMAKE_INSTALL_PREFIX=$INST \ - -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH} ${HDF5_ROOT}" \ + -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" \ -GNinja \ .. diff --git a/build-aux/frontier_load_modules.sh b/build-aux/frontier_load_modules.sh index cec973a82..83e2e7c53 100644 --- a/build-aux/frontier_load_modules.sh +++ b/build-aux/frontier_load_modules.sh @@ -7,12 +7,8 @@ # Usage: source frontier_load_modules.sh module reset -module load gcc/11.2.0 -module load openblas -module load hdf5 -module load rocm/5.4.0 -module load fftw -module load cmake +module load gcc/12.2.0 +module load rocm/5.7.0 module load ninja # After 2 weeks of digging through opaque linking and runtime errors, diff --git a/cmake/dca_external_libs.cmake b/cmake/dca_external_libs.cmake index 9d35c9e26..92a0cd628 100644 --- a/cmake/dca_external_libs.cmake +++ b/cmake/dca_external_libs.cmake @@ -42,7 +42,7 @@ list(APPEND DCA_EXTERNAL_LIBS ${BLAS_LIBRARIES}) if (NOT HDF5_LIBRARIES) message("Trying to find HDF5 library") - set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE true) + set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE false) set(HDF5_PREFER_PARALLEL false) find_package(HDF5 REQUIRED COMPONENTS C CXX) message("HDF5: ${HDF5_FOUND} ${HDF5_LIBRARIES}") diff --git a/include/dca/parallel/mpi_concurrency/mpi_type_map.hpp b/include/dca/parallel/mpi_concurrency/mpi_type_map.hpp index aa538e401..86e20dbe3 100644 --- a/include/dca/parallel/mpi_concurrency/mpi_type_map.hpp +++ b/include/dca/parallel/mpi_concurrency/mpi_type_map.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #include "dca_mpi.h" diff --git a/include/dca/phys/dca_loop/dca_loop.hpp b/include/dca/phys/dca_loop/dca_loop.hpp index 1a6b30374..195b98c64 100644 --- a/include/dca/phys/dca_loop/dca_loop.hpp +++ b/include/dca/phys/dca_loop/dca_loop.hpp @@ -236,7 +236,6 @@ void DcaLoop::initialize() { } else if (parameters.get_initial_self_energy() != "zero") { #ifdef DCA_HAVE_ADIOS2 - io::IOType iotype = io::extensionToIOType(autoresume_filename); if (io::extensionToIOType(parameters.get_initial_self_energy()) == io::IOType::ADIOS2) MOMS.initializeSigma(concurrency.get_adios(), parameters.get_initial_self_energy()); else diff --git a/include/dca/phys/dca_step/cluster_solver/ctint/accumulator/ctint_accumulator.hpp b/include/dca/phys/dca_step/cluster_solver/ctint/accumulator/ctint_accumulator.hpp index 44cc711af..69ac57e18 100644 --- a/include/dca/phys/dca_step/cluster_solver/ctint/accumulator/ctint_accumulator.hpp +++ b/include/dca/phys/dca_step/cluster_solver/ctint/accumulator/ctint_accumulator.hpp @@ -40,8 +40,8 @@ template { public: constexpr static ClusterSolverId solver_id{ClusterSolverId::CT_INT}; - using Real = typename dca::config::McOptions::MC_REAL; - using Scalar = typename dca::util::ScalarSelect::type; + using Real = typename Parameters::Real; + using Scalar = typename Parameters::Scalar; using Base = MC_accumulator_data; using this_type = CtintAccumulator; using Base::accumulated_phase_; diff --git a/include/dca/phys/dca_step/cluster_solver/ctint/ctint_cluster_solver.hpp b/include/dca/phys/dca_step/cluster_solver/ctint/ctint_cluster_solver.hpp index 78c235904..a68e8f26f 100644 --- a/include/dca/phys/dca_step/cluster_solver/ctint/ctint_cluster_solver.hpp +++ b/include/dca/phys/dca_step/cluster_solver/ctint/ctint_cluster_solver.hpp @@ -521,7 +521,8 @@ auto CtintClusterSolver::local_G_k_w( SpGreensFunction M; math::transform::FunctionTransform::execute(M_r, M); - const double sign = accumulator_.get_accumulated_sign(); + // This phase can be a long long if we're dealing with real Scalars and immense iterations. + const long double sign = accumulator_.get_accumulated_phase(); M /= sign; SpGreensFunction G_k_w("G_k_w"); diff --git a/test/integration/statistical_tests/real_materials/CMakeLists.txt b/test/integration/statistical_tests/real_materials/CMakeLists.txt index 723f40ccb..00577c694 100644 --- a/test/integration/statistical_tests/real_materials/CMakeLists.txt +++ b/test/integration/statistical_tests/real_materials/CMakeLists.txt @@ -4,7 +4,7 @@ if(DCA_WITH_TESTS_STOCHASTIC) # leave it to break the build if its not maintained. I found it completely inoperable after 4 years of not being built. add_executable(NiO_fullDCA NiO_fullDCA.cpp) target_link_libraries(NiO_fullDCA ${DCA_LIBS} ${DCA_THREADING_LIBS}) - target_include_directories(NiO_fullDCA PRIVATE ${DCA_INCLUDE_DIRS}) + target_include_directories(NiO_fullDCA PRIVATE ${DCA_INCLUDE_DIRS} ${PROJECT_SOURCE_DIR}) target_compile_definitions(NiO_fullDCA PRIVATE TEST_DIRECTORY="${CMAKE_CURRENT_SOURCE_DIR}/") target_compile_definitions(NiO_fullDCA PRIVATE DCA_SOURCE_DIR="${PROJECT_SOURCE_DIR}") diff --git a/test/integration/statistical_tests/real_materials/NiO_fullDCA.cpp b/test/integration/statistical_tests/real_materials/NiO_fullDCA.cpp index 237f766e2..0d86993e1 100644 --- a/test/integration/statistical_tests/real_materials/NiO_fullDCA.cpp +++ b/test/integration/statistical_tests/real_materials/NiO_fullDCA.cpp @@ -22,6 +22,14 @@ #include "dca/util/git_version.hpp" #include "dca/distribution/dist_types.hpp" #include "dca_loop_wrapper.hpp" + +#include "test/mock_mcconfig.hpp" +namespace dca { +namespace config { +using McOptions = MockMcOptions; +} // namespace config +} // namespace dca + #include "NiO_setup.hpp" constexpr auto SOLVER = dca::ClusterSolverId::CT_INT; @@ -51,7 +59,7 @@ struct MakeOutput; template struct MakeOutput{ - void operator()(DCACALC& calculation, Parameters& parameters, Concurrency& concurrency) { + void operator()(DCACALC& calculation, Parameters& parameters [[maybe_unused]], Concurrency& concurrency) { const int n_frequencies = 10; auto G_k_cut = dca::math::util::bandDiagonal( diff --git a/test/integration/statistical_tests/real_materials/NiO_setup.hpp b/test/integration/statistical_tests/real_materials/NiO_setup.hpp index abfcd1afd..f3b4a468a 100644 --- a/test/integration/statistical_tests/real_materials/NiO_setup.hpp +++ b/test/integration/statistical_tests/real_materials/NiO_setup.hpp @@ -50,11 +50,13 @@ constexpr DeviceType default_device = GPU; constexpr DeviceType default_device = CPU; #endif // DCA_HAVE_CUDA +using Scalar = double; + const std::string test_directory = DCA_SOURCE_DIR "/test/integration/statistical_tests/real_materials/"; using Model = dca::phys::models::TightBindingModel>>; + dca::phys::models::Material::NiO_unsymmetric, dca::phys::domains::no_symmetry<3>>>; using RandomNumberGenerator = dca::math::random::StdRandomWrapper; using dca::ClusterSolverId; @@ -62,7 +64,7 @@ using dca::ClusterSolverId; template using TestParameters = dca::phys::params::Parameters; + RandomNumberGenerator, name, dca::NumericalTraits, Scalar>>; template using DcaData = dca::phys::DcaData>; diff --git a/test/integration/statistical_tests/real_materials/NIO_stattest_gs.cpp b/test/integration/statistical_tests/real_materials/NiO_stattest_gs.cpp similarity index 100% rename from test/integration/statistical_tests/real_materials/NIO_stattest_gs.cpp rename to test/integration/statistical_tests/real_materials/NiO_stattest_gs.cpp diff --git a/test/integration/statistical_tests/square_lattice/ctint_square_lattice_validation_stattest.cpp b/test/integration/statistical_tests/square_lattice/ctint_square_lattice_validation_stattest.cpp index 01ef6ec85..bb48f3544 100644 --- a/test/integration/statistical_tests/square_lattice/ctint_square_lattice_validation_stattest.cpp +++ b/test/integration/statistical_tests/square_lattice/ctint_square_lattice_validation_stattest.cpp @@ -17,6 +17,14 @@ #include "dca/math/statistical_testing/statistical_testing.hpp" #include "dca/config/profiler.hpp" + +#include "test/mock_mcconfig.hpp" +namespace dca { +namespace config { +using McOptions = MockMcOptions; +} // namespace config +} // namespace dca + #include "test/integration/statistical_tests/square_lattice/square_lattice_setup.hpp" dca::testing::DcaMpiTestEnvironment* dca_test_env; diff --git a/test/integration/statistical_tests/square_lattice/square_lattice_setup.hpp b/test/integration/statistical_tests/square_lattice/square_lattice_setup.hpp index 108baa62e..d3b84bde4 100644 --- a/test/integration/statistical_tests/square_lattice/square_lattice_setup.hpp +++ b/test/integration/statistical_tests/square_lattice/square_lattice_setup.hpp @@ -62,7 +62,7 @@ using dca::ClusterSolverId; template using ParametersType = dca::phys::params::Parameters; + dca::profiling::NullProfiler, Model, RandomNumberGenerator, CS_NAME, dca::NumericalTraits, Scalar>>; template using DcaData = dca::phys::DcaData>;