diff --git a/.github/workflows/RunnerFullCI.yml b/.github/workflows/RunnerFullCI.yml index 265a1431..8f368ae0 100644 --- a/.github/workflows/RunnerFullCI.yml +++ b/.github/workflows/RunnerFullCI.yml @@ -83,7 +83,26 @@ jobs: cmake --install build ctest --test-dir build rm -rf build - + + # Configure-Build-Run-Run + - name: Compile NHVHPC GPU cuMPI inplace + run: | + ls -ltr ~/.bashrc + source ~/.bashrc + source ~/.profile_github + module load CMake + module load nvhpc + export FC=mpif90 + cmake -S . -B build -DBUILD_TESTING=ON -DBUILD_TARGET=gpu -DENABLE_NCCL=no -DENABLE_INPLACE=ON + cmake --build build -j 4 + cmake --install build + ctest --test-dir build + cmake -S . -B build -DNX=128 -DNY=128 -DNX=128 + cmake --build build -j 4 + cmake --install build + ctest --test-dir build + rm -rf build + # Configure-Build-Run - name: Compile NHVHPC GPU NCCL run: | diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 1d7bc673..c208dfcf 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -18,3 +18,6 @@ add_subdirectory(fft_physical_x) add_subdirectory(fft_physical_z) add_subdirectory(halo_test) add_subdirectory(io_test) + +# Set real/complex tests +#set(COMPLEX_TESTS "OFF" CACHE STRING "Enables complex numbers for tests that support it") diff --git a/examples/io_test/CMakeLists.txt b/examples/io_test/CMakeLists.txt index 5a88a69e..cf8a28c8 100644 --- a/examples/io_test/CMakeLists.txt +++ b/examples/io_test/CMakeLists.txt @@ -3,6 +3,7 @@ file(GLOB files_read io_read.f90) file(GLOB files_var_test io_var_test.f90) file(GLOB files_plane_test io_plane_test.f90) file(GLOB files_bench io_bench.f90) +file(GLOB files_visu io_visu.f90) include_directories(${CMAKE_SOURCE_DIR}/src) @@ -11,14 +12,17 @@ add_executable(io_read ${files_read}) add_executable(io_var_test ${files_var_test}) add_executable(io_plane_test ${files_plane_test}) add_executable(io_bench ${files_bench}) +add_executable(io_visu ${files_visu}) target_link_libraries(io_test PRIVATE decomp2d examples_utils) target_link_libraries(io_read PRIVATE decomp2d examples_utils) target_link_libraries(io_var_test PRIVATE decomp2d examples_utils) target_link_libraries(io_plane_test PRIVATE decomp2d examples_utils) target_link_libraries(io_bench PRIVATE decomp2d examples_utils) +target_link_libraries(io_visu PRIVATE decomp2d examples_utils) # Run the test(s) +# Note visu is not a test - it is an example to show/create visu files set(run_dir "${test_dir}/io_test") file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir}) message(STATUS "Example dir ${run_dir}") @@ -64,3 +68,14 @@ if (BUILD_TARGET MATCHES "gpu") else () add_test(NAME io_bench COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $ ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir}) endif () +# Run the test(s) +set(run_dir "${test_dir}/io_visu") +file (COPY "${CMAKE_SOURCE_DIR}/examples/io_test/adios2_config.xml" DESTINATION ${run_dir}) +message(STATUS "Example dir ${run_dir}") +file(MAKE_DIRECTORY ${run_dir}) +if (BUILD_TARGET MATCHES "gpu") + file(COPY bind.sh DESTINATION ${run_dir}) + add_test(NAME io_visu COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ./bind.sh $ ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir}) +else () + add_test(NAME io_visu COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} $ ${TEST_ARGUMENTS} WORKING_DIRECTORY ${run_dir}) +endif () diff --git a/examples/io_test/adios2_config.xml b/examples/io_test/adios2_config.xml index 92ece049..69867f4f 100644 --- a/examples/io_test/adios2_config.xml +++ b/examples/io_test/adios2_config.xml @@ -16,4 +16,12 @@ + + + + + + + + diff --git a/examples/io_test/io_visu.f90 b/examples/io_test/io_visu.f90 new file mode 100644 index 00000000..7cc82cd7 --- /dev/null +++ b/examples/io_test/io_visu.f90 @@ -0,0 +1,257 @@ +!!! SPDX-License-Identifier: BSD-3-Clause +!!! +!!! Example code to demonstrate visualisation of fields. + +program visu + + use mpi + + use decomp_2d + use decomp_2d_constants + use decomp_2d_io + use decomp_2d_mpi + use decomp_2d_testing + + implicit none + + integer, parameter :: nx_base = 17, ny_base = 13, nz_base = 11 + integer :: nx, ny, nz + integer :: p_row = 0, p_col = 0 + + real(mytype), allocatable, dimension(:, :, :) :: u1, u2, u3 + + integer, parameter :: output2D = 0 ! Which plane to write in 2D (0 for 3D) + character(len=*), parameter :: io_name = "visu-io" +#ifndef ADIOS2 + logical :: dir_exists +#endif + + integer :: i, j, k + integer :: ierr + + call init_example() + call init_data() + + call write_data() + call write_visu() + + call fin() + +contains + + subroutine init_example() + + integer :: resize_domain + integer :: nranks_tot + + call MPI_INIT(ierr) + ! To resize the domain we need to know global number of ranks + ! This operation is also done as part of decomp_2d_init + call MPI_COMM_SIZE(MPI_COMM_WORLD, nranks_tot, ierr) + resize_domain = int(nranks_tot / 4) + 1 + nx = nx_base * resize_domain + ny = ny_base * resize_domain + nz = nz_base * resize_domain + ! Now we can check if user put some inputs + call decomp_2d_testing_init(p_row, p_col, nx, ny, nz) + + call decomp_2d_init(nx, ny, nz, p_row, p_col) + + call decomp_2d_testing_log() + + call decomp_2d_io_init() + call decomp_2d_init_io(io_name) + call decomp_2d_register_variable(io_name, "u1.dat", 1, 0, output2D, mytype) + call decomp_2d_register_variable(io_name, "u2.dat", 2, 0, output2D, mytype) + call decomp_2d_register_variable(io_name, "u3.dat", 3, 0, output2D, mytype) + + end subroutine init_example + + subroutine init_data() + + integer :: xen1, xen2, xen3 + integer :: yen1, yen2, yen3 + integer :: zen1, zen2, zen3 + + call alloc_x(u1) + call alloc_y(u2) + call alloc_z(u3) + xen1 = xsize(1) + xen2 = xsize(2) + xen3 = xsize(3) + yen1 = ysize(1) + yen2 = ysize(2) + yen3 = ysize(3) + zen1 = zsize(1) + zen2 = zsize(2) + zen3 = zsize(3) + + ! distribute the data + !$acc data copy(u1,u2,u3) + + !$acc parallel loop default(present) + do k = 1, xen3 + do j = 1, xen2 + do i = 1, xen1 + u1(i, j, k) = real(100 + nrank, mytype) + end do + end do + end do + !$acc end loop + !$acc parallel loop default(present) + do k = 1, yen3 + do j = 1, yen2 + do i = 1, yen1 + u2(i, j, k) = real(100 + nrank, mytype) + end do + end do + end do + !$acc end loop + !$acc parallel loop default(present) + do k = 1, zen3 + do j = 1, zen2 + do i = 1, zen1 + u3(i, j, k) = real(100 + nrank, mytype) + end do + end do + end do + !$acc end loop + !$acc update self (u1) + !$acc update self (u2) + !$acc update self (u3) + !$acc end data + + end subroutine init_data + + subroutine write_data() + + !! Write arrays + visu data from orientations 1, 2 and 3 +#ifndef ADIOS2 + if (nrank == 0) then + inquire (file="out", exist=dir_exists) + if (.not. dir_exists) then + call execute_command_line("mkdir out 2> /dev/null") + end if + end if +#endif + + ! Standard I/O pattern - file per field +#ifdef ADIOS2 + call decomp_2d_open_io(io_name, "out", decomp_2d_write_mode) + call decomp_2d_start_io(io_name, "out") +#endif + + call decomp_2d_write_one(1, u1, 'out', 'u1.dat', 0, io_name) + call decomp_2d_write_one(2, u2, 'out', 'u2.dat', 0, io_name) + call decomp_2d_write_one(3, u3, 'out', 'u3.dat', 0, io_name) + +#ifdef ADIOS2 + call decomp_2d_end_io(io_name, "out") + call decomp_2d_close_io(io_name, "out") +#endif + + end subroutine write_data + + subroutine write_visu() + ! This subroutine is based on the xdmf writers in Xcompact3d. + ! Copyright (c) 2012-2022, Xcompact3d + ! SPDX-License-Identifier: BSD 3-Clause + + integer :: ioxdmf + + character(len=:), allocatable :: fmt + + integer :: precision + + integer :: varctr + character(len=16) :: filename + character(len=2) :: varname + + if (nrank == 0) then + OPEN (newunit=ioxdmf, file="./out.xdmf") + + write (ioxdmf, '(A22)') '' + write (ioxdmf, *) '' + write (ioxdmf, *) '' + write (ioxdmf, *) '' + + write (ioxdmf, '(A)') ' ' + write (ioxdmf, '(A)') ' ' + + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' 0.0 0.0 0.0' + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' ' + if (mytype == kind(0.0d0)) then + fmt = "(A, E24.17, A, E24.17, A, E24.17)" + else + fmt = "(A, E16.9, A, E16.9, A, E16.9)" + end if + write (ioxdmf, fmt) ' ', 1.0_mytype, " ", 1.0_mytype, " ", 1.0_mytype + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' ' + + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' ' + + do varctr = 1, 3 + write (varname, "(A, I0)") "u", varctr + write (filename, '(A, I0, A)') "./out/u", varctr, ".dat" + write (ioxdmf, *) ' ' +#ifndef ADIOS2 + write (ioxdmf, *) ' ' + + write (ioxdmf, *) ' '//trim(filename) + + write (ioxdmf, *) ' ' + write (ioxdmf, *) ' ' + end do + + write (ioxdmf, '(/)') + write (ioxdmf, *) ' ' + write (ioxdmf, *) '' + write (ioxdmf, '(A7)') '' + close (ioxdmf) + end if + + end subroutine write_visu + + subroutine fin() + + integer :: ierr + + call decomp_2d_finalize + call MPI_FINALIZE(ierr) + + end subroutine fin + +end program visu +