Skip to content

Commit

Permalink
Mpi local tests (#432)
Browse files Browse the repository at this point in the history 
# Add Tests for MPI version of code

### Task List
- [x] add tests
- [x] rebase to develop
- [x] fix array ordering
- [x] ~update the readthedocs / readme with install instructions~ (moved
to #455)

---
# Change Description

I rebased @draenog's `mpi_local` and `mpi_local_tests` branches. This
introduced a bug due to array reordering that occurred as part of #426.
I have modified the MPI parts of `./core/src/RectGridIO.cpp` to reorder
array so that they now match the order in the serial version of the
code.

---
# Test Description

To run `testRectGrid_MPI` you will need the parallel netcdf config file
`partition_metadata_1.nc`. I don't want to put it in the github repo for
now. As discussed with @timspainNERSC , soon we will hopefully have an
ftp server for netcdf files.

You can generate the input using the following netcdf definition file
`partition_metadata_1.cdl`.
```
// partition_metadata_1.cdl
netcdf partition_metadata_1 {
dimensions:
	P = 1 ;
	L = 1 ;
	globalX = 25 ;
	globalY = 15 ;

group: bounding_boxes {
  variables:
  	int global_x(P) ;
  	int global_y(P) ;
  	int local_extent_x(P) ;
  	int local_extent_y(P) ;
  data:

   global_x = 0 ;

   global_y = 0 ;

   local_extent_x = 25 ;

   local_extent_y = 15 ;
  } // group bounding_boxes
}
```

Then use `ncgen` to make the netcdf file `partition_metadata_1.nc`
```
ncgen partition_metadata_1.cdl -o partition_metadata_1.nc
```
  • Loading branch information
@TomMelt
TomMelt authored Dec 4, 2023
2 parents c64bb14 + 3c0d866 commit 476e2cc
Show file tree
Hide file tree
Showing 9 changed files with 674 additions and 41 deletions.
42 changes: 20 additions & 22 deletions core/src/RectGridIO.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ void dimensionSetter(const netCDF::NcGroup& dataGroup, const std::string& fieldN
for (size_t d = 2; d < nDims; ++d) {
dims[d] = dataGroup.getVar(fieldName).getDim(d).getSize();
}
// The dimensions in the netCDF are in the reverse order compared to ModelArray
std::reverse(dims.begin(), dims.end());
// A special case for Type::Z: use NZLevels for the third dimension
if (type == ModelArray::Type::Z)
dims[2] = NZLevels::get();
ModelArray::setDimensions(type, dims);
}
#else
Expand Down Expand Up @@ -97,10 +102,10 @@ ModelState RectGridIO::getModelState(const std::string& filePath)
// on MPI decomposition
std::vector<size_t> start(2);
std::vector<size_t> size(2);
start[0] = metadata.localCornerX;
start[1] = metadata.localCornerY;
size[0] = metadata.localExtentX;
size[1] = metadata.localExtentY;
start[0] = metadata.localCornerY;
start[1] = metadata.localCornerX;
size[0] = metadata.localExtentY;
size[1] = metadata.localExtentX;

state.data[maskName] = ModelArray::HField();
dataGroup.getVar(maskName).getVar(start, size, &state.data[maskName][0]);
Expand All @@ -111,13 +116,6 @@ ModelState RectGridIO::getModelState(const std::string& filePath)
state.data[hsnowName] = ModelArray::HField();
dataGroup.getVar(hsnowName).getVar(start, size, &state.data[hsnowName][0]);

// The domain is not decomposed in z direction so set the extend in this direction
// to full range
start.push_back(0);
size.push_back(dataGroup.getDim("nLayers").getSize());

state.data[ticeName] = ModelArray::ZField();
dataGroup.getVar(ticeName).getVar(start, size, &state.data[ticeName][0]);

#else
// Get the sizes of the four types of field
Expand All @@ -138,14 +136,16 @@ ModelState RectGridIO::getModelState(const std::string& filePath)
dataGroup.getVar(ciceName).getVar(&state.data[ciceName][0]);
state.data[hsnowName] = ModelArray::HField();
dataGroup.getVar(hsnowName).getVar(&state.data[hsnowName][0]);
#endif
// Z direction is outside MPI ifdef as the domain is never decomposed in this direction

// Since the ZFierld might not have the same dimensions as the tice field
// in the file, a little more work is required.
state.data[ticeName] = ModelArray::ZField();
std::vector<size_t> startVector = { 0, 0, 0 };
std::vector<size_t> zArrayDims = ModelArray::dimensions(ModelArray::Type::Z);
std::reverse(zArrayDims.begin(), zArrayDims.end());
dataGroup.getVar(ticeName).getVar(startVector, zArrayDims, &state.data[ticeName][0]);
#endif

ncFile.close();
return state;
Expand Down Expand Up @@ -211,16 +211,14 @@ void RectGridIO::dumpModelState(const ModelState& state, const ModelMetadata& me
std::vector<netCDF::NcDim> dims2 = { yDim, xDim };
std::vector<netCDF::NcDim> dims3 = { zDim, yDim, xDim };
#ifdef USE_MPI
// Set the origins and extensions for reading 3D data based
// on MPI decomposition
std::vector<size_t> start3 = { 0, static_cast<size_t>(metadata.localCornerY), static_cast<size_t>(metadata.localCornerX) };
std::vector<size_t> size3 = { static_cast<size_t>(nz), static_cast<size_t>(metadata.localExtentY), static_cast<size_t>(metadata.localExtentX) };
// Set the origins and extensions for reading 2D data based
// on MPI decomposition
std::vector<size_t> start(3);
std::vector<size_t> size(3);
start[0] = metadata.localCornerX;
start[1] = metadata.localCornerY;
start[2] = 0;
size[0] = metadata.localExtentX;
size[1] = metadata.localExtentY;
size[2] = nz;
std::vector<size_t> start2(start3.begin() + 1, start3.end());
std::vector<size_t> size2(size3.begin() + 1, size3.end());
#endif

for (const auto entry : state.data) {
Expand All @@ -229,15 +227,15 @@ void RectGridIO::dumpModelState(const ModelState& state, const ModelMetadata& me
netCDF::NcVar var(dataGroup.addVar(name, netCDF::ncDouble, dims2));
var.putAtt(mdiName, netCDF::ncDouble, MissingData::value);
#ifdef USE_MPI
var.putVar(start, size, entry.second.getData());
var.putVar(start2, size2, entry.second.getData());
#else
var.putVar(entry.second.getData());
#endif
} else if (entry.second.getType() == ModelArray::Type::Z && entry.second.trueSize() > 0) {
netCDF::NcVar var(dataGroup.addVar(name, netCDF::ncDouble, dims3));
var.putAtt(mdiName, netCDF::ncDouble, MissingData::value);
#ifdef USE_MPI
var.putVar(start, size, entry.second.getData());
var.putVar(start3, size3, entry.second.getData());
#else
var.putVar(entry.second.getData());
#endif
Expand Down
36 changes: 26 additions & 10 deletions core/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,25 @@ target_link_libraries(testModelArray PRIVATE doctest::doctest Eigen3::Eigen)

set(MODEL_INCLUDE_DIR "../../core/src/discontinuousgalerkin")

add_executable(testRectGrid "RectGrid_test.cpp")
target_include_directories(testRectGrid PRIVATE ${MODEL_INCLUDE_DIR})
target_link_libraries(testRectGrid PRIVATE nextsimlib doctest::doctest)

add_executable(testParaGrid "ParaGrid_test.cpp")
target_include_directories(testParaGrid PRIVATE ${MODEL_INCLUDE_DIR})
target_link_libraries(testParaGrid PRIVATE nextsimlib doctest::doctest)
if(ENABLE_MPI)
add_executable(testRectGrid_MPI "RectGrid_test.cpp" "MainMPI.cpp")
target_compile_definitions(testRectGrid_MPI PRIVATE USE_MPI)
target_include_directories(testRectGrid_MPI PRIVATE ${MODEL_INCLUDE_DIR})
target_link_libraries(testRectGrid_MPI PRIVATE nextsimlib doctest::doctest)
else()
add_executable(testRectGrid "RectGrid_test.cpp")
target_include_directories(testRectGrid PRIVATE ${MODEL_INCLUDE_DIR})
target_link_libraries(testRectGrid PRIVATE nextsimlib doctest::doctest)
endif()

if(ENABLE_MPI)
message(WARNING "testParaGrid has been temporarily disabled when running with MPI enabled")
else()
add_executable(testParaGrid "ParaGrid_test.cpp")
target_compile_definitions(testParaGrid PRIVATE TEST_FILE_SOURCE=${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(testParaGrid PRIVATE ${MODEL_INCLUDE_DIR})
target_link_libraries(testParaGrid PRIVATE nextsimlib doctest::doctest)
endif()

add_executable(testModelComponent "ModelComponent_test.cpp")
target_include_directories(testModelComponent PRIVATE ${MODEL_INCLUDE_DIR})
Expand All @@ -62,9 +74,13 @@ add_executable(testPrognosticData "PrognosticData_test.cpp" "DynamicsModuleForPD
target_include_directories(testPrognosticData PRIVATE ${PHYSICS_INCLUDE_DIRS} ${MODEL_INCLUDE_DIR})
target_link_libraries(testPrognosticData PRIVATE nextsimlib doctest::doctest)

add_executable(testConfigOutput "ConfigOutput_test.cpp")
target_include_directories(testConfigOutput PRIVATE ${MODEL_INCLUDE_DIR})
target_link_libraries(testConfigOutput PRIVATE nextsimlib doctest::doctest)
if(ENABLE_MPI)
message(WARNING "testConfigOutput has been temporarily disabled when running with MPI enabled")
else()
add_executable(testConfigOutput "ConfigOutput_test.cpp")
target_include_directories(testConfigOutput PRIVATE ${MODEL_INCLUDE_DIR})
target_link_libraries(testConfigOutput PRIVATE nextsimlib doctest::doctest)
endif()

add_executable(testMonthlyCubicBSpline
"MonthlyCubicBSpline_test.cpp"
Expand Down
19 changes: 19 additions & 0 deletions core/test/MainMPI.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#define DOCTEST_CONFIG_IMPLEMENT

#include <doctest/extensions/doctest_mpi.h>

int main(int argc, char** argv) {
doctest::mpi_init_thread(argc,argv,MPI_THREAD_MULTIPLE);

doctest::Context ctx;
ctx.setOption("reporters", "MpiConsoleReporter");
ctx.setOption("reporters", "MpiFileReporter");
ctx.setOption("force-colors", true);
ctx.applyCommandLine(argc, argv);

int test_result = ctx.run();

doctest::mpi_finalize();

return test_result;
}
24 changes: 24 additions & 0 deletions core/test/RectGrid_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,13 @@
* @author Tim Spain <timothy.spain@nersc.no>
*/

#ifdef USE_MPI
#include <doctest/extensions/doctest_mpi.h>
#else
#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
#include <doctest/doctest.h>
#endif


#include "include/CommonRestartMetadata.hpp"
#include "include/NZLevels.hpp"
Expand All @@ -18,10 +23,16 @@
#include <fstream>

const std::string filename = "RectGrid_test.nc";
const std::string partition_filename = "partition_metadata_1.nc";

namespace Nextsim {
TEST_SUITE_BEGIN("RectGrid");
#ifdef USE_MPI
// Number of ranks should not be hardcoded here
MPI_TEST_CASE("Write and read a ModelState-based RectGrid restart file", 1)
#else
TEST_CASE("Write and read a ModelState-based RectGrid restart file")
#endif
{
RectangularGrid grid;
grid.setIO(new RectGridIO(grid));
Expand Down Expand Up @@ -68,6 +79,15 @@ TEST_CASE("Write and read a ModelState-based RectGrid restart file")
ModelMetadata metadata;
metadata.setTime(TimePoint("2000-01-01T00:00:00Z"));

#ifdef USE_MPI
metadata.setMpiMetadata(test_comm);
metadata.globalExtentX = nx;
metadata.globalExtentY = ny;
metadata.localCornerX = 0;
metadata.localCornerY = 0;
metadata.localExtentX = nx;
metadata.localExtentY = ny;
#endif
grid.dumpModelState(state, metadata, filename);

ModelArray::setDimensions(ModelArray::Type::H, { 1, 1 });
Expand All @@ -77,7 +97,11 @@ TEST_CASE("Write and read a ModelState-based RectGrid restart file")
size_t targetY = 7;

gridIn.setIO(new RectGridIO(grid));
#ifdef USE_MPI
ModelState ms = gridIn.getModelState(filename, partition_filename, metadata);
#else
ModelState ms = gridIn.getModelState(filename);
#endif

REQUIRE(ModelArray::dimensions(ModelArray::Type::H)[0] == nx);
REQUIRE(ModelArray::dimensions(ModelArray::Type::H)[1] == ny);
Expand Down
33 changes: 24 additions & 9 deletions lib/doctest/doctest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
//
// doctest.h - the lightest feature-rich C++ single-header testing framework for unit tests and TDD
//
// Copyright (c) 2016-2021 Viktor Kirilov
// Copyright (c) 2016-2023 Viktor Kirilov
//
// Distributed under the MIT Software License
// See accompanying file LICENSE.txt or copy at
Expand Down Expand Up @@ -48,7 +48,7 @@

#define DOCTEST_VERSION_MAJOR 2
#define DOCTEST_VERSION_MINOR 4
#define DOCTEST_VERSION_PATCH 10
#define DOCTEST_VERSION_PATCH 11

// util we need here
#define DOCTEST_TOSTR_IMPL(x) #x
Expand Down Expand Up @@ -1313,9 +1313,9 @@ namespace detail {
template<class T, unsigned N> struct decay_array<T[N]> { using type = T*; };
template<class T> struct decay_array<T[]> { using type = T*; };

template<class T> struct not_char_pointer { static DOCTEST_CONSTEXPR value = 1; };
template<> struct not_char_pointer<char*> { static DOCTEST_CONSTEXPR value = 0; };
template<> struct not_char_pointer<const char*> { static DOCTEST_CONSTEXPR value = 0; };
template<class T> struct not_char_pointer { static DOCTEST_CONSTEXPR int value = 1; };
template<> struct not_char_pointer<char*> { static DOCTEST_CONSTEXPR int value = 0; };
template<> struct not_char_pointer<const char*> { static DOCTEST_CONSTEXPR int value = 0; };

template<class T> struct can_use_op : public not_char_pointer<typename decay_array<T>::type> {};
#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING
Expand Down Expand Up @@ -5906,7 +5906,22 @@ namespace {
testCaseData.addFailure(rb.m_decomp.c_str(), assertString(rb.m_at), os.str());
}

void log_message(const MessageData&) override {}
void log_message(const MessageData& mb) override {
if(mb.m_severity & assertType::is_warn) // report only failures
return;

DOCTEST_LOCK_MUTEX(mutex)

std::ostringstream os;
os << skipPathFromFilename(mb.m_file) << (opt.gnu_file_line ? ":" : "(")
<< line(mb.m_line) << (opt.gnu_file_line ? ":" : "):") << std::endl;

os << mb.m_string.c_str() << "\n";
log_contexts(os);

testCaseData.addFailure(mb.m_string.c_str(),
mb.m_severity & assertType::is_check ? "FAIL_CHECK" : "FAIL", os.str());
}

void test_case_skipped(const TestCaseData&) override {}

Expand Down Expand Up @@ -6246,9 +6261,9 @@ namespace {
separator_to_stream();
s << std::dec;

auto totwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters, static_cast<unsigned>(p.numAsserts))) + 1)));
auto passwidth = int(std::ceil(log10((std::max(p.numTestCasesPassingFilters - p.numTestCasesFailed, static_cast<unsigned>(p.numAsserts - p.numAssertsFailed))) + 1)));
auto failwidth = int(std::ceil(log10((std::max(p.numTestCasesFailed, static_cast<unsigned>(p.numAssertsFailed))) + 1)));
auto totwidth = int(std::ceil(log10(static_cast<double>(std::max(p.numTestCasesPassingFilters, static_cast<unsigned>(p.numAsserts))) + 1)));
auto passwidth = int(std::ceil(log10(static_cast<double>(std::max(p.numTestCasesPassingFilters - p.numTestCasesFailed, static_cast<unsigned>(p.numAsserts - p.numAssertsFailed))) + 1)));
auto failwidth = int(std::ceil(log10(static_cast<double>(std::max(p.numTestCasesFailed, static_cast<unsigned>(p.numAssertsFailed))) + 1)));
const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
s << Color::Cyan << "[doctest] " << Color::None << "test cases: " << std::setw(totwidth)
<< p.numTestCasesPassingFilters << " | "
Expand Down
Loading

0 comments on commit 476e2cc

Please sign in to comment.