From 9be2161240b7ca3756701dd1842c2a17be181f5d Mon Sep 17 00:00:00 2001 From: melt Date: Tue, 21 May 2024 13:06:54 +0100 Subject: [PATCH 01/25] modify paragrid input to new format --- run/make_init_para24x30.py | 89 ++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/run/make_init_para24x30.py b/run/make_init_para24x30.py index e80029895..9353f739d 100644 --- a/run/make_init_para24x30.py +++ b/run/make_init_para24x30.py @@ -1,8 +1,7 @@ +import time + import netCDF4 import numpy as np -import numpy.ma as ma -import time -import math # Creates a 24x30 ParaGrid restart file filled with data from TOPAZ on 2010-01-01 if __name__ == "__main__": @@ -11,21 +10,19 @@ nfirst = 24 nsecond = 30 nLayers = 1 - ncg = 1 - n_dg = 1 + n_dg = 3 n_dgstress = 3 n_coords = 2 - - + root = netCDF4.Dataset("init_para24x30.nc", "w", format="NETCDF4") - + structure_name = "parametric_rectangular" structgrp = root.createGroup("structure") structgrp.type = structure_name - + metagrp = root.createGroup("metadata") metagrp.type = structure_name - confgrp = metagrp.createGroup("configuration") # But add nothing to it + confgrp = metagrp.createGroup("configuration") # But add nothing to it timegrp = metagrp.createGroup("time") time_var = timegrp.createVariable("time", "i8") data_time = 1263204000 @@ -40,64 +37,70 @@ yDim = datagrp.createDimension("ydim", nfirst) xDim = datagrp.createDimension("xdim", nsecond) yVertexDim = datagrp.createDimension("yvertex", nfirst + 1) - xVertexDim = datagrp.createDimension("xvertex", nsecond+ 1) - ycg_dim = datagrp.createDimension("y_cg", nfirst * ncg + 1) - xcg_dim = datagrp.createDimension("x_cg", nsecond * ncg + 1) + xVertexDim = datagrp.createDimension("xvertex", nsecond + 1) dg_comp = datagrp.createDimension("dg_comp", n_dg) dgs_comp = datagrp.createDimension("dgstress_comp", n_dgstress) n_coords_comp = datagrp.createDimension("ncoords", n_coords) - field_dims = ("ydim", "xdim") + dg_dims = ("ydim", "xdim", "dg_comp") coord_dims = ("yvertex", "xvertex", "ncoords") zfield_dims = ("zdim", "ydim", "xdim") # Array coordinates - space = 25000. # 25 km in metres + space = 25000.0 # 25 km in metres node_x = np.zeros((nfirst + 1, nsecond + 1)) node_y = np.zeros((nfirst + 1, nsecond + 1)) - + x_vertex1d = np.arange(0, space * (nsecond + 1), space) - y_vertex1d = np.arange(0, space * (nfirst +1), space) - + y_vertex1d = np.arange(0, space * (nfirst + 1), space) + x_vertex = np.resize(x_vertex1d, (nfirst + 1, nsecond + 1)) y_vertex = np.resize(y_vertex1d, (nsecond + 1, nfirst + 1)).transpose() - + coords = datagrp.createVariable("coords", "f8", coord_dims) - coords[:,:,0] = x_vertex - coords[:,:,1] = y_vertex - + coords[:, :, 0] = x_vertex + coords[:, :, 1] = y_vertex + x_cell1d = np.arange(space * 0.5, space * (nsecond + 0.5), space) y_cell1d = np.arange(space * 0.5, space * (nfirst + 0.5), space) - + x_cell = np.resize(x_cell1d, (nfirst, nsecond)) y_cell = np.resize(y_cell1d, (nsecond, nfirst)).transpose() - + elem_x = datagrp.createVariable("x", "f8", field_dims) elem_x[:, :] = x_cell elem_y = datagrp.createVariable("y", "f8", field_dims) elem_y[:, :] = y_cell - + grid_azimuth = datagrp.createVariable("grid_azimuth", "f8", field_dims) # Return the grid azimuth to the range -180˚ to 180˚ - grid_azimuth[:, :] = 0. - + grid_azimuth[:, :] = 0.0 + # All sea, everywhere mask = datagrp.createVariable("mask", "f8", field_dims) mask[:, :] = 1 - + # Ice concentration and thickness - cice = datagrp.createVariable("cice", "f8", field_dims) - hice = datagrp.createVariable("hice", "f8", field_dims) - cice[:, :] = 0.96875 # 31/32 - hice[:, :] = 1.5 - + cice = datagrp.createVariable("cice", "f8", dg_dims) + hice = datagrp.createVariable("hice", "f8", dg_dims) + cice[:, :, :] = 0.0 + cice[:, :, 0] = 0.96875 # 31/32 + hice[:, :, 0] = 1.5 + hice[:, :, 1] = 2.0 + hice[:, :, 2] = 3.0 + # Snow thickness - hsnow = datagrp.createVariable("hsnow", "f8", field_dims) - hsnow[:, :] = 0.125 + hsnow = datagrp.createVariable("hsnow", "f8", dg_dims) + hsnow[:, :, :] = 0.0 + hsnow[:, :, 0] = 0.125 + + # damage + damage = datagrp.createVariable("damage", "f8", dg_dims) + damage[:, :, :] = 0.0 # SSS sss = datagrp.createVariable("sss", "f8", field_dims) - sss[:, :] = 32. + sss[:, :] = 32.0 mu = -0.055 @@ -107,15 +110,15 @@ # Ice temperature tice = datagrp.createVariable("tice", "f8", zfield_dims) - ice_melt = mu * 5 # Melting point of sea ice (salinity = 5) in ˚C + ice_melt = mu * 5 # Melting point of sea ice (salinity = 5) in ˚C # Tice outside the ice pack is the melting point of pure water ice, which is conveniently 0˚C tice[0, :, :] = ice_melt - + # Ice starts at rest - u = datagrp.createVariable("u", "f8", field_dims) - u[:, :] = 0 + u = datagrp.createVariable("u", "f8", dg_dims) + u[:, :, :] = 0 + + v = datagrp.createVariable("v", "f8", dg_dims) + v[:, :, :] = 0 - v = datagrp.createVariable("v", "f8", field_dims) - v[:, :] = 0 - root.close() From 8705f99ce9a19ba1adb87c32a34c52a40b884299 Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 22 May 2024 13:10:08 +0100 Subject: [PATCH 02/25] add mpi version of paragrid test to cmake --- core/test/CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index a8f41bf80..58702b30a 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -51,12 +51,19 @@ if(ENABLE_MPI) target_compile_definitions(testRectGrid_MPI3 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") target_include_directories(testRectGrid_MPI3 PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest) + # Generate partition file needed for MPI test from its dump add_custom_command(OUTPUT partition_metadata_3.nc COMMAND ncgen -b -o partition_metadata_3.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3.cdl DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3.cdl ) add_custom_target(generate_partition_file ALL DEPENDS partition_metadata_3.nc) + + add_executable(testParaGrid_MPI1 "ParaGrid_test.cpp" "MainMPI.cpp") + target_compile_definitions(testParaGrid_MPI1 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") + target_include_directories(testParaGrid_MPI1 PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") + target_link_libraries(testParaGrid_MPI1 PRIVATE nextsimlib doctest::doctest) + if(ENABLE_XIOS) FILE(CREATE_LINK "${CMAKE_SOURCE_DIR}/core/test/iodef.xml" "${CMAKE_CURRENT_BINARY_DIR}/iodef.xml" SYMBOLIC) add_custom_command(OUTPUT xios_test_input.nc @@ -115,11 +122,7 @@ else() target_compile_definitions(testRectGrid PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") target_include_directories(testRectGrid PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") target_link_libraries(testRectGrid PRIVATE nextsimlib doctest::doctest) -endif() -if(ENABLE_MPI) - message(WARNING "testParaGrid has been temporarily disabled when running with MPI enabled") -else() add_executable(testParaGrid "ParaGrid_test.cpp") target_compile_definitions(testParaGrid PRIVATE TEST_FILE_SOURCE=${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(testParaGrid PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") From 4429bdbed30a49c9fcae0161ef29e03cb2e04339 Mon Sep 17 00:00:00 2001 From: melt Date: Thu, 23 May 2024 18:11:55 +0100 Subject: [PATCH 03/25] mpi-ify ParaGrid add global_length, local_length, and start variables to `DimensionSpec` add logic to assign local dims and start in ParaGridIO update ParaGrid_test.cpp to handle MPI test wip: currently only runs for 1 MPI rank --- core/src/ModelArray.cpp | 25 +- core/src/ParaGridIO.cpp | 64 ++- core/src/include/ModelArray.hpp | 14 +- core/src/include/ParaGridIO.hpp | 8 + .../include/ParametricGrid.hpp | 8 +- core/test/ParaGrid_test.cpp | 510 ++++++++++-------- 6 files changed, 372 insertions(+), 257 deletions(-) diff --git a/core/src/ModelArray.cpp b/core/src/ModelArray.cpp index 954ac773a..522eac88d 100644 --- a/core/src/ModelArray.cpp +++ b/core/src/ModelArray.cpp @@ -188,7 +188,7 @@ void ModelArray::setDimensions(Type type, const MultiDim& newDims) { std::vector& dimSpecs = typeDimensions.at(type); for (size_t i = 0; i < dimSpecs.size(); ++i) { - definedDimensions.at(dimSpecs[i]).length = newDims[i]; + definedDimensions.at(dimSpecs[i]).local_length = newDims[i]; } validateMaps(); } @@ -200,9 +200,22 @@ void ModelArray::setNComponents(std::map cMap) } } -void ModelArray::setDimension(Dimension dim, size_t length) -{ - definedDimensions.at(dim).length = length; +#ifdef USE_MPI +void ModelArray::setDimension(Dimension dim, size_t global_length, size_t local_length, size_t start) +#else +void ModelArray::setDimension(Dimension dim, size_t global_length) +#endif +{ +#ifdef USE_MPI + definedDimensions.at(dim).global_length = global_length; + definedDimensions.at(dim).local_length = local_length; + definedDimensions.at(dim).start = start; +#else + // if MPI is not used then set the local_length to be the same as the global + definedDimensions.at(dim).global_length = global_length; + definedDimensions.at(dim).local_length = global_length; + definedDimensions.at(dim).start = 0; +#endif validateMaps(); } @@ -271,7 +284,7 @@ void ModelArray::DimensionMap::validate() std::vector& typeDims = entry.second; dims.resize(typeDims.size()); for (size_t i = 0; i < typeDims.size(); ++i) { - dims[i] = definedDimensions.at(typeDims[i]).length; + dims[i] = definedDimensions.at(typeDims[i]).local_length; } } } @@ -282,7 +295,7 @@ void ModelArray::SizeMap::validate() size_t size = 1; std::vector& typeDims = entry.second; for (size_t i = 0; i < typeDims.size(); ++i) { - size *= definedDimensions.at(typeDims[i]).length; + size *= definedDimensions.at(typeDims[i]).local_length; } m_sizes.at(entry.first) = size; } diff --git a/core/src/ParaGridIO.cpp b/core/src/ParaGridIO.cpp index fa85822c8..3c3df897b 100644 --- a/core/src/ParaGridIO.cpp +++ b/core/src/ParaGridIO.cpp @@ -57,7 +57,11 @@ const std::map ParaGridIO::isDG = { }; std::map ParaGridIO::dimCompMap; +#ifdef USE_MPI +std::map ParaGridIO::openFiles; +#else std::map ParaGridIO::openFiles; +#endif std::map ParaGridIO::timeIndexByFile; void ParaGridIO::makeDimCompMap() @@ -77,12 +81,20 @@ void ParaGridIO::makeDimCompMap() ParaGridIO::~ParaGridIO() = default; +#ifdef USE_MPI +ModelState ParaGridIO::getModelState(const std::string& filePath, ModelMetadata& metadata) +#else ModelState ParaGridIO::getModelState(const std::string& filePath) +#endif { ModelState state; try { +#ifdef USE_MPI + netCDF::NcFilePar ncFile(filePath, netCDF::NcFile::read, metadata.mpiComm); +#else netCDF::NcFile ncFile(filePath, netCDF::NcFile::read); +#endif netCDF::NcGroup metaGroup(ncFile.getGroup(IStructure::metadataNodeName())); netCDF::NcGroup dataGroup(ncFile.getGroup(IStructure::dataNodeName())); @@ -110,9 +122,39 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) if (entry.first == ModelArray::Dimension::Z) { // A special case, as the number of levels in the file might not be // the number that the selected ice thermodynamics requires. - ModelArray::setDimension(entry.first, NZLevels::get()); + ModelArray::setDimension(entry.first, NZLevels::get(), NZLevels::get(), 0); } else { + // this needs MPIifying +#ifdef USE_MPI + auto dimName = dim.getName(); + size_t local_length = 0; + size_t start = 0; + printf("dimName = %s\n", dimName.c_str()); + if (dimName.compare("x") == 0) { + local_length = metadata.localExtentX; + start = metadata.localCornerX; + } + else if (dimName.compare("y") == 0) { + local_length = metadata.localExtentY; + start = metadata.localCornerY; + } + else if (dimName.compare("xvertex") == 0) { + local_length = metadata.localExtentX + 1; + start = metadata.localCornerX; + } + else if (dimName.compare("yvertex") == 0) { + local_length = metadata.localExtentY + 1; + start = metadata.localCornerY; + } + else{ + local_length = dim.getSize(); + start = 0; + } + printf("dim.getSize() = %zu, local_length = %zu, start = %zu\n", dim.getSize() , local_length, start); + ModelArray::setDimension(entry.first, dim.getSize(), local_length, start); +#else ModelArray::setDimension(entry.first, dim.getSize()); +#endif } } @@ -194,7 +236,7 @@ ModelState ParaGridIO::readForcingTimeStatic( = ModelArray::typeDimensions.at(ModelArray::Type::H); for (auto riter = dimensions.rbegin(); riter != dimensions.rend(); ++riter) { indexArray.push_back(0); - extentArray.push_back(ModelArray::definedDimensions.at(*riter).length); + extentArray.push_back(ModelArray::definedDimensions.at(*riter).local_length); } for (const std::string& varName : forcings) { @@ -229,7 +271,7 @@ void ParaGridIO::dumpModelState( for (auto entry : ModelArray::definedDimensions) { ModelArray::Dimension dim = entry.first; size_t dimSz = (dimCompMap.count(dim)) ? ModelArray::nComponents(dimCompMap.at(dim)) - : dimSz = entry.second.length; + : dimSz = entry.second.local_length; ncFromMAMap[dim] = dataGroup.addDim(entry.second.name, dimSz); // TODO Do I need to add data, even if it is just integers 0...n-1? } @@ -277,12 +319,20 @@ void ParaGridIO::writeDiagnosticTime( size_t nt = (isNew) ? 0 : ++timeIndexByFile.at(filePath); if (isNew) { // Open a new file and emplace it in the map of open files. +#ifdef USE_MPI + openFiles.try_emplace(filePath, filePath, netCDF::NcFile::read, meta.mpiComm); +#else openFiles.try_emplace(filePath, filePath, netCDF::NcFile::replace); +#endif // Set the initial time to be zero (assigned above) timeIndexByFile[filePath] = nt; } // Get the file handle +#ifdef USE_MPI + netCDF::NcFilePar& ncFile = openFiles.at(filePath); +#else netCDF::NcFile& ncFile = openFiles.at(filePath); +#endif // Get the netCDF groups, creating them if necessary netCDF::NcGroup metaGroup = (isNew) ? ncFile.addGroup(IStructure::metadataNodeName()) @@ -303,7 +353,7 @@ void ParaGridIO::writeDiagnosticTime( for (auto entry : ModelArray::definedDimensions) { ModelArray::Dimension dim = entry.first; size_t dimSz = (dimCompMap.count(dim)) ? ModelArray::nComponents(dimCompMap.at(dim)) - : dimSz = entry.second.length; + : dimSz = entry.second.local_length; ncFromMAMap[dim] = (isNew) ? dataGroup.addDim(entry.second.name, dimSz) : dataGroup.getDim(entry.second.name); } @@ -334,7 +384,7 @@ void ParaGridIO::writeDiagnosticTime( ModelArray::Dimension& maDim = *iter; ncDims.push_back(ncFromMAMap.at(maDim)); indexArray.push_back(0); - extentArray.push_back(ModelArray::definedDimensions.at(maDim).length); + extentArray.push_back(ModelArray::definedDimensions.at(maDim).local_length); } dimMap[type] = ncDims; indexArrays[type] = indexArray; @@ -361,9 +411,9 @@ void ParaGridIO::writeDiagnosticTime( maskIndexes = { 0, 0 }; maskExtents = { ModelArray::definedDimensions .at(ModelArray::typeDimensions.at(ModelArray::Type::H)[0]) - .length, + .local_length, ModelArray::definedDimensions.at(ModelArray::typeDimensions.at(ModelArray::Type::H)[1]) - .length }; + .local_length }; } // Put the time axis variable diff --git a/core/src/include/ModelArray.hpp b/core/src/include/ModelArray.hpp index 2ae398445..068cc2265 100644 --- a/core/src/include/ModelArray.hpp +++ b/core/src/include/ModelArray.hpp @@ -57,7 +57,9 @@ class ModelArray { struct DimensionSpec { std::string name; std::string altName; - size_t length; + size_t global_length; + size_t local_length; + size_t start; }; typedef std::map> TypeDimensions; @@ -233,7 +235,7 @@ class ModelArray { //! Returns the size of the data array of this object. size_t trueSize() const { return m_data.rows(); } //! Returns the size of a dimension - static size_t size(Dimension dim) { return definedDimensions.at(dim).length; } + static size_t size(Dimension dim) { return definedDimensions.at(dim).local_length; } //! Returns a read-only pointer to the underlying data buffer. const double* getData() const { return m_data.data(); } @@ -273,7 +275,7 @@ class ModelArray { * @param dim The dimension to be altered. * @param length The new length of the dimension. */ - static void setDimension(Dimension dim, size_t length); + static void setDimension(Dimension dim, size_t global_length, size_t local_length=0, size_t size=0); //! Conditionally updates the size of the object data buffer to match the //! class specification. @@ -281,7 +283,7 @@ class ModelArray { { if (size() != trueSize()) { if (hasDoF(type)) { - m_data.resize(m_sz.at(type), definedDimensions.at(componentMap.at(type)).length); + m_data.resize(m_sz.at(type), definedDimensions.at(componentMap.at(type)).local_length); } else { m_data.resize(m_sz.at(type), Eigen::NoChange); } @@ -444,7 +446,7 @@ class ModelArray { static void setNComponents(Type type, size_t nComp) { if (hasDoF(type)) { - definedDimensions.at(componentMap.at(type)).length = nComp; + definedDimensions.at(componentMap.at(type)).local_length = nComp; } } @@ -537,7 +539,7 @@ class ModelArray { //! specified type of ModelArray. inline static size_t nComponents(const Type type) { - return (hasDoF(type)) ? definedDimensions.at(componentMap.at(type)).length : 1; + return (hasDoF(type)) ? definedDimensions.at(componentMap.at(type)).local_length : 1; } //! Returns whether this type of ModelArray has additional discontinuous //! Galerkin components. diff --git a/core/src/include/ParaGridIO.hpp b/core/src/include/ParaGridIO.hpp index 844efb738..55fdf7d1b 100644 --- a/core/src/include/ParaGridIO.hpp +++ b/core/src/include/ParaGridIO.hpp @@ -35,7 +35,11 @@ class ParaGridIO : public ParametricGrid::IParaGridIO { * Retrieves the ModelState from a restart file of the parametric_grid type. * @param filePath The file path containing the file to be read. */ +#ifdef USE_MPI + ModelState getModelState(const std::string& filePath, ModelMetadata& metadata) override; +#else ModelState getModelState(const std::string& filePath) override; +#endif /*! * @brief Writes the ModelState to a given file location from the provided @@ -101,7 +105,11 @@ class ParaGridIO : public ParametricGrid::IParaGridIO { // Existing or open files are a property of the computer outside the individual // class instance, so they are static. +#ifdef USE_MPI + static std::map openFiles; +#else static std::map openFiles; +#endif static std::map timeIndexByFile; }; diff --git a/core/src/modules/StructureModule/include/ParametricGrid.hpp b/core/src/modules/StructureModule/include/ParametricGrid.hpp index bf352b46a..56d34227f 100644 --- a/core/src/modules/StructureModule/include/ParametricGrid.hpp +++ b/core/src/modules/StructureModule/include/ParametricGrid.hpp @@ -38,7 +38,7 @@ class ParametricGrid : public IStructure { #ifdef USE_MPI ModelState getModelState(const std::string& filePath, ModelMetadata& metadata) override { - return pio ? pio->getModelState(filePath) : ModelState(); + return pio ? pio->getModelState(filePath, metadata) : ModelState(); } #else ModelState getModelState(const std::string& filePath) override @@ -62,7 +62,7 @@ class ParametricGrid : public IStructure { int nIceLayers() const override { - return ModelArray::definedDimensions.at(ModelArray::Dimension::Z).length; + return ModelArray::definedDimensions.at(ModelArray::Dimension::Z).local_length; }; class IParaGridIO { @@ -73,7 +73,11 @@ class ParametricGrid : public IStructure { } virtual ~IParaGridIO() = default; +#ifdef USE_MPI + virtual ModelState getModelState(const std::string& filePath, ModelMetadata& metadata) = 0; +#else virtual ModelState getModelState(const std::string& filePath) = 0; +#endif virtual void dumpModelState( const ModelState& state, const ModelMetadata& metadata, const std::string& filePath) = 0; diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index 2cf06dd9e..55ced18cd 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -5,8 +5,12 @@ * @author Tim Spain */ +#ifdef USE_MPI +#include +#else #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include +#endif #include "include/Configurator.hpp" #include "include/ConfiguredModule.hpp" @@ -26,8 +30,13 @@ #include #include -const std::string filename = "paraGrid_test.nc"; +const std::string test_files_dir = TEST_FILES_DIR; +const std::string filename = test_files_dir + "/paraGrid_test.nc"; const std::string diagFile = "paraGrid_diag.nc"; +const std::string date_string = "2000-01-01T00:00:00Z"; +#ifdef USE_MPI +const std::string partition_filename = test_files_dir + "/partition_metadata_1.nc"; +#endif static const int DG = 3; static const int DGSTRESS = 6; @@ -38,7 +47,11 @@ namespace Nextsim { size_t c = 0; TEST_SUITE_BEGIN("ParaGrid"); +#ifdef USE_MPI +MPI_TEST_CASE("Write and read a ModelState-based ParaGrid restart file", 1) +#else TEST_CASE("Write and read a ModelState-based ParaGrid restart file") +#endif { Module::setImplementation("Nextsim::ParametricGrid"); @@ -53,19 +66,23 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") size_t ny = 15; size_t nz = 3; NZLevels::set(nz); - size_t nxcg = CG * nx + 1; - size_t nycg = CG * ny + 1; double yFactor = 0.01; double xFactor = 0.0001; +#ifdef USE_MPI + ModelArray::setDimension(ModelArray::Dimension::X, nx, nx, 0); + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1, nx + 1, 0); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1, ny + 1, 0); +#else ModelArray::setDimension(ModelArray::Dimension::X, nx); ModelArray::setDimension(ModelArray::Dimension::Y, ny); ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1); ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1); - ModelArray::setDimension(ModelArray::Dimension::XCG, nxcg); - ModelArray::setDimension(ModelArray::Dimension::YCG, nycg); +#endif ModelArray::setNComponents(ModelArray::Type::DG, DG); ModelArray::setNComponents(ModelArray::Type::DGSTRESS, DGSTRESS); @@ -89,7 +106,10 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") DGField hice = fractionalDG + 10; DGField cice = fractionalDG + 20; - HField hsnow = fractional + 30; + DGField hsnow = fractionalDG + 30; + DGField tom = fractionalDG + 40; + DGField damage = fractionalDG * 0.; + HField sss = fractional; ZField tice(ModelArray::Type::Z); tice.resize(); for (size_t i = 0; i < ModelArray::size(ModelArray::Type::H); ++i) { @@ -104,10 +124,10 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") double scale = 1e5; // Vertex coordinates - for (size_t i = 0; i < ModelArray::definedDimensions.at(ModelArray::Dimension::XVERTEX).length; + for (size_t i = 0; i < ModelArray::definedDimensions.at(ModelArray::Dimension::XVERTEX).local_length; ++i) { for (size_t j = 0; - j < ModelArray::definedDimensions.at(ModelArray::Dimension::YVERTEX).length; ++j) { + j < ModelArray::definedDimensions.at(ModelArray::Dimension::YVERTEX).local_length; ++j) { double x = i - 0.5 - nx / 2; double y = j - 0.5 - ny / 2; coordinates.components({ i, j })[0] = x * scale; @@ -141,6 +161,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") { hiceName, hice }, { ciceName, cice }, { hsnowName, hsnow }, + { damageName, damage }, { ticeName, tice }, }, {} }; @@ -171,8 +192,6 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") ModelArray::setDimension(ModelArray::Dimension::Z, 1); ModelArray::setDimension(ModelArray::Dimension::XVERTEX, 1); ModelArray::setDimension(ModelArray::Dimension::YVERTEX, 1); - ModelArray::setDimension(ModelArray::Dimension::XCG, 1); - ModelArray::setDimension(ModelArray::Dimension::YCG, 1); // In the full model numbers of DG components are set at compile time, so they are not reset REQUIRE(ModelArray::nComponents(ModelArray::Type::DG) == DG); REQUIRE(ModelArray::nComponents(ModelArray::Type::VERTEX) == ModelArray::nCoords); @@ -181,7 +200,13 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") ParaGridIO* readIO = new ParaGridIO(gridIn); gridIn.setIO(readIO); +#ifdef USE_MPI + ModelMetadata metadataIn(partition_filename, test_comm); + metadataIn.setTime(TimePoint(date_string)); + ModelState ms = gridIn.getModelState(filename, metadataIn); +#else ModelState ms = gridIn.getModelState(filename); +#endif REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == nx); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[1] == ny); @@ -225,234 +250,247 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") REQUIRE(ms.data.count(gridAzimuthName) > 0); REQUIRE(ms.data.at(gridAzimuthName)(0, 0) == gridAzimuth0); - std::filesystem::remove(filename); -} - -TEST_CASE("Write a diagnostic ParaGrid file") -{ - Module::setImplementation("Nextsim::ParametricGrid"); - - REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); - - std::filesystem::remove(diagFile); - - ParametricGrid grid; - ParaGridIO* pio = new ParaGridIO(grid); - grid.setIO(pio); - - // Set the dimension lengths - size_t nx = 30; - size_t ny = 20; - size_t nz = 3; - NZLevels::set(nz); - size_t nxcg = CG * nx + 1; - size_t nycg = CG * ny + 1; - - double yFactor = 0.01; - double xFactor = 0.0001; - - ModelArray::setDimension(ModelArray::Dimension::X, nx); - ModelArray::setDimension(ModelArray::Dimension::Y, ny); - ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); - ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1); - ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1); - ModelArray::setDimension(ModelArray::Dimension::XCG, nxcg); - ModelArray::setDimension(ModelArray::Dimension::YCG, nycg); - - ModelArray::setNComponents(ModelArray::Type::DG, DG); - ModelArray::setNComponents(ModelArray::Type::DGSTRESS, DGSTRESS); - ModelArray::setNComponents(ModelArray::Type::VERTEX, ModelArray::nCoords); - - HField fractional(ModelArray::Type::H); - DGField fractionalDG(ModelArray::Type::DG); - HField mask(ModelArray::Type::H); - fractional.resize(); - fractionalDG.resize(); - for (size_t j = 0; j < ny; ++j) { - for (size_t i = 0; i < nx; ++i) { - fractional(i, j) = j * yFactor + i * xFactor; - mask(i, j) = fractional(i, j); - // = (i - nx / 2) * (i - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * - // ny) ? 0 : 1; - for (size_t d = 0; d < DG; ++d) { - fractionalDG.components({ i, j })[d] = fractional(i, j) + d; - } - } - } - double prec = 1e-9; - REQUIRE(fractional(12, 12) - fractional(11, 12) == doctest::Approx(xFactor).epsilon(prec)); - REQUIRE(fractional(12, 12) - fractional(12, 11) == doctest::Approx(yFactor).epsilon(prec)); - - REQUIRE(fractionalDG(12, 12) - fractionalDG(11, 12) == doctest::Approx(xFactor).epsilon(prec)); - REQUIRE(fractionalDG(12, 12) - fractionalDG(12, 11) == doctest::Approx(yFactor).epsilon(prec)); - - DGField hice = fractionalDG + 10; - DGField cice = fractionalDG + 20; - - VertexField coordinates(ModelArray::Type::VERTEX); - coordinates.resize(); - // Planar coordinates - double scale = 1e5; - - for (size_t i = 0; i < ModelArray::definedDimensions.at(ModelArray::Dimension::XVERTEX).length; - ++i) { - for (size_t j = 0; - j < ModelArray::definedDimensions.at(ModelArray::Dimension::YVERTEX).length; ++j) { - double x = i - 0.5 - nx / 2; - double y = j - 0.5 - ny / 2; - coordinates.components({ i, j })[0] = x * scale; - coordinates.components({ i, j })[1] = y * scale; - } - } - - REQUIRE(coordinates.components({ 12, 13 })[0] - coordinates.components({ 11, 13 })[0] == scale); - REQUIRE(coordinates.components({ 12, 13 })[1] - coordinates.components({ 12, 12 })[1] == scale); - - HField x; - HField y; - x.resize(); - y.resize(); - // Element coordinates - for (size_t j = 0; j < ModelArray::size(ModelArray::Dimension::Y); ++j) { - double yy = scale * (j - ny / 2); - for (size_t i = 0; i < ModelArray::size(ModelArray::Dimension::X); ++i) { - double xx = scale * (i - nx / 2); - x(i, j) = xx; - y(i, j) = yy; - } - } - - HField gridAzimuth; - double gridAzimuth0 = 45.; - gridAzimuth = gridAzimuth0; - - ModelState state = { { - { maskName, mask }, - { hiceName, hice }, - { ciceName, cice }, - }, - {} }; - - // A model state to set the coordinates in the metadata object - ModelState coordState = { { - { xName, x }, - { yName, y }, - { coordsName, coordinates }, - { gridAzimuthName, gridAzimuth }, - }, - {} }; - - ModelMetadata metadata; - metadata.setTime(TimePoint("2000-01-01T00:00:00Z")); - // The coordinates are passed through the metadata object as affix - // coordinates is the correct way to add coordinates to a ModelState - metadata.extractCoordinates(coordState); - metadata.affixCoordinates(state); - - grid.dumpModelState(state, metadata, diagFile, false); - - for (int t = 1; t < 5; ++t) { - hice += 100; - cice += 100; - state = { { - { hiceName, hice }, - { ciceName, cice }, - }, - {} }; - metadata.incrementTime(Duration(3600)); - - grid.dumpModelState(state, metadata, diagFile, false); - } - pio->close(diagFile); - - // What do we have in the file? - netCDF::NcFile ncFile(diagFile, netCDF::NcFile::read); - - REQUIRE(ncFile.getGroups().size() == 3); - netCDF::NcGroup structGrp(ncFile.getGroup(IStructure::structureNodeName())); - netCDF::NcGroup metaGrp(ncFile.getGroup(IStructure::metadataNodeName())); - netCDF::NcGroup dataGrp(ncFile.getGroup(IStructure::dataNodeName())); - - std::string structureType; - structGrp.getAtt(grid.typeNodeName()).getValues(structureType); - REQUIRE(structureType == grid.structureType()); - - // TODO test metadata - - // test data - REQUIRE(dataGrp.getVarCount() == 8); - netCDF::NcVar hiceVar = dataGrp.getVar(hiceName); - netCDF::NcVar ciceVar = dataGrp.getVar(ciceName); - netCDF::NcVar maskVar = dataGrp.getVar(maskName); - netCDF::NcVar timeVar = dataGrp.getVar(timeName); - - // hice - REQUIRE(hiceVar.getDimCount() == 4); - - // coordinates - REQUIRE(dataGrp.getVars().count(xName) > 0); - REQUIRE(dataGrp.getVars().count(yName) > 0); - REQUIRE(dataGrp.getVars().count(coordsName) > 0); - REQUIRE(dataGrp.getVars().count(gridAzimuthName) > 0); - - ncFile.close(); - - std::filesystem::remove(diagFile); + // std::filesystem::remove(filename); } -#define TO_STR(s) TO_STRI(s) -#define TO_STRI(s) #s -#ifndef TEST_FILE_SOURCE -#define TEST_FILE_SOURCE . -#endif - -TEST_CASE("Test array ordering") -{ - std::string inputFilename = "ParaGridIO_input_test.nc"; - - Module::setImplementation("Nextsim::ParametricGrid"); - - REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); - - size_t nx = 9; - size_t ny = 11; - NZLevels::set(1); - - double xFactor = 10; - - ModelArray::setDimension(ModelArray::Dimension::X, nx); - ModelArray::setDimension(ModelArray::Dimension::Y, ny); - ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); - - HField index2d(ModelArray::Type::H); - index2d.resize(); - std::string fieldName = "index2d"; - std::set fields = { fieldName }; - TimePoint time; - - ModelState state = ParaGridIO::readForcingTimeStatic( - fields, time, TO_STR(TEST_FILE_SOURCE) + std::string("/") + inputFilename); - REQUIRE(state.data.count(fieldName) > 0); - index2d = state.data.at(fieldName); - REQUIRE(index2d(3, 5) == 35); - // And that's all that's needed -} - -#undef TO_STR -#undef TO_STRI - -TEST_CASE("Check an exception is thrown for an invalid file name") -{ - ParametricGrid gridIn; - ParaGridIO* readIO = new ParaGridIO(gridIn); - gridIn.setIO(readIO); - - ModelState state; - - // MD5 hash of the current output of $ date - std::string longRandomFilename("a44f5cc1f7934a8ae8dd03a95308745d.nc"); - REQUIRE_THROWS(state = gridIn.getModelState(longRandomFilename)); -} +// #ifdef USE_MPI +// MPI_TEST_CASE("Write a diagnostic ParaGrid file", 1) +// #else +// TEST_CASE("Write a diagnostic ParaGrid file") +// #endif +// { +// Module::setImplementation("Nextsim::ParametricGrid"); + +// REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); + +// std::filesystem::remove(diagFile); + +// ParametricGrid grid; +// ParaGridIO* pio = new ParaGridIO(grid); +// grid.setIO(pio); + +// // Set the dimension lengths +// size_t nx = 30; +// size_t ny = 20; +// size_t nz = 3; +// NZLevels::set(nz); +// size_t nxcg = CG * nx + 1; +// size_t nycg = CG * ny + 1; + +// double yFactor = 0.01; +// double xFactor = 0.0001; + +// ModelArray::setDimension(ModelArray::Dimension::X, nx); +// ModelArray::setDimension(ModelArray::Dimension::Y, ny); +// ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); +// ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1); +// ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1); +// ModelArray::setDimension(ModelArray::Dimension::XCG, nxcg); +// ModelArray::setDimension(ModelArray::Dimension::YCG, nycg); + +// ModelArray::setNComponents(ModelArray::Type::DG, DG); +// ModelArray::setNComponents(ModelArray::Type::DGSTRESS, DGSTRESS); +// ModelArray::setNComponents(ModelArray::Type::VERTEX, ModelArray::nCoords); + +// HField fractional(ModelArray::Type::H); +// DGField fractionalDG(ModelArray::Type::DG); +// HField mask(ModelArray::Type::H); +// fractional.resize(); +// fractionalDG.resize(); +// for (size_t j = 0; j < ny; ++j) { +// for (size_t i = 0; i < nx; ++i) { +// fractional(i, j) = j * yFactor + i * xFactor; +// mask(i, j) = fractional(i, j); +// // = (i - nx / 2) * (i - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * +// // ny) ? 0 : 1; +// for (size_t d = 0; d < DG; ++d) { +// fractionalDG.components({ i, j })[d] = fractional(i, j) + d; +// } +// } +// } +// double prec = 1e-9; +// REQUIRE(fractional(12, 12) - fractional(11, 12) == doctest::Approx(xFactor).epsilon(prec)); +// REQUIRE(fractional(12, 12) - fractional(12, 11) == doctest::Approx(yFactor).epsilon(prec)); + +// REQUIRE(fractionalDG(12, 12) - fractionalDG(11, 12) == doctest::Approx(xFactor).epsilon(prec)); +// REQUIRE(fractionalDG(12, 12) - fractionalDG(12, 11) == doctest::Approx(yFactor).epsilon(prec)); + +// DGField hice = fractionalDG + 10; +// DGField cice = fractionalDG + 20; + +// VertexField coordinates(ModelArray::Type::VERTEX); +// coordinates.resize(); +// // Planar coordinates +// double scale = 1e5; + +// for (size_t i = 0; i < ModelArray::definedDimensions.at(ModelArray::Dimension::XVERTEX).length; +// ++i) { +// for (size_t j = 0; +// j < ModelArray::definedDimensions.at(ModelArray::Dimension::YVERTEX).length; ++j) { +// double x = i - 0.5 - nx / 2; +// double y = j - 0.5 - ny / 2; +// coordinates.components({ i, j })[0] = x * scale; +// coordinates.components({ i, j })[1] = y * scale; +// } +// } + +// REQUIRE(coordinates.components({ 12, 13 })[0] - coordinates.components({ 11, 13 })[0] == scale); +// REQUIRE(coordinates.components({ 12, 13 })[1] - coordinates.components({ 12, 12 })[1] == scale); + +// HField x; +// HField y; +// x.resize(); +// y.resize(); +// // Element coordinates +// for (size_t j = 0; j < ModelArray::size(ModelArray::Dimension::Y); ++j) { +// double yy = scale * (j - ny / 2); +// for (size_t i = 0; i < ModelArray::size(ModelArray::Dimension::X); ++i) { +// double xx = scale * (i - nx / 2); +// x(i, j) = xx; +// y(i, j) = yy; +// } +// } + +// HField gridAzimuth; +// double gridAzimuth0 = 45.; +// gridAzimuth = gridAzimuth0; + +// ModelState state = { { +// { maskName, mask }, +// { hiceName, hice }, +// { ciceName, cice }, +// }, +// {} }; + +// // A model state to set the coordinates in the metadata object +// ModelState coordState = { { +// { xName, x }, +// { yName, y }, +// { coordsName, coordinates }, +// { gridAzimuthName, gridAzimuth }, +// }, +// {} }; + +// ModelMetadata metadata; +// metadata.setTime(TimePoint("2000-01-01T00:00:00Z")); +// // The coordinates are passed through the metadata object as affix +// // coordinates is the correct way to add coordinates to a ModelState +// metadata.extractCoordinates(coordState); +// metadata.affixCoordinates(state); + +// grid.dumpModelState(state, metadata, diagFile, false); + +// for (int t = 1; t < 5; ++t) { +// hice += 100; +// cice += 100; +// state = { { +// { hiceName, hice }, +// { ciceName, cice }, +// }, +// {} }; +// metadata.incrementTime(Duration(3600)); + +// grid.dumpModelState(state, metadata, diagFile, false); +// } +// pio->close(diagFile); + +// // What do we have in the file? +// netCDF::NcFile ncFile(diagFile, netCDF::NcFile::read); + +// REQUIRE(ncFile.getGroups().size() == 3); +// netCDF::NcGroup structGrp(ncFile.getGroup(IStructure::structureNodeName())); +// netCDF::NcGroup metaGrp(ncFile.getGroup(IStructure::metadataNodeName())); +// netCDF::NcGroup dataGrp(ncFile.getGroup(IStructure::dataNodeName())); + +// std::string structureType; +// structGrp.getAtt(grid.typeNodeName()).getValues(structureType); +// REQUIRE(structureType == grid.structureType()); + +// // TODO test metadata + +// // test data +// REQUIRE(dataGrp.getVarCount() == 8); +// netCDF::NcVar hiceVar = dataGrp.getVar(hiceName); +// netCDF::NcVar ciceVar = dataGrp.getVar(ciceName); +// netCDF::NcVar maskVar = dataGrp.getVar(maskName); +// netCDF::NcVar timeVar = dataGrp.getVar(timeName); + +// // hice +// REQUIRE(hiceVar.getDimCount() == 4); + +// // coordinates +// REQUIRE(dataGrp.getVars().count(xName) > 0); +// REQUIRE(dataGrp.getVars().count(yName) > 0); +// REQUIRE(dataGrp.getVars().count(coordsName) > 0); +// REQUIRE(dataGrp.getVars().count(gridAzimuthName) > 0); + +// ncFile.close(); + +// std::filesystem::remove(diagFile); +// } + +// #define TO_STR(s) TO_STRI(s) +// #define TO_STRI(s) #s +// #ifndef TEST_FILE_SOURCE +// #define TEST_FILE_SOURCE . +// #endif + +// #ifdef USE_MPI +// MPI_TEST_CASE("Test array ordering", 1) +// #else +// TEST_CASE("Test array ordering") +// #endif +// { +// std::string inputFilename = "ParaGridIO_input_test.nc"; + +// Module::setImplementation("Nextsim::ParametricGrid"); + +// REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); + +// size_t nx = 9; +// size_t ny = 11; +// NZLevels::set(1); + +// double xFactor = 10; + +// ModelArray::setDimension(ModelArray::Dimension::X, nx); +// ModelArray::setDimension(ModelArray::Dimension::Y, ny); +// ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); + +// HField index2d(ModelArray::Type::H); +// index2d.resize(); +// std::string fieldName = "index2d"; +// std::set fields = { fieldName }; +// TimePoint time; + +// ModelState state = ParaGridIO::readForcingTimeStatic( +// fields, time, TO_STR(TEST_FILE_SOURCE) + std::string("/") + inputFilename); +// REQUIRE(state.data.count(fieldName) > 0); +// index2d = state.data.at(fieldName); +// REQUIRE(index2d(3, 5) == 35); +// // And that's all that's needed +// } + +// #undef TO_STR +// #undef TO_STRI + +// #ifdef USE_MPI +// MPI_TEST_CASE("Check an exception is thrown for an invalid file name", 1) +// #else +// TEST_CASE("Check an exception is thrown for an invalid file name") +// #endif +// { +// ParametricGrid gridIn; +// ParaGridIO* readIO = new ParaGridIO(gridIn); +// gridIn.setIO(readIO); + +// ModelState state; +// +// // MD5 hash of the current output of $ date +// std::string longRandomFilename("a44f5cc1f7934a8ae8dd03a95308745d.nc"); +// REQUIRE_THROWS(state = gridIn.getModelState(longRandomFilename)); + +// } TEST_CASE("Check if a file with the old dimension names can be read") { From 7a34d5355b35c12ded10bfe55730eadcf02d7b60 Mon Sep 17 00:00:00 2001 From: melt Date: Thu, 6 Jun 2024 19:52:58 +0100 Subject: [PATCH 04/25] fix cmake for paragrid test --- core/test/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 58702b30a..1d23a6c49 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -59,10 +59,10 @@ if(ENABLE_MPI) ) add_custom_target(generate_partition_file ALL DEPENDS partition_metadata_3.nc) - add_executable(testParaGrid_MPI1 "ParaGrid_test.cpp" "MainMPI.cpp") - target_compile_definitions(testParaGrid_MPI1 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") - target_include_directories(testParaGrid_MPI1 PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") - target_link_libraries(testParaGrid_MPI1 PRIVATE nextsimlib doctest::doctest) + add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp") + target_compile_definitions(testParaGrid_MPI2 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") + target_include_directories(testParaGrid_MPI2 PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") + target_link_libraries(testParaGrid_MPI2 PRIVATE nextsimlib doctest::doctest) if(ENABLE_XIOS) FILE(CREATE_LINK "${CMAKE_SOURCE_DIR}/core/test/iodef.xml" "${CMAKE_CURRENT_BINARY_DIR}/iodef.xml" SYMBOLIC) @@ -124,7 +124,7 @@ else() target_link_libraries(testRectGrid PRIVATE nextsimlib doctest::doctest) add_executable(testParaGrid "ParaGrid_test.cpp") - target_compile_definitions(testParaGrid PRIVATE TEST_FILE_SOURCE=${CMAKE_CURRENT_SOURCE_DIR}) + target_compile_definitions(testParaGrid PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") target_include_directories(testParaGrid PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") target_link_libraries(testParaGrid PRIVATE nextsimlib doctest::doctest) file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/old_names.nc" From 211e75e60d0e0afa9a325af6c7eabd2b1a20333d Mon Sep 17 00:00:00 2001 From: melt Date: Thu, 6 Jun 2024 19:55:11 +0100 Subject: [PATCH 05/25] add MPI backend for ParaGrid IO add MPI backend and associated test for the IO functionality (get/dump model state) This currently passes but I still need to add MPI interface for the dynamics --- core/src/ParaGridIO.cpp | 83 ++++++++++++++++++-------- core/src/include/ModelArray.hpp | 5 ++ core/test/ParaGrid_test.cpp | 102 +++++++++++++++++++++----------- 3 files changed, 130 insertions(+), 60 deletions(-) diff --git a/core/src/ParaGridIO.cpp b/core/src/ParaGridIO.cpp index 3c3df897b..5bdf0c126 100644 --- a/core/src/ParaGridIO.cpp +++ b/core/src/ParaGridIO.cpp @@ -101,7 +101,8 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) // Dimensions and DG components std::multimap dimMap = dataGroup.getDims(); for (auto entry : ModelArray::definedDimensions) { - if (dimCompMap.count(entry.first) > 0) + auto dimType = entry.first; + if (dimCompMap.count(dimType) > 0) // TODO Assertions that DG in the file equals the compile time DG in the model. See // #205 continue; @@ -119,30 +120,33 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) std::string("No netCDF dimension found corresponding to the dimension named ") + dimensionSpec.name + std::string(" or ") + dimensionSpec.altName); } - if (entry.first == ModelArray::Dimension::Z) { + if (dimType == ModelArray::Dimension::Z) { // A special case, as the number of levels in the file might not be // the number that the selected ice thermodynamics requires. - ModelArray::setDimension(entry.first, NZLevels::get(), NZLevels::get(), 0); +#ifdef USE_MPI + ModelArray::setDimension(dimType, NZLevels::get(), NZLevels::get(), 0); +#else + ModelArray::setDimension(dimType, NZLevels::get()); +#endif } else { // this needs MPIifying #ifdef USE_MPI auto dimName = dim.getName(); size_t local_length = 0; size_t start = 0; - printf("dimName = %s\n", dimName.c_str()); - if (dimName.compare("x") == 0) { + if (dimType == ModelArray::Dimension::X) { local_length = metadata.localExtentX; start = metadata.localCornerX; } - else if (dimName.compare("y") == 0) { + else if (dimType == ModelArray::Dimension::Y) { local_length = metadata.localExtentY; start = metadata.localCornerY; } - else if (dimName.compare("xvertex") == 0) { + else if (dimType == ModelArray::Dimension::XVERTEX) { local_length = metadata.localExtentX + 1; start = metadata.localCornerX; } - else if (dimName.compare("yvertex") == 0) { + else if (dimType == ModelArray::Dimension::YVERTEX) { local_length = metadata.localExtentY + 1; start = metadata.localCornerY; } @@ -150,10 +154,9 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) local_length = dim.getSize(); start = 0; } - printf("dim.getSize() = %zu, local_length = %zu, start = %zu\n", dim.getSize() , local_length, start); - ModelArray::setDimension(entry.first, dim.getSize(), local_length, start); + ModelArray::setDimension(dimType, dim.getSize(), local_length, start); #else - ModelArray::setDimension(entry.first, dim.getSize()); + ModelArray::setDimension(dimType, dim.getSize()); #endif } } @@ -174,21 +177,30 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) std::string("No ModelArray::Type corresponds to the dimensional key ") + dimKey); } - ModelArray::Type newType = dimensionKeys.at(dimKey); - state.data[varName] = ModelArray(newType); + ModelArray::Type type = dimensionKeys.at(dimKey); + state.data[varName] = ModelArray(type); ModelArray& data = state.data.at(varName); data.resize(); - if (newType == ModelArray::Type::Z) { - std::vector startVector(ModelArray::nDimensions(newType), 0); - std::vector extentVector = ModelArray::dimensions(newType); - // Reverse the extent vector to go from logical (x, y, z) ordering - // of indexes to netCDF storage ordering. - std::reverse(extentVector.begin(), extentVector.end()); - var.getVar(startVector, extentVector, &data[0]); - } else { - var.getVar(&data[0]); + std::vector start; + std::vector count; + if (ModelArray::hasDoF(type)){ + auto ncomps = data.nComponents(); + start.push_back(0); + count.push_back(ncomps); } + for (ModelArray::Dimension dt : ModelArray::typeDimensions.at(type)){ + auto dim = ModelArray::definedDimensions.at(dt); + start.push_back(dim.start); + count.push_back(dim.local_length); + + } + // dims are looped in [dg], x, y, [z] order so start and count + // order must be reveresed to match order netcdf expects + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); + + var.getVar(start, count, &data[0]); } ncFile.close(); } catch (const netCDF::exceptions::NcException& nce) { @@ -259,7 +271,12 @@ ModelState ParaGridIO::readForcingTimeStatic( void ParaGridIO::dumpModelState( const ModelState& state, const ModelMetadata& metadata, const std::string& filePath) { + +#ifdef USE_MPI + netCDF::NcFilePar ncFile(filePath, netCDF::NcFile::replace, metadata.mpiComm); +#else netCDF::NcFile ncFile(filePath, netCDF::NcFile::replace); +#endif CommonRestartMetadata::writeStructureType(ncFile, metadata); netCDF::NcGroup metaGroup = ncFile.addGroup(IStructure::metadataNodeName()); @@ -271,7 +288,7 @@ void ParaGridIO::dumpModelState( for (auto entry : ModelArray::definedDimensions) { ModelArray::Dimension dim = entry.first; size_t dimSz = (dimCompMap.count(dim)) ? ModelArray::nComponents(dimCompMap.at(dim)) - : dimSz = entry.second.local_length; + : dimSz = entry.second.global_length; ncFromMAMap[dim] = dataGroup.addDim(entry.second.name, dimSz); // TODO Do I need to add data, even if it is just integers 0...n-1? } @@ -302,10 +319,28 @@ void ParaGridIO::dumpModelState( if (restartFields.count(entry.first)) { // Get the type, then relevant vector of NetCDF dimensions ModelArray::Type type = entry.second.getType(); + std::vector start; + std::vector count; + if (ModelArray::hasDoF(type)){ + auto ncomps = entry.second.nComponents(); + start.push_back(0); + count.push_back(ncomps); + } + for (ModelArray::Dimension dt : entry.second.typeDimensions.at(type)){ + auto dim = entry.second.definedDimensions.at(dt); + start.push_back(dim.start); + count.push_back(dim.local_length); + + } + // dims are looped in [dg], x, y, [z] order so start and count + // order must be reveresed to match order netcdf expects + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); + std::vector& ncDims = dimMap.at(type); netCDF::NcVar var(dataGroup.addVar(entry.first, netCDF::ncDouble, ncDims)); var.putAtt(mdiName, netCDF::ncDouble, MissingData::value); - var.putVar(entry.second.getData()); + var.putVar(start, count, entry.second.getData()); } } diff --git a/core/src/include/ModelArray.hpp b/core/src/include/ModelArray.hpp index 068cc2265..f9503dc37 100644 --- a/core/src/include/ModelArray.hpp +++ b/core/src/include/ModelArray.hpp @@ -275,7 +275,11 @@ class ModelArray { * @param dim The dimension to be altered. * @param length The new length of the dimension. */ +#ifdef USE_MPI static void setDimension(Dimension dim, size_t global_length, size_t local_length=0, size_t size=0); +#else + static void setDimension(Dimension dim, size_t global_length); +#endif //! Conditionally updates the size of the object data buffer to match the //! class specification. @@ -447,6 +451,7 @@ class ModelArray { { if (hasDoF(type)) { definedDimensions.at(componentMap.at(type)).local_length = nComp; + definedDimensions.at(componentMap.at(type)).global_length = nComp; } } diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index 55ced18cd..38cf8c791 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -35,7 +35,7 @@ const std::string filename = test_files_dir + "/paraGrid_test.nc"; const std::string diagFile = "paraGrid_diag.nc"; const std::string date_string = "2000-01-01T00:00:00Z"; #ifdef USE_MPI -const std::string partition_filename = test_files_dir + "/partition_metadata_1.nc"; +const std::string partition_filename = test_files_dir + "/partition_metadata_2.nc"; #endif static const int DG = 3; @@ -48,7 +48,7 @@ size_t c = 0; TEST_SUITE_BEGIN("ParaGrid"); #ifdef USE_MPI -MPI_TEST_CASE("Write and read a ModelState-based ParaGrid restart file", 1) +MPI_TEST_CASE("Write and read a ModelState-based ParaGrid restart file", 2) #else TEST_CASE("Write and read a ModelState-based ParaGrid restart file") #endif @@ -62,20 +62,29 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") grid.setIO(pio); // Set the dimension lengths - size_t nx = 25; - size_t ny = 15; + size_t nx = 10; + size_t ny = 9; size_t nz = 3; NZLevels::set(nz); double yFactor = 0.01; - double xFactor = 0.0001; + double xFactor = 0.1; #ifdef USE_MPI - ModelArray::setDimension(ModelArray::Dimension::X, nx, nx, 0); - ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); - ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); - ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1, nx + 1, 0); - ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1, ny + 1, 0); + if (test_rank == 0) { + ModelArray::setDimension(ModelArray::Dimension::X, nx, 4, 0); + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1, 4 + 1, 0); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1, ny + 1, 0); + } + if (test_rank == 1) { + ModelArray::setDimension(ModelArray::Dimension::X, nx, 6, 4); + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1, 6 + 1, 4); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1, ny + 1, 0); + } #else ModelArray::setDimension(ModelArray::Dimension::X, nx); ModelArray::setDimension(ModelArray::Dimension::Y, ny); @@ -93,11 +102,13 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") HField mask(ModelArray::Type::H); fractional.resize(); fractionalDG.resize(); + auto start = ModelArray::definedDimensions.at(ModelArray::Dimension::X).start; + auto local_nx = ModelArray::definedDimensions.at(ModelArray::Dimension::X).local_length; for (size_t j = 0; j < ny; ++j) { - for (size_t i = 0; i < nx; ++i) { - fractional(i, j) = j * yFactor + i * xFactor; + for (size_t i = 0; i < local_nx; ++i) { + fractional(i, j) = j * yFactor + (i+start) * xFactor; mask(i, j) - = (i - nx / 2) * (i - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * ny) ? 0 : 1; + = ((i + start) - nx / 2) * ((i + start) - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * ny) ? 0 : 1; for (size_t d = 0; d < DG; ++d) { fractionalDG.components({ i, j })[d] = fractional(i, j) + d; } @@ -107,7 +118,6 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") DGField hice = fractionalDG + 10; DGField cice = fractionalDG + 20; DGField hsnow = fractionalDG + 30; - DGField tom = fractionalDG + 40; DGField damage = fractionalDG * 0.; HField sss = fractional; ZField tice(ModelArray::Type::Z); @@ -124,29 +134,28 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") double scale = 1e5; // Vertex coordinates - for (size_t i = 0; i < ModelArray::definedDimensions.at(ModelArray::Dimension::XVERTEX).local_length; - ++i) { - for (size_t j = 0; - j < ModelArray::definedDimensions.at(ModelArray::Dimension::YVERTEX).local_length; ++j) { - double x = i - 0.5 - nx / 2; - double y = j - 0.5 - ny / 2; + auto local_nvx = ModelArray::definedDimensions.at(ModelArray::Dimension::XVERTEX).local_length; + for (size_t i = 0; i < local_nvx; ++i) { + for (size_t j = 0; j < ny + 1; ++j) { + double x = (i + start) - 0.5 - float(nx) / 2; + double y = j - 0.5 - float(ny) / 2; coordinates.components({ i, j })[0] = x * scale; coordinates.components({ i, j })[1] = y * scale; } } - REQUIRE(coordinates.components({ 12, 13 })[0] - coordinates.components({ 11, 13 })[0] == scale); - REQUIRE(coordinates.components({ 12, 13 })[1] - coordinates.components({ 12, 12 })[1] == scale); + REQUIRE(coordinates.components({ 3, 8 })[0] - coordinates.components({ 2, 8 })[0] == scale); + REQUIRE(coordinates.components({ 3, 8 })[1] - coordinates.components({ 3, 7 })[1] == scale); HField x; HField y; x.resize(); y.resize(); // Element coordinates - for (size_t j = 0; j < ModelArray::size(ModelArray::Dimension::Y); ++j) { - double yy = scale * (j - ny / 2); - for (size_t i = 0; i < ModelArray::size(ModelArray::Dimension::X); ++i) { - double xx = scale * (i - nx / 2); + for (size_t j = 0; j < ny; ++j) { + double yy = scale * (j - float(ny) / 2); + for (size_t i = 0; i < local_nx; ++i) { + double xx = scale * ((i + start) - float(nx) / 2); x(i, j) = xx; y(i, j) = yy; } @@ -182,6 +191,27 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") metadata.extractCoordinates(coordState); metadata.affixCoordinates(state); +#ifdef USE_MPI + metadata.setMpiMetadata(test_comm); + if (metadata.mpiMyRank == 0) { + metadata.setMpiMetadata(test_comm); + metadata.globalExtentX = nx; + metadata.globalExtentY = ny; + metadata.localCornerX = start; + metadata.localCornerY = 0; + metadata.localExtentX = local_nx; + metadata.localExtentY = ny; + } + if (metadata.mpiMyRank == 1) { + metadata.setMpiMetadata(test_comm); + metadata.globalExtentX = local_nx; + metadata.globalExtentY = ny; + metadata.localCornerX = start; + metadata.localCornerY = 0; + metadata.localExtentX = local_nx; + metadata.localExtentY = ny; + } +#endif grid.dumpModelState(state, metadata, filename, true); REQUIRE(std::filesystem::exists(std::filesystem::path(filename))); @@ -208,7 +238,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") ModelState ms = gridIn.getModelState(filename); #endif - REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == nx); + REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == local_nx); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[1] == ny); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[2] == NZLevels::get()); @@ -218,39 +248,39 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") REQUIRE(ModelArray::nDimensions(ModelArray::Type::Z) == 3); REQUIRE(ticeRef.getType() == ModelArray::Type::Z); REQUIRE(ticeRef.nDimensions() == 3); - REQUIRE(ticeRef.dimensions()[0] == nx); + REQUIRE(ticeRef.dimensions()[0] == local_nx); REQUIRE(ticeRef.dimensions()[1] == ny); REQUIRE(ticeRef.dimensions()[2] == NZLevels::get()); ModelArray& hiceRef = ms.data.at(hiceName); REQUIRE(hiceRef.nDimensions() == 2); - REQUIRE(hiceRef.dimensions()[0] == nx); + REQUIRE(hiceRef.dimensions()[0] == local_nx); REQUIRE(hiceRef.dimensions()[1] == ny); REQUIRE(ModelArray::nComponents(ModelArray::Type::DG) == DG); REQUIRE(hiceRef.nComponents() == DG); - REQUIRE(ticeRef(12, 14, 1) == tice(12, 14, 1)); + REQUIRE(ticeRef(4, 9, 1) == tice(4, 9, 1)); // Here we don't bother passing the coordinate arrays through a ModelMetadata object ModelArray& coordRef = ms.data.at(coordsName); REQUIRE(coordRef.nDimensions() == 2); REQUIRE(coordRef.nComponents() == 2); - REQUIRE(coordRef.dimensions()[0] == nx + 1); + REQUIRE(coordRef.dimensions()[0] == local_nvx); REQUIRE(coordRef.dimensions()[1] == ny + 1); - REQUIRE(coordRef.components({ 12, 13 })[0] - coordRef.components({ 11, 13 })[0] == scale); - REQUIRE(coordRef.components({ 12, 13 })[1] - coordRef.components({ 12, 12 })[1] == scale); + REQUIRE(coordRef.components({ 3, 8 })[0] - coordRef.components({ 2, 8 })[0] == scale); + REQUIRE(coordRef.components({ 3, 8 })[1] - coordRef.components({ 3, 7 })[1] == scale); REQUIRE(ms.data.count(xName) > 0); ModelArray& xRef = ms.data.at(xName); - REQUIRE(xRef(12, 13) == coordRef.components({ 12, 13 })[0] + scale / 2); + REQUIRE(xRef(3, 8) == coordRef.components({ 3, 7 })[0] + scale / 2); REQUIRE(ms.data.count(yName) > 0); ModelArray& yRef = ms.data.at(yName); - REQUIRE(yRef(12, 13) == coordRef.components({ 12, 13 })[1] + scale / 2); + REQUIRE(yRef(3, 8) == coordRef.components({ 2, 8 })[1] + scale / 2); REQUIRE(ms.data.count(gridAzimuthName) > 0); REQUIRE(ms.data.at(gridAzimuthName)(0, 0) == gridAzimuth0); - // std::filesystem::remove(filename); + std::filesystem::remove(filename); } // #ifdef USE_MPI From e09ef811ef546c3ec256a469b45aa6ce00ae4210 Mon Sep 17 00:00:00 2001 From: melt Date: Tue, 11 Jun 2024 18:35:11 +0100 Subject: [PATCH 06/25] finish implementing MPI for ParaGrid * add mpi to ParaGridIO.cpp * finish mpi test cases for ParaGrid_test.cpp * reinstate ConfigOutput_test.cpp for MPI builds * update core/test/CMakeLists.txt to build tests and required input files --- core/src/ParaGridIO.cpp | 70 +-- core/src/ParallelNetcdfFile.cpp | 7 + core/src/include/ModelArray.hpp | 2 +- core/src/include/ParallelNetcdfFile.hpp | 3 + core/test/CMakeLists.txt | 31 +- core/test/ConfigOutput_test.cpp | 44 +- core/test/ParaGrid_test.cpp | 607 +++++++++++++----------- core/test/partition_metadata_2.nc.dump | 60 +++ 8 files changed, 500 insertions(+), 324 deletions(-) create mode 100644 core/test/partition_metadata_2.nc.dump diff --git a/core/src/ParaGridIO.cpp b/core/src/ParaGridIO.cpp index 5bdf0c126..65370e064 100644 --- a/core/src/ParaGridIO.cpp +++ b/core/src/ParaGridIO.cpp @@ -355,7 +355,7 @@ void ParaGridIO::writeDiagnosticTime( if (isNew) { // Open a new file and emplace it in the map of open files. #ifdef USE_MPI - openFiles.try_emplace(filePath, filePath, netCDF::NcFile::read, meta.mpiComm); + openFiles.try_emplace(filePath, filePath, netCDF::NcFile::replace, meta.mpiComm); #else openFiles.try_emplace(filePath, filePath, netCDF::NcFile::replace); #endif @@ -388,7 +388,7 @@ void ParaGridIO::writeDiagnosticTime( for (auto entry : ModelArray::definedDimensions) { ModelArray::Dimension dim = entry.first; size_t dimSz = (dimCompMap.count(dim)) ? ModelArray::nComponents(dimCompMap.at(dim)) - : dimSz = entry.second.local_length; + : dimSz = entry.second.global_length; ncFromMAMap[dim] = (isNew) ? dataGroup.addDim(entry.second.name, dimSz) : dataGroup.getDim(entry.second.name); } @@ -396,43 +396,49 @@ void ParaGridIO::writeDiagnosticTime( // Also create the sets of dimensions to be connected to the data fields std::map> dimMap; // Create the index and size arrays - // The index arrays always start from zero, except in the first/time axis - std::map> indexArrays; - std::map> extentArrays; + std::map> startMap; + std::map> countMap; for (auto entry : ModelArray::typeDimensions) { ModelArray::Type type = entry.first; std::vector ncDims; - std::vector indexArray; - std::vector extentArray; + std::vector start; + std::vector count; + + // Everything that has components needs that dimension, too + if (ModelArray::hasDoF(type)){ + if (type == ModelArray::Type::VERTEX && !isNew) + continue; + auto ncomps = ModelArray::nComponents(type); + auto dim = ModelArray::componentMap.at(type); + ncDims.push_back(ncFromMAMap.at(dim)); + start.push_back(0); + count.push_back(ncomps); + } + for (auto dt : entry.second){ + auto dim = ModelArray::definedDimensions.at(dt); + ncDims.push_back(ncFromMAMap.at(dt)); + start.push_back(dim.start); + count.push_back(dim.local_length); + } + + std::reverse(ncDims.begin(), ncDims.end()); + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); // Deal with VERTEX in each case // Add the time dimension for all types that are not VERTEX if (type != ModelArray::Type::VERTEX) { ncDims.push_back(timeDim); - indexArray.push_back(nt); - extentArray.push_back(1UL); + start.push_back(nt); + count.push_back(1UL); } else if (!isNew) { // For VERTEX in an existing file, there is nothing more to be done continue; } - for (auto iter = entry.second.rbegin(); iter != entry.second.rend(); ++iter) { - ModelArray::Dimension& maDim = *iter; - ncDims.push_back(ncFromMAMap.at(maDim)); - indexArray.push_back(0); - extentArray.push_back(ModelArray::definedDimensions.at(maDim).local_length); - } + dimMap[type] = ncDims; - indexArrays[type] = indexArray; - extentArrays[type] = extentArray; - } - // Everything that has components needs that dimension, too - for (auto entry : dimCompMap) { - // Skip VERTEX fields on existing files - if (entry.second == ModelArray::Type::VERTEX && !isNew) - continue; - dimMap.at(entry.second).push_back(ncFromMAMap.at(entry.first)); - indexArrays.at(entry.second).push_back(0); - extentArrays.at(entry.second).push_back(ModelArray::nComponents(entry.second)); + startMap[type] = start; + countMap[type] = count; } // Create a special timeless set of dimensions for the landmask @@ -456,6 +462,9 @@ void ParaGridIO::writeDiagnosticTime( netCDF::NcVar timeVar((isNew) ? dataGroup.addVar(timeName, netCDF::ncDouble, timeDimVec) : dataGroup.getVar(timeName)); double secondsSinceEpoch = (meta.time() - TimePoint()).seconds(); +#ifdef USE_MPI + netCDF::setVariableCollective(timeVar, dataGroup); +#endif timeVar.putVar({ nt }, { 1 }, &secondsSinceEpoch); // Write the data @@ -468,6 +477,9 @@ void ParaGridIO::writeDiagnosticTime( // Land mask in a new file (since it was skipped above in existing files) netCDF::NcVar var(dataGroup.addVar(maskName, netCDF::ncDouble, maskDims)); // No missing data +#ifdef USE_MPI + netCDF::setVariableCollective(var, dataGroup); +#endif var.putVar(maskIndexes, maskExtents, entry.second.getData()); } else { @@ -477,8 +489,10 @@ void ParaGridIO::writeDiagnosticTime( : dataGroup.getVar(entry.first)); if (isNew) var.putAtt(mdiName, netCDF::ncDouble, MissingData::value); - - var.putVar(indexArrays.at(type), extentArrays.at(type), entry.second.getData()); +#ifdef USE_MPI + netCDF::setVariableCollective(var, dataGroup); +#endif + var.putVar(startMap.at(type), countMap.at(type), entry.second.getData()); } } } diff --git a/core/src/ParallelNetcdfFile.cpp b/core/src/ParallelNetcdfFile.cpp index c883f22d0..4e9d244d1 100644 --- a/core/src/ParallelNetcdfFile.cpp +++ b/core/src/ParallelNetcdfFile.cpp @@ -44,3 +44,10 @@ void NcFilePar::open_par( nullObject = false; } + +void netCDF::setVariableCollective(NcVar var, NcGroup group) +{ + // This function will change the parallel access of a variable from independent to collective. + // This is required to write to a variable in parallel when using MPI. + ncCheck(nc_var_par_access(group.getId(), var.getId(), NC_COLLECTIVE), __FILE__, __LINE__); +} diff --git a/core/src/include/ModelArray.hpp b/core/src/include/ModelArray.hpp index f9503dc37..0f19a6140 100644 --- a/core/src/include/ModelArray.hpp +++ b/core/src/include/ModelArray.hpp @@ -276,7 +276,7 @@ class ModelArray { * @param length The new length of the dimension. */ #ifdef USE_MPI - static void setDimension(Dimension dim, size_t global_length, size_t local_length=0, size_t size=0); + static void setDimension(Dimension dim, size_t global_length, size_t local_length, size_t size); #else static void setDimension(Dimension dim, size_t global_length); #endif diff --git a/core/src/include/ParallelNetcdfFile.hpp b/core/src/include/ParallelNetcdfFile.hpp index 42d067cad..44f4b17ee 100644 --- a/core/src/include/ParallelNetcdfFile.hpp +++ b/core/src/include/ParallelNetcdfFile.hpp @@ -1,5 +1,6 @@ #include #include +#include namespace netCDF { @@ -13,4 +14,6 @@ class NcFilePar : public netCDF::NcFile { void open_par(const std::string& path, const FileMode fMode, MPI_Comm comm, MPI_Info info); }; +void setVariableCollective(NcVar var, NcGroup group); + } diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 1d23a6c49..641e5c503 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -47,23 +47,32 @@ target_link_libraries(testModelArray PRIVATE doctest::doctest Eigen3::Eigen) set(MODEL_INCLUDE_DIR "../../core/src/discontinuousgalerkin") if(ENABLE_MPI) - add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp") - target_compile_definitions(testRectGrid_MPI3 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") - target_include_directories(testRectGrid_MPI3 PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") - target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest) - # Generate partition file needed for MPI test from its dump add_custom_command(OUTPUT partition_metadata_3.nc COMMAND ncgen -b -o partition_metadata_3.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3.cdl DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3.cdl ) - add_custom_target(generate_partition_file ALL DEPENDS partition_metadata_3.nc) + add_custom_command(OUTPUT partition_metadata_2.nc + COMMAND ncgen -b -o partition_metadata_2.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.nc.dump + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.nc.dump + ) + add_custom_target(generate_partition_files ALL DEPENDS partition_metadata_3.nc partition_metadata_2.nc) + + add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp") + target_compile_definitions(testRectGrid_MPI3 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") + target_include_directories(testRectGrid_MPI3 PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") + target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest) add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp") - target_compile_definitions(testParaGrid_MPI2 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") + target_compile_definitions(testParaGrid_MPI2 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" TEST_FILE_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}\") target_include_directories(testParaGrid_MPI2 PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") target_link_libraries(testParaGrid_MPI2 PRIVATE nextsimlib doctest::doctest) + add_executable(testConfigOutput_MPI2 "ConfigOutput_test.cpp" "MainMPI.cpp") + target_compile_definitions(testConfigOutput_MPI2 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") + target_include_directories(testConfigOutput_MPI2 PRIVATE ${MODEL_INCLUDE_DIR}) + target_link_libraries(testConfigOutput_MPI2 PRIVATE nextsimlib doctest::doctest) + if(ENABLE_XIOS) FILE(CREATE_LINK "${CMAKE_SOURCE_DIR}/core/test/iodef.xml" "${CMAKE_CURRENT_BINARY_DIR}/iodef.xml" SYMBOLIC) add_custom_command(OUTPUT xios_test_input.nc @@ -124,11 +133,17 @@ else() target_link_libraries(testRectGrid PRIVATE nextsimlib doctest::doctest) add_executable(testParaGrid "ParaGrid_test.cpp") - target_compile_definitions(testParaGrid PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") + target_compile_definitions(testParaGrid PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\" TEST_FILE_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}\") target_include_directories(testParaGrid PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") target_link_libraries(testParaGrid PRIVATE nextsimlib doctest::doctest) + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/old_names.nc" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") + + add_executable(testConfigOutput "ConfigOutput_test.cpp") + target_compile_definitions(testConfigOutput PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") + target_include_directories(testConfigOutput PRIVATE ${MODEL_INCLUDE_DIR}) + target_link_libraries(testConfigOutput PRIVATE nextsimlib doctest::doctest) endif() add_executable(testModelComponent "ModelComponent_test.cpp") diff --git a/core/test/ConfigOutput_test.cpp b/core/test/ConfigOutput_test.cpp index 4b41db3a1..914dc0e8e 100644 --- a/core/test/ConfigOutput_test.cpp +++ b/core/test/ConfigOutput_test.cpp @@ -5,8 +5,12 @@ * @author Tim Spain */ +#ifdef USE_MPI +#include +#else #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include +#endif #include "DiagnosticOutputModule/include/ConfigOutput.hpp" @@ -29,19 +33,39 @@ #include #include +const std::string test_files_dir = TEST_FILES_DIR; +#ifdef USE_MPI +const std::string partition_filename = test_files_dir + "/partition_metadata_2.nc"; +#endif + namespace Nextsim { TEST_SUITE_BEGIN("ConfigOutput"); +#ifdef USE_MPI +MPI_TEST_CASE("Test periodic output", 2) +#else TEST_CASE("Test periodic output") +#endif { size_t nx = 2; size_t ny = 5; size_t nz = 3; NZLevels::set(nz); +#ifdef USE_MPI + if (test_rank == 0) { + ModelArray::setDimension(ModelArray::Dimension::X, nx, 1, 0); + } + if (test_rank == 1) { + ModelArray::setDimension(ModelArray::Dimension::X, nx, 1, 1); + } + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); +#else ModelArray::setDimension(ModelArray::Dimension::X, nx); ModelArray::setDimension(ModelArray::Dimension::Y, ny); ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); +#endif std::stringstream config; config << "[Modules]" << std::endl; @@ -79,21 +103,29 @@ TEST_CASE("Test periodic output") ModelMetadata meta; meta.setTime(TimePoint("2020-01-01T00:00:00Z")); +#ifdef USE_MPI + meta.setMpiMetadata(test_comm); +#endif + IDiagnosticOutput& ido = Module::getImplementation(); tryConfigure(ido); + auto dim_x = ModelArray::Dimension::X; + auto start_x = ModelArray::definedDimensions.at(dim_x).start; + auto local_nx = ModelArray::definedDimensions.at(dim_x).local_length; + for (size_t k = 0; k < nz; ++k) { for (size_t j = 0; j < ny; ++j) { - for (size_t i = 0; i < nx; ++i) { - tice(i, j, k) = 0.1 * k + 0.4 + 0.01 * (j * nx + i); + for (size_t i = 0; i < local_nx; ++i) { + tice(i, j, k) = 0.1 * k + 0.4 + 0.01 * (j * nx + (i + start_x)); } } } for (size_t j = 0; j < ny; ++j) { - for (size_t i = 0; i < nx; ++i) { - hice(i, j) = 0 + 0.01 * (j * nx + i); - cice(i, j) = 0.1 + 0.01 * (j * nx + i); - hsnow(i, j) = 0.2 + 0.01 * (j * nx + i); + for (size_t i = 0; i < local_nx; ++i) { + hice(i, j) = 0 + 0.01 * (j * nx + (i + start_x)); + cice(i, j) = 0.1 + 0.01 * (j * nx + (i + start_x)); + hsnow(i, j) = 0.2 + 0.01 * (j * nx + (i + start_x)); } } std::vector diagFiles; diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index 38cf8c791..738bffa5b 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -5,6 +5,8 @@ * @author Tim Spain */ +#include "ModelArray.hpp" +#include #ifdef USE_MPI #include #else @@ -42,10 +44,49 @@ static const int DG = 3; static const int DGSTRESS = 6; static const int CG = 2; +const size_t nx = 10; +const size_t ny = 9; +const size_t nz = 3; +const double yFactor = 0.01; +const double xFactor = 0.1; +const double scale = 1e5; + namespace Nextsim { size_t c = 0; +void initialize_test_data(HField& hfield, DGField& dgfield, HField& mask){ + hfield.resize(); + dgfield.resize(); + mask.resize(); + auto dim_x = ModelArray::Dimension::X; + auto start_x = ModelArray::definedDimensions.at(dim_x).start; + auto local_nx = ModelArray::definedDimensions.at(dim_x).local_length; + for (size_t j = 0; j < ny; ++j) { + for (size_t i = 0; i < local_nx; ++i) { + hfield(i, j) = j * yFactor + (i+start_x) * xFactor; + mask(i, j) = ((i + start_x) - nx / 2) * ((i + start_x) - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * ny) ? 0 : 1; + for (size_t d = 0; d < DG; ++d) { + dgfield.components({ i, j })[d] = hfield(i, j) + d; + } + } + } +}; + +void initialize_test_coordinates(VertexField& coordinates){ + auto dim_x_vertex = ModelArray::Dimension::XVERTEX; + auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).local_length; + auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; + for (size_t i = 0; i < local_nx_vertex; ++i) { + for (size_t j = 0; j < ny + 1; ++j) { + double x = (i + start_x_vertex) - 0.5 - float(nx) / 2; + double y = j - 0.5 - float(ny) / 2; + coordinates.components({ i, j })[0] = x * scale; + coordinates.components({ i, j })[1] = y * scale; + } + } +}; + TEST_SUITE_BEGIN("ParaGrid"); #ifdef USE_MPI MPI_TEST_CASE("Write and read a ModelState-based ParaGrid restart file", 2) @@ -62,29 +103,21 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") grid.setIO(pio); // Set the dimension lengths - size_t nx = 10; - size_t ny = 9; - size_t nz = 3; NZLevels::set(nz); - double yFactor = 0.01; - double xFactor = 0.1; #ifdef USE_MPI if (test_rank == 0) { ModelArray::setDimension(ModelArray::Dimension::X, nx, 4, 0); - ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); - ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1, 4 + 1, 0); - ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1, ny + 1, 0); } if (test_rank == 1) { ModelArray::setDimension(ModelArray::Dimension::X, nx, 6, 4); - ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); - ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1, 6 + 1, 4); - ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1, ny + 1, 0); } + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1, ny + 1, 0); #else ModelArray::setDimension(ModelArray::Dimension::X, nx); ModelArray::setDimension(ModelArray::Dimension::Y, ny); @@ -100,20 +133,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") HField fractional(ModelArray::Type::H); DGField fractionalDG(ModelArray::Type::DG); HField mask(ModelArray::Type::H); - fractional.resize(); - fractionalDG.resize(); - auto start = ModelArray::definedDimensions.at(ModelArray::Dimension::X).start; - auto local_nx = ModelArray::definedDimensions.at(ModelArray::Dimension::X).local_length; - for (size_t j = 0; j < ny; ++j) { - for (size_t i = 0; i < local_nx; ++i) { - fractional(i, j) = j * yFactor + (i+start) * xFactor; - mask(i, j) - = ((i + start) - nx / 2) * ((i + start) - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * ny) ? 0 : 1; - for (size_t d = 0; d < DG; ++d) { - fractionalDG.components({ i, j })[d] = fractional(i, j) + d; - } - } - } + initialize_test_data(fractional, fractionalDG, mask); DGField hice = fractionalDG + 10; DGField cice = fractionalDG + 20; @@ -129,24 +149,20 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") } VertexField coordinates(ModelArray::Type::VERTEX); - coordinates.resize(); - // Planar coordinates - double scale = 1e5; - - // Vertex coordinates - auto local_nvx = ModelArray::definedDimensions.at(ModelArray::Dimension::XVERTEX).local_length; - for (size_t i = 0; i < local_nvx; ++i) { - for (size_t j = 0; j < ny + 1; ++j) { - double x = (i + start) - 0.5 - float(nx) / 2; - double y = j - 0.5 - float(ny) / 2; - coordinates.components({ i, j })[0] = x * scale; - coordinates.components({ i, j })[1] = y * scale; - } - } + initialize_test_coordinates(coordinates); REQUIRE(coordinates.components({ 3, 8 })[0] - coordinates.components({ 2, 8 })[0] == scale); REQUIRE(coordinates.components({ 3, 8 })[1] - coordinates.components({ 3, 7 })[1] == scale); + // MPI domain is only split in x-direction for this test + // the following will be set correctly with MPI ON and OFF + auto dim_x = ModelArray::Dimension::X; + auto start_x = ModelArray::definedDimensions.at(dim_x).start; + auto local_nx = ModelArray::definedDimensions.at(dim_x).local_length; + auto dim_x_vertex = ModelArray::Dimension::XVERTEX; + auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).local_length; + auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; + HField x; HField y; x.resize(); @@ -155,7 +171,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") for (size_t j = 0; j < ny; ++j) { double yy = scale * (j - float(ny) / 2); for (size_t i = 0; i < local_nx; ++i) { - double xx = scale * ((i + start) - float(nx) / 2); + double xx = scale * ((i + start_x) - float(nx) / 2); x(i, j) = xx; y(i, j) = yy; } @@ -197,7 +213,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") metadata.setMpiMetadata(test_comm); metadata.globalExtentX = nx; metadata.globalExtentY = ny; - metadata.localCornerX = start; + metadata.localCornerX = start_x; metadata.localCornerY = 0; metadata.localExtentX = local_nx; metadata.localExtentY = ny; @@ -206,7 +222,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") metadata.setMpiMetadata(test_comm); metadata.globalExtentX = local_nx; metadata.globalExtentY = ny; - metadata.localCornerX = start; + metadata.localCornerX = start_x; metadata.localCornerY = 0; metadata.localExtentX = local_nx; metadata.localExtentY = ny; @@ -217,11 +233,19 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") REQUIRE(std::filesystem::exists(std::filesystem::path(filename))); // Reset the array dimensions to make sure that the read function gets them correct +#ifdef USE_MPI + ModelArray::setDimension(ModelArray::Dimension::X, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::Y, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, 1, 1, 0); +#else ModelArray::setDimension(ModelArray::Dimension::X, 1); ModelArray::setDimension(ModelArray::Dimension::Y, 1); ModelArray::setDimension(ModelArray::Dimension::Z, 1); ModelArray::setDimension(ModelArray::Dimension::XVERTEX, 1); ModelArray::setDimension(ModelArray::Dimension::YVERTEX, 1); +#endif // In the full model numbers of DG components are set at compile time, so they are not reset REQUIRE(ModelArray::nComponents(ModelArray::Type::DG) == DG); REQUIRE(ModelArray::nComponents(ModelArray::Type::VERTEX) == ModelArray::nCoords); @@ -265,7 +289,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") ModelArray& coordRef = ms.data.at(coordsName); REQUIRE(coordRef.nDimensions() == 2); REQUIRE(coordRef.nComponents() == 2); - REQUIRE(coordRef.dimensions()[0] == local_nvx); + REQUIRE(coordRef.dimensions()[0] == local_nx_vertex); REQUIRE(coordRef.dimensions()[1] == ny + 1); REQUIRE(coordRef.components({ 3, 8 })[0] - coordRef.components({ 2, 8 })[0] == scale); REQUIRE(coordRef.components({ 3, 8 })[1] - coordRef.components({ 3, 7 })[1] == scale); @@ -283,244 +307,265 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") std::filesystem::remove(filename); } -// #ifdef USE_MPI -// MPI_TEST_CASE("Write a diagnostic ParaGrid file", 1) -// #else -// TEST_CASE("Write a diagnostic ParaGrid file") -// #endif -// { -// Module::setImplementation("Nextsim::ParametricGrid"); - -// REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); - -// std::filesystem::remove(diagFile); - -// ParametricGrid grid; -// ParaGridIO* pio = new ParaGridIO(grid); -// grid.setIO(pio); - -// // Set the dimension lengths -// size_t nx = 30; -// size_t ny = 20; -// size_t nz = 3; -// NZLevels::set(nz); -// size_t nxcg = CG * nx + 1; -// size_t nycg = CG * ny + 1; - -// double yFactor = 0.01; -// double xFactor = 0.0001; - -// ModelArray::setDimension(ModelArray::Dimension::X, nx); -// ModelArray::setDimension(ModelArray::Dimension::Y, ny); -// ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); -// ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1); -// ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1); -// ModelArray::setDimension(ModelArray::Dimension::XCG, nxcg); -// ModelArray::setDimension(ModelArray::Dimension::YCG, nycg); - -// ModelArray::setNComponents(ModelArray::Type::DG, DG); -// ModelArray::setNComponents(ModelArray::Type::DGSTRESS, DGSTRESS); -// ModelArray::setNComponents(ModelArray::Type::VERTEX, ModelArray::nCoords); - -// HField fractional(ModelArray::Type::H); -// DGField fractionalDG(ModelArray::Type::DG); -// HField mask(ModelArray::Type::H); -// fractional.resize(); -// fractionalDG.resize(); -// for (size_t j = 0; j < ny; ++j) { -// for (size_t i = 0; i < nx; ++i) { -// fractional(i, j) = j * yFactor + i * xFactor; -// mask(i, j) = fractional(i, j); -// // = (i - nx / 2) * (i - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * -// // ny) ? 0 : 1; -// for (size_t d = 0; d < DG; ++d) { -// fractionalDG.components({ i, j })[d] = fractional(i, j) + d; -// } -// } -// } -// double prec = 1e-9; -// REQUIRE(fractional(12, 12) - fractional(11, 12) == doctest::Approx(xFactor).epsilon(prec)); -// REQUIRE(fractional(12, 12) - fractional(12, 11) == doctest::Approx(yFactor).epsilon(prec)); - -// REQUIRE(fractionalDG(12, 12) - fractionalDG(11, 12) == doctest::Approx(xFactor).epsilon(prec)); -// REQUIRE(fractionalDG(12, 12) - fractionalDG(12, 11) == doctest::Approx(yFactor).epsilon(prec)); - -// DGField hice = fractionalDG + 10; -// DGField cice = fractionalDG + 20; - -// VertexField coordinates(ModelArray::Type::VERTEX); -// coordinates.resize(); -// // Planar coordinates -// double scale = 1e5; - -// for (size_t i = 0; i < ModelArray::definedDimensions.at(ModelArray::Dimension::XVERTEX).length; -// ++i) { -// for (size_t j = 0; -// j < ModelArray::definedDimensions.at(ModelArray::Dimension::YVERTEX).length; ++j) { -// double x = i - 0.5 - nx / 2; -// double y = j - 0.5 - ny / 2; -// coordinates.components({ i, j })[0] = x * scale; -// coordinates.components({ i, j })[1] = y * scale; -// } -// } - -// REQUIRE(coordinates.components({ 12, 13 })[0] - coordinates.components({ 11, 13 })[0] == scale); -// REQUIRE(coordinates.components({ 12, 13 })[1] - coordinates.components({ 12, 12 })[1] == scale); - -// HField x; -// HField y; -// x.resize(); -// y.resize(); -// // Element coordinates -// for (size_t j = 0; j < ModelArray::size(ModelArray::Dimension::Y); ++j) { -// double yy = scale * (j - ny / 2); -// for (size_t i = 0; i < ModelArray::size(ModelArray::Dimension::X); ++i) { -// double xx = scale * (i - nx / 2); -// x(i, j) = xx; -// y(i, j) = yy; -// } -// } - -// HField gridAzimuth; -// double gridAzimuth0 = 45.; -// gridAzimuth = gridAzimuth0; - -// ModelState state = { { -// { maskName, mask }, -// { hiceName, hice }, -// { ciceName, cice }, -// }, -// {} }; - -// // A model state to set the coordinates in the metadata object -// ModelState coordState = { { -// { xName, x }, -// { yName, y }, -// { coordsName, coordinates }, -// { gridAzimuthName, gridAzimuth }, -// }, -// {} }; - -// ModelMetadata metadata; -// metadata.setTime(TimePoint("2000-01-01T00:00:00Z")); -// // The coordinates are passed through the metadata object as affix -// // coordinates is the correct way to add coordinates to a ModelState -// metadata.extractCoordinates(coordState); -// metadata.affixCoordinates(state); - -// grid.dumpModelState(state, metadata, diagFile, false); - -// for (int t = 1; t < 5; ++t) { -// hice += 100; -// cice += 100; -// state = { { -// { hiceName, hice }, -// { ciceName, cice }, -// }, -// {} }; -// metadata.incrementTime(Duration(3600)); - -// grid.dumpModelState(state, metadata, diagFile, false); -// } -// pio->close(diagFile); - -// // What do we have in the file? -// netCDF::NcFile ncFile(diagFile, netCDF::NcFile::read); - -// REQUIRE(ncFile.getGroups().size() == 3); -// netCDF::NcGroup structGrp(ncFile.getGroup(IStructure::structureNodeName())); -// netCDF::NcGroup metaGrp(ncFile.getGroup(IStructure::metadataNodeName())); -// netCDF::NcGroup dataGrp(ncFile.getGroup(IStructure::dataNodeName())); - -// std::string structureType; -// structGrp.getAtt(grid.typeNodeName()).getValues(structureType); -// REQUIRE(structureType == grid.structureType()); - -// // TODO test metadata - -// // test data -// REQUIRE(dataGrp.getVarCount() == 8); -// netCDF::NcVar hiceVar = dataGrp.getVar(hiceName); -// netCDF::NcVar ciceVar = dataGrp.getVar(ciceName); -// netCDF::NcVar maskVar = dataGrp.getVar(maskName); -// netCDF::NcVar timeVar = dataGrp.getVar(timeName); - -// // hice -// REQUIRE(hiceVar.getDimCount() == 4); - -// // coordinates -// REQUIRE(dataGrp.getVars().count(xName) > 0); -// REQUIRE(dataGrp.getVars().count(yName) > 0); -// REQUIRE(dataGrp.getVars().count(coordsName) > 0); -// REQUIRE(dataGrp.getVars().count(gridAzimuthName) > 0); - -// ncFile.close(); - -// std::filesystem::remove(diagFile); -// } - -// #define TO_STR(s) TO_STRI(s) -// #define TO_STRI(s) #s -// #ifndef TEST_FILE_SOURCE -// #define TEST_FILE_SOURCE . -// #endif - -// #ifdef USE_MPI -// MPI_TEST_CASE("Test array ordering", 1) -// #else -// TEST_CASE("Test array ordering") -// #endif -// { -// std::string inputFilename = "ParaGridIO_input_test.nc"; - -// Module::setImplementation("Nextsim::ParametricGrid"); - -// REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); - -// size_t nx = 9; -// size_t ny = 11; -// NZLevels::set(1); - -// double xFactor = 10; - -// ModelArray::setDimension(ModelArray::Dimension::X, nx); -// ModelArray::setDimension(ModelArray::Dimension::Y, ny); -// ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); - -// HField index2d(ModelArray::Type::H); -// index2d.resize(); -// std::string fieldName = "index2d"; -// std::set fields = { fieldName }; -// TimePoint time; - -// ModelState state = ParaGridIO::readForcingTimeStatic( -// fields, time, TO_STR(TEST_FILE_SOURCE) + std::string("/") + inputFilename); -// REQUIRE(state.data.count(fieldName) > 0); -// index2d = state.data.at(fieldName); -// REQUIRE(index2d(3, 5) == 35); -// // And that's all that's needed -// } - -// #undef TO_STR -// #undef TO_STRI - -// #ifdef USE_MPI -// MPI_TEST_CASE("Check an exception is thrown for an invalid file name", 1) -// #else -// TEST_CASE("Check an exception is thrown for an invalid file name") -// #endif -// { -// ParametricGrid gridIn; -// ParaGridIO* readIO = new ParaGridIO(gridIn); -// gridIn.setIO(readIO); - -// ModelState state; -// -// // MD5 hash of the current output of $ date -// std::string longRandomFilename("a44f5cc1f7934a8ae8dd03a95308745d.nc"); -// REQUIRE_THROWS(state = gridIn.getModelState(longRandomFilename)); - -// } + +#ifdef USE_MPI +MPI_TEST_CASE("Write a diagnostic ParaGrid file", 2) +#else +TEST_CASE("Write a diagnostic ParaGrid file") +#endif +{ + Module::setImplementation("Nextsim::ParametricGrid"); + + REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); + + std::filesystem::remove(diagFile); + + + ParametricGrid grid; + ParaGridIO* pio = new ParaGridIO(grid); + grid.setIO(pio); + + NZLevels::set(nz); + +#ifdef USE_MPI + if (test_rank == 0) { + ModelArray::setDimension(ModelArray::Dimension::X, nx, 4, 0); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1, 4 + 1, 0); + } + if (test_rank == 1) { + ModelArray::setDimension(ModelArray::Dimension::X, nx, 6, 4); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1, 6 + 1, 4); + } + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1, ny + 1, 0); +#else + ModelArray::setDimension(ModelArray::Dimension::X, nx); + ModelArray::setDimension(ModelArray::Dimension::Y, ny); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nx + 1); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, ny + 1); +#endif + + ModelArray::setNComponents(ModelArray::Type::DG, DG); + ModelArray::setNComponents(ModelArray::Type::DGSTRESS, DGSTRESS); + ModelArray::setNComponents(ModelArray::Type::VERTEX, ModelArray::nCoords); + + // MPI domain is only split in x-direction for this test + // the following will be set correctly with MPI ON and OFF + auto dim_x = ModelArray::Dimension::X; + auto start_x = ModelArray::definedDimensions.at(dim_x).start; + auto local_nx = ModelArray::definedDimensions.at(dim_x).local_length; + auto dim_x_vertex = ModelArray::Dimension::XVERTEX; + auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).local_length; + auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; + + HField fractional(ModelArray::Type::H); + DGField fractionalDG(ModelArray::Type::DG); + HField mask(ModelArray::Type::H); + initialize_test_data(fractional, fractionalDG, mask); + + REQUIRE(fractional.nDimensions() == 2); + + DGField hice = fractionalDG + 10; + DGField cice = fractionalDG + 20; + + VertexField coordinates(ModelArray::Type::VERTEX); + initialize_test_coordinates(coordinates); + + HField x; + HField y; + x.resize(); + y.resize(); + // Element coordinates + for (size_t j = 0; j < ny; ++j) { + double yy = scale * (j - float(ny) / 2); + for (size_t i = 0; i < local_nx; ++i) { + double xx = scale * ((i + start_x) - float(nx) / 2); + x(i, j) = xx; + y(i, j) = yy; + } + } + + HField gridAzimuth; + double gridAzimuth0 = 45.; + gridAzimuth = gridAzimuth0; + + ModelState state = { { + { maskName, mask }, + { hiceName, hice }, + { ciceName, cice }, + }, + {} }; + + // A model state to set the coordinates in the metadata object + ModelState coordState = { { + { xName, x }, + { yName, y }, + { coordsName, coordinates }, + { gridAzimuthName, gridAzimuth }, + }, + {} }; + + + ModelMetadata metadata; + metadata.setTime(TimePoint("2000-01-01T00:00:00Z")); + // The coordinates are passed through the metadata object as affix + // coordinates is the correct way to add coordinates to a ModelState + metadata.extractCoordinates(coordState); + metadata.affixCoordinates(state); + +#ifdef USE_MPI + metadata.setMpiMetadata(test_comm); + if (metadata.mpiMyRank == 0) { + metadata.setMpiMetadata(test_comm); + metadata.globalExtentX = nx; + metadata.globalExtentY = ny; + metadata.localCornerX = start_x; + metadata.localCornerY = 0; + metadata.localExtentX = local_nx; + metadata.localExtentY = ny; + } + if (metadata.mpiMyRank == 1) { + metadata.setMpiMetadata(test_comm); + metadata.globalExtentX = local_nx; + metadata.globalExtentY = ny; + metadata.localCornerX = start_x; + metadata.localCornerY = 0; + metadata.localExtentX = local_nx; + metadata.localExtentY = ny; + } +#endif + + for (int t = 1; t < 5; ++t) { + hice += 100; + cice += 100; + state = { { + { hiceName, hice }, + { ciceName, cice }, + { xName, x }, + { yName, y }, + { coordsName, coordinates }, + { gridAzimuthName, gridAzimuth }, + }, + {} }; + metadata.incrementTime(Duration(3600)); + + grid.dumpModelState(state, metadata, diagFile, false); + } + pio->close(diagFile); + + REQUIRE(std::filesystem::exists(std::filesystem::path(diagFile))); + + + // What do we have in the file? + netCDF::NcFile ncFile(diagFile, netCDF::NcFile::read); + + REQUIRE(ncFile.getGroups().size() == 3); + netCDF::NcGroup structGrp(ncFile.getGroup(IStructure::structureNodeName())); + netCDF::NcGroup metaGrp(ncFile.getGroup(IStructure::metadataNodeName())); + netCDF::NcGroup dataGrp(ncFile.getGroup(IStructure::dataNodeName())); + + std::string structureType; + structGrp.getAtt(grid.typeNodeName()).getValues(structureType); + REQUIRE(structureType == grid.structureType()); + + // TODO test metadata + + // test data + REQUIRE(dataGrp.getVarCount() == 7); + netCDF::NcVar hiceVar = dataGrp.getVar(hiceName); + netCDF::NcVar ciceVar = dataGrp.getVar(ciceName); + netCDF::NcVar maskVar = dataGrp.getVar(maskName); + netCDF::NcVar timeVar = dataGrp.getVar(timeName); + + // hice + REQUIRE(hiceVar.getDimCount() == 4); + + // coordinates + REQUIRE(dataGrp.getVars().count(xName) > 0); + REQUIRE(dataGrp.getVars().count(yName) > 0); + REQUIRE(dataGrp.getVars().count(coordsName) > 0); + REQUIRE(dataGrp.getVars().count(gridAzimuthName) > 0); + + ncFile.close(); + + std::filesystem::remove(diagFile); +} + +#ifndef TEST_FILE_SOURCE +#define TEST_FILE_SOURCE "." +#endif + +#ifdef USE_MPI +MPI_TEST_CASE("Test array ordering", 2) +#else +TEST_CASE("Test array ordering") +#endif +{ + std::string inputFilename = std::string(TEST_FILE_SOURCE) + "/ParaGridIO_input_test.nc"; + + Module::setImplementation("Nextsim::ParametricGrid"); + + REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); + + size_t nx = 9; + size_t ny = 11; + NZLevels::set(1); + + double xFactor = 10; + +#ifdef USE_MPI + if (test_rank == 0) { + ModelArray::setDimension(ModelArray::Dimension::X, nx, 4, 0); + } + if (test_rank == 1) { + ModelArray::setDimension(ModelArray::Dimension::X, nx, 5, 4); + } + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get(), NZLevels::get(), 0); +#else + ModelArray::setDimension(ModelArray::Dimension::X, nx); + ModelArray::setDimension(ModelArray::Dimension::Y, ny); + ModelArray::setDimension(ModelArray::Dimension::Z, NZLevels::get()); +#endif + + HField index2d(ModelArray::Type::H); + index2d.resize(); + std::string fieldName = "index2d"; + std::set fields = { fieldName }; + TimePoint time; + + printf("path = %s\n", inputFilename.c_str()); + ModelState state = ParaGridIO::readForcingTimeStatic(fields, time, inputFilename); + REQUIRE(state.data.count(fieldName) > 0); + index2d = state.data.at(fieldName); + REQUIRE(index2d(3, 5) == 35); +} + +#ifdef USE_MPI +MPI_TEST_CASE("Check an exception is thrown for an invalid file name", 2) +#else +TEST_CASE("Check an exception is thrown for an invalid file name") +#endif +{ + ParametricGrid gridIn; + ParaGridIO* readIO = new ParaGridIO(gridIn); + gridIn.setIO(readIO); + + ModelState state; + + // MD5 hash of the current output of $ date + std::string longRandomFilename("a44f5cc1f7934a8ae8dd03a95308745d.nc"); +#ifdef USE_MPI + ModelMetadata metadataIn(partition_filename, test_comm); + metadataIn.setTime(TimePoint(date_string)); + REQUIRE_THROWS(state = gridIn.getModelState(longRandomFilename, metadataIn)); +#else + REQUIRE_THROWS(state = gridIn.getModelState(longRandomFilename)); +#endif + +} TEST_CASE("Check if a file with the old dimension names can be read") { diff --git a/core/test/partition_metadata_2.nc.dump b/core/test/partition_metadata_2.nc.dump new file mode 100644 index 000000000..c500e440e --- /dev/null +++ b/core/test/partition_metadata_2.nc.dump @@ -0,0 +1,60 @@ +netcdf partition_metadata_2 { +dimensions: + globalX = 10 ; + globalY = 9 ; + P = 2 ; + T = 1 ; + B = 1 ; + L = UNLIMITED ; // (0 currently) + R = UNLIMITED ; // (0 currently) + +group: bounding_boxes { + variables: + int global_x(P) ; + int global_y(P) ; + int local_extent_x(P) ; + int local_extent_y(P) ; + data: + + global_x = 0, 4 ; + + global_y = 0, 0 ; + + local_extent_x = 4, 6 ; + + local_extent_y = 9, 9 ; + } // group bounding_boxes + +group: connectivity { + variables: + int top_neighbors(P) ; + int top_neighbor_ids(T) ; + int top_neighbor_halos(T) ; + int bottom_neighbors(P) ; + int bottom_neighbor_ids(B) ; + int bottom_neighbor_halos(B) ; + int left_neighbors(P) ; + int left_neighbor_ids(L) ; + int left_neighbor_halos(L) ; + int right_neighbors(P) ; + int right_neighbor_ids(R) ; + int right_neighbor_halos(R) ; + data: + + top_neighbors = 0, 1 ; + + top_neighbor_ids = 0 ; + + top_neighbor_halos = 9 ; + + bottom_neighbors = 1, 0 ; + + bottom_neighbor_ids = 1 ; + + bottom_neighbor_halos = 9 ; + + left_neighbors = 0, 0 ; + + right_neighbors = 0, 0 ; + } // group connectivity +} From af0bff64bb78f476e5a1283850674e331eddb249 Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 12 Jun 2024 13:11:35 +0100 Subject: [PATCH 07/25] mpi-ify old name test in paragrid --- core/test/CMakeLists.txt | 3 -- core/test/ParaGrid_test.cpp | 103 +++++++++++++++++++----------------- core/test/old_names.py | 56 ++++++++++---------- 3 files changed, 83 insertions(+), 79 deletions(-) diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 641e5c503..acd3d6064 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -137,9 +137,6 @@ else() target_include_directories(testParaGrid PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") target_link_libraries(testParaGrid PRIVATE nextsimlib doctest::doctest) - file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/old_names.nc" - DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") - add_executable(testConfigOutput "ConfigOutput_test.cpp") target_compile_definitions(testConfigOutput PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") target_include_directories(testConfigOutput PRIVATE ${MODEL_INCLUDE_DIR}) diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index 738bffa5b..ce1e789e1 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -209,24 +209,12 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") #ifdef USE_MPI metadata.setMpiMetadata(test_comm); - if (metadata.mpiMyRank == 0) { - metadata.setMpiMetadata(test_comm); - metadata.globalExtentX = nx; - metadata.globalExtentY = ny; - metadata.localCornerX = start_x; - metadata.localCornerY = 0; - metadata.localExtentX = local_nx; - metadata.localExtentY = ny; - } - if (metadata.mpiMyRank == 1) { - metadata.setMpiMetadata(test_comm); - metadata.globalExtentX = local_nx; - metadata.globalExtentY = ny; - metadata.localCornerX = start_x; - metadata.localCornerY = 0; - metadata.localExtentX = local_nx; - metadata.localExtentY = ny; - } + metadata.globalExtentX = nx; + metadata.globalExtentY = ny; + metadata.localCornerX = start_x; + metadata.localCornerY = 0; + metadata.localExtentX = local_nx; + metadata.localExtentY = ny; #endif grid.dumpModelState(state, metadata, filename, true); @@ -417,24 +405,12 @@ TEST_CASE("Write a diagnostic ParaGrid file") #ifdef USE_MPI metadata.setMpiMetadata(test_comm); - if (metadata.mpiMyRank == 0) { - metadata.setMpiMetadata(test_comm); - metadata.globalExtentX = nx; - metadata.globalExtentY = ny; - metadata.localCornerX = start_x; - metadata.localCornerY = 0; - metadata.localExtentX = local_nx; - metadata.localExtentY = ny; - } - if (metadata.mpiMyRank == 1) { - metadata.setMpiMetadata(test_comm); - metadata.globalExtentX = local_nx; - metadata.globalExtentY = ny; - metadata.localCornerX = start_x; - metadata.localCornerY = 0; - metadata.localExtentX = local_nx; - metadata.localExtentY = ny; - } + metadata.globalExtentX = nx; + metadata.globalExtentY = ny; + metadata.localCornerX = start_x; + metadata.localCornerY = 0; + metadata.localExtentX = local_nx; + metadata.localExtentY = ny; #endif for (int t = 1; t < 5; ++t) { @@ -536,7 +512,6 @@ TEST_CASE("Test array ordering") std::set fields = { fieldName }; TimePoint time; - printf("path = %s\n", inputFilename.c_str()); ModelState state = ParaGridIO::readForcingTimeStatic(fields, time, inputFilename); REQUIRE(state.data.count(fieldName) > 0); index2d = state.data.at(fieldName); @@ -567,15 +542,19 @@ TEST_CASE("Check an exception is thrown for an invalid file name") } +#ifdef USE_MPI +MPI_TEST_CASE("Check if a file with the old dimension names can be read", 2) +#else TEST_CASE("Check if a file with the old dimension names can be read") +#endif { - std::string inputFilename = "old_names.nc"; + std::string inputFilename = std::string(TEST_FILE_SOURCE) + "/old_names.nc"; Module::setImplementation("Nextsim::ParametricGrid"); REQUIRE(Module::getImplementation().structureType() == "parametric_rectangular"); - size_t nx = 1; + size_t nx = 2; size_t ny = 1; NZLevels::set(1); @@ -584,23 +563,51 @@ TEST_CASE("Check if a file with the old dimension names can be read") gridIn.setIO(readIO); // Reset the array dimensions to make sure that the read function gets them correct - ModelArray::setDimension(ModelArray::Dimension::X, 2); - ModelArray::setDimension(ModelArray::Dimension::Y, 2); - ModelArray::setDimension(ModelArray::Dimension::Z, 2); - ModelArray::setDimension(ModelArray::Dimension::XVERTEX, 3); - ModelArray::setDimension(ModelArray::Dimension::YVERTEX, 3); - ModelArray::setDimension(ModelArray::Dimension::XCG, 2); - ModelArray::setDimension(ModelArray::Dimension::YCG, 2); +#ifdef USE_MPI + ModelArray::setDimension(ModelArray::Dimension::X, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::Y, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::XCG, 1, 1, 0); + ModelArray::setDimension(ModelArray::Dimension::YCG, 1, 1, 0); +#else + ModelArray::setDimension(ModelArray::Dimension::X, 1); + ModelArray::setDimension(ModelArray::Dimension::Y, 1); + ModelArray::setDimension(ModelArray::Dimension::Z, 1); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, 1); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, 1); + ModelArray::setDimension(ModelArray::Dimension::XCG, 1); + ModelArray::setDimension(ModelArray::Dimension::YCG, 1); +#endif // In the full model numbers of DG components are set at compile time, so they are not reset REQUIRE(ModelArray::nComponents(ModelArray::Type::DG) == DG); REQUIRE(ModelArray::nComponents(ModelArray::Type::VERTEX) == ModelArray::nCoords); +#ifdef USE_MPI + ModelMetadata metadata; + metadata.setMpiMetadata(test_comm); + if (metadata.mpiMyRank == 0) { + metadata.localCornerX = 0; + } + if (metadata.mpiMyRank == 1) { + metadata.localCornerX = 1; + } + metadata.globalExtentX = nx; + metadata.globalExtentY = ny; + metadata.localCornerY = 0; + metadata.localExtentX = 1; + metadata.localExtentY = ny; + metadata.setTime(TimePoint(date_string)); + ModelState ms = gridIn.getModelState(inputFilename, metadata); +#else ModelState ms = gridIn.getModelState(inputFilename); +#endif - REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == nx); + auto local_nx = ModelArray::definedDimensions.at(ModelArray::Dimension::X).local_length; + REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == local_nx); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[1] == ny); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[2] == NZLevels::get()); - } TEST_SUITE_END(); diff --git a/core/test/old_names.py b/core/test/old_names.py index 79da9c7fa..9790e17c8 100644 --- a/core/test/old_names.py +++ b/core/test/old_names.py @@ -1,29 +1,29 @@ +import time + import netCDF4 import numpy as np -import time # Create a restart file that specifically uses the old dimension names if __name__ == "__main__": # Grid dimensions. x varies fastest nfirst = 1 - nsecond = 1 + nsecond = 2 nLayers = 1 ncg = 1 n_dg = 1 n_dgstress = 1 n_coords = 2 - - + root = netCDF4.Dataset("old_names.nc", "w", format="NETCDF4") - + structure_name = "parametric_rectangular" structgrp = root.createGroup("structure") structgrp.type = structure_name - + metagrp = root.createGroup("metadata") metagrp.type = structure_name - confgrp = metagrp.createGroup("configuration") # But add nothing to it + confgrp = metagrp.createGroup("configuration") # But add nothing to it timegrp = metagrp.createGroup("time") time_var = timegrp.createVariable("time", "i8") data_time = 1263204000 @@ -38,62 +38,62 @@ yDim = datagrp.createDimension("y", nfirst) xDim = datagrp.createDimension("x", nsecond) yVertexDim = datagrp.createDimension("yvertex", nfirst + 1) - xVertexDim = datagrp.createDimension("xvertex", nsecond+ 1) + xVertexDim = datagrp.createDimension("xvertex", nsecond + 1) ycg_dim = datagrp.createDimension("y_cg", nfirst * ncg + 1) xcg_dim = datagrp.createDimension("x_cg", nsecond * ncg + 1) dg_comp = datagrp.createDimension("dg_comp", n_dg) dgs_comp = datagrp.createDimension("dgstress_comp", n_dgstress) n_coords_comp = datagrp.createDimension("ncoords", n_coords) - + field_dims = ("y", "x") coord_dims = ("yvertex", "xvertex", "ncoords") zfield_dims = ("z", "y", "x") # Array coordinates - space = 25000. # 25 km in metres + space = 25000.0 # 25 km in metres x_vertex1d = np.arange(0, space * (nsecond + 1), space) - y_vertex1d = np.arange(0, space * (nfirst +1), space) - + y_vertex1d = np.arange(0, space * (nfirst + 1), space) + x_vertex = np.resize(x_vertex1d, (nfirst + 1, nsecond + 1)) y_vertex = np.resize(y_vertex1d, (nsecond + 1, nfirst + 1)).transpose() - + coords = datagrp.createVariable("coords", "f8", coord_dims) - coords[:,:,0] = x_vertex - coords[:,:,1] = y_vertex - + coords[:, :, 0] = x_vertex + coords[:, :, 1] = y_vertex + x_cell1d = np.arange(space * 0.5, space * (nsecond + 0.5), space) y_cell1d = np.arange(space * 0.5, space * (nfirst + 0.5), space) - + x_cell = np.resize(x_cell1d, (nfirst, nsecond)) y_cell = np.resize(y_cell1d, (nsecond, nfirst)).transpose() - + elem_x = datagrp.createVariable("x", "f8", field_dims) elem_x[:, :] = x_cell elem_y = datagrp.createVariable("y", "f8", field_dims) elem_y[:, :] = y_cell - + grid_azimuth = datagrp.createVariable("grid_azimuth", "f8", field_dims) # Return the grid azimuth to the range -180˚ to 180˚ - grid_azimuth[:, :] = 0. - + grid_azimuth[:, :] = 0.0 + # All sea, everywhere mask = datagrp.createVariable("mask", "f8", field_dims) mask[:, :] = 1 - + # Ice concentration and thickness cice = datagrp.createVariable("cice", "f8", field_dims) hice = datagrp.createVariable("hice", "f8", field_dims) - cice[:, :] = 0.96875 # 31/32 + cice[:, :] = 0.96875 # 31/32 hice[:, :] = 1.5 - + # Snow thickness hsnow = datagrp.createVariable("hsnow", "f8", field_dims) hsnow[:, :] = 0.125 # SSS sss = datagrp.createVariable("sss", "f8", field_dims) - sss[:, :] = 32. + sss[:, :] = 32.0 mu = -0.055 @@ -103,15 +103,15 @@ # Ice temperature tice = datagrp.createVariable("tice", "f8", zfield_dims) - ice_melt = mu * 5 # Melting point of sea ice (salinity = 5) in ˚C + ice_melt = mu * 5 # Melting point of sea ice (salinity = 5) in ˚C # Tice outside the ice pack is the melting point of pure water ice, which is conveniently 0˚C tice[0, :, :] = ice_melt - + # Ice starts at rest u = datagrp.createVariable("u", "f8", field_dims) u[:, :] = 0 v = datagrp.createVariable("v", "f8", field_dims) v[:, :] = 0 - + root.close() From 39f06874efff14cda2e6b00809c5d4633d4e65c6 Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 12 Jun 2024 13:17:03 +0100 Subject: [PATCH 08/25] chore: fix clang format --- core/src/ModelArray.cpp | 3 +- core/src/ParaGridIO.cpp | 93 +++++++++++++++------------------ core/src/include/ModelArray.hpp | 3 +- 3 files changed, 47 insertions(+), 52 deletions(-) diff --git a/core/src/ModelArray.cpp b/core/src/ModelArray.cpp index 522eac88d..d3b228066 100644 --- a/core/src/ModelArray.cpp +++ b/core/src/ModelArray.cpp @@ -201,7 +201,8 @@ void ModelArray::setNComponents(std::map cMap) } #ifdef USE_MPI -void ModelArray::setDimension(Dimension dim, size_t global_length, size_t local_length, size_t start) +void ModelArray::setDimension( + Dimension dim, size_t global_length, size_t local_length, size_t start) #else void ModelArray::setDimension(Dimension dim, size_t global_length) #endif diff --git a/core/src/ParaGridIO.cpp b/core/src/ParaGridIO.cpp index 65370e064..c2b442c65 100644 --- a/core/src/ParaGridIO.cpp +++ b/core/src/ParaGridIO.cpp @@ -129,30 +129,25 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) ModelArray::setDimension(dimType, NZLevels::get()); #endif } else { - // this needs MPIifying #ifdef USE_MPI auto dimName = dim.getName(); size_t local_length = 0; size_t start = 0; if (dimType == ModelArray::Dimension::X) { - local_length = metadata.localExtentX; - start = metadata.localCornerX; - } - else if (dimType == ModelArray::Dimension::Y) { - local_length = metadata.localExtentY; - start = metadata.localCornerY; - } - else if (dimType == ModelArray::Dimension::XVERTEX) { - local_length = metadata.localExtentX + 1; - start = metadata.localCornerX; - } - else if (dimType == ModelArray::Dimension::YVERTEX) { - local_length = metadata.localExtentY + 1; - start = metadata.localCornerY; - } - else{ - local_length = dim.getSize(); - start = 0; + local_length = metadata.localExtentX; + start = metadata.localCornerX; + } else if (dimType == ModelArray::Dimension::Y) { + local_length = metadata.localExtentY; + start = metadata.localCornerY; + } else if (dimType == ModelArray::Dimension::XVERTEX) { + local_length = metadata.localExtentX + 1; + start = metadata.localCornerX; + } else if (dimType == ModelArray::Dimension::YVERTEX) { + local_length = metadata.localExtentY + 1; + start = metadata.localCornerY; + } else { + local_length = dim.getSize(); + start = 0; } ModelArray::setDimension(dimType, dim.getSize(), local_length, start); #else @@ -184,16 +179,15 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) std::vector start; std::vector count; - if (ModelArray::hasDoF(type)){ - auto ncomps = data.nComponents(); - start.push_back(0); - count.push_back(ncomps); + if (ModelArray::hasDoF(type)) { + auto ncomps = data.nComponents(); + start.push_back(0); + count.push_back(ncomps); } - for (ModelArray::Dimension dt : ModelArray::typeDimensions.at(type)){ - auto dim = ModelArray::definedDimensions.at(dt); - start.push_back(dim.start); - count.push_back(dim.local_length); - + for (ModelArray::Dimension dt : ModelArray::typeDimensions.at(type)) { + auto dim = ModelArray::definedDimensions.at(dt); + start.push_back(dim.start); + count.push_back(dim.local_length); } // dims are looped in [dg], x, y, [z] order so start and count // order must be reveresed to match order netcdf expects @@ -321,16 +315,15 @@ void ParaGridIO::dumpModelState( ModelArray::Type type = entry.second.getType(); std::vector start; std::vector count; - if (ModelArray::hasDoF(type)){ - auto ncomps = entry.second.nComponents(); - start.push_back(0); - count.push_back(ncomps); + if (ModelArray::hasDoF(type)) { + auto ncomps = entry.second.nComponents(); + start.push_back(0); + count.push_back(ncomps); } - for (ModelArray::Dimension dt : entry.second.typeDimensions.at(type)){ - auto dim = entry.second.definedDimensions.at(dt); - start.push_back(dim.start); - count.push_back(dim.local_length); - + for (ModelArray::Dimension dt : entry.second.typeDimensions.at(type)) { + auto dim = entry.second.definedDimensions.at(dt); + start.push_back(dim.start); + count.push_back(dim.local_length); } // dims are looped in [dg], x, y, [z] order so start and count // order must be reveresed to match order netcdf expects @@ -405,20 +398,20 @@ void ParaGridIO::writeDiagnosticTime( std::vector count; // Everything that has components needs that dimension, too - if (ModelArray::hasDoF(type)){ - if (type == ModelArray::Type::VERTEX && !isNew) - continue; - auto ncomps = ModelArray::nComponents(type); - auto dim = ModelArray::componentMap.at(type); - ncDims.push_back(ncFromMAMap.at(dim)); - start.push_back(0); - count.push_back(ncomps); + if (ModelArray::hasDoF(type)) { + if (type == ModelArray::Type::VERTEX && !isNew) + continue; + auto ncomps = ModelArray::nComponents(type); + auto dim = ModelArray::componentMap.at(type); + ncDims.push_back(ncFromMAMap.at(dim)); + start.push_back(0); + count.push_back(ncomps); } - for (auto dt : entry.second){ - auto dim = ModelArray::definedDimensions.at(dt); - ncDims.push_back(ncFromMAMap.at(dt)); - start.push_back(dim.start); - count.push_back(dim.local_length); + for (auto dt : entry.second) { + auto dim = ModelArray::definedDimensions.at(dt); + ncDims.push_back(ncFromMAMap.at(dt)); + start.push_back(dim.start); + count.push_back(dim.local_length); } std::reverse(ncDims.begin(), ncDims.end()); diff --git a/core/src/include/ModelArray.hpp b/core/src/include/ModelArray.hpp index 0f19a6140..c26bcc9c4 100644 --- a/core/src/include/ModelArray.hpp +++ b/core/src/include/ModelArray.hpp @@ -287,7 +287,8 @@ class ModelArray { { if (size() != trueSize()) { if (hasDoF(type)) { - m_data.resize(m_sz.at(type), definedDimensions.at(componentMap.at(type)).local_length); + m_data.resize( + m_sz.at(type), definedDimensions.at(componentMap.at(type)).local_length); } else { m_data.resize(m_sz.at(type), Eigen::NoChange); } From 10b6684b3bf6a0b5d31323b8a84ddf7622e8fe40 Mon Sep 17 00:00:00 2001 From: melt Date: Thu, 13 Jun 2024 13:51:57 +0100 Subject: [PATCH 09/25] mpi-ify physics tests parallelize ERA5Atm_test and TOPAZOcn_test update CMakeLists.txt to only build MPI tests when MPI is enabled only test explicitly using mpi doctest will be built and run under MPI test run --- .github/workflows/clang-compile-tests.yml | 4 +- core/test/CMakeLists.txt | 113 +++++++++---------- physics/test/CMakeLists.txt | 128 ++++++++++++---------- physics/test/ERA5Atm_test.cpp | 31 ++++-- physics/test/TOPAZOcn_test.cpp | 30 +++-- 5 files changed, 161 insertions(+), 145 deletions(-) diff --git a/.github/workflows/clang-compile-tests.yml b/.github/workflows/clang-compile-tests.yml index 600628ac3..379ae4946 100644 --- a/.github/workflows/clang-compile-tests.yml +++ b/.github/workflows/clang-compile-tests.yml @@ -54,7 +54,7 @@ jobs: for component in core physics dynamics do cd $component/test - for file in $(find test* -maxdepth 0 -type f | grep -vP "_MPI\d*$") + for file in $(find test* -maxdepth 0 -type f) do echo $file ./$file @@ -91,7 +91,7 @@ jobs: for component in core physics dynamics do cd $component/test - for file in $(find test* -maxdepth 0 -type f | grep -P "_MPI\d+$") + for file in $(find test* -maxdepth 0 -type f) do echo $file nprocs=$(echo $file | sed -r "s/.*MPI([0-9]+)/\1/") diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index acd3d6064..127014586 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -16,34 +16,6 @@ set(ModulesRoot "../src/modules") execute_process(COMMAND "${Python_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/old_names.py" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") include_directories(${COMMON_INCLUDE_DIRS}) - -add_executable(testIterator "Iterator_test.cpp") -target_link_libraries(testIterator PRIVATE nextsimlib doctest::doctest) - -add_executable(testCommandLineParser "CommandLineParser_test.cpp" "ArgV.cpp") -target_link_libraries(testCommandLineParser PRIVATE nextsimlib doctest::doctest) - -add_executable(testConfigurator "Configurator_test.cpp" "ArgV.cpp") -target_link_libraries(testConfigurator PRIVATE nextsimlib doctest::doctest) - -add_executable(testConfiguredModule "ConfiguredModule_test.cpp" "ArgV.cpp") -target_link_libraries(testConfiguredModule PRIVATE nextsimlib doctest::doctest) - -add_executable(testTimer "Timer_test.cpp") -target_link_libraries(testTimer PRIVATE nextsimlib doctest::doctest) - -add_executable(testScopedTimer "ScopedTimer_test.cpp" "../src/ScopedTimer.cpp") -target_link_libraries(testScopedTimer PRIVATE nextsimlib doctest::doctest) - -add_executable(testTimeClasses "Time_test.cpp") -target_link_libraries(testTimeClasses PRIVATE nextsimlib doctest::doctest) - -set(MODEL_INCLUDE_DIR "./testmodelarraydetails") - -add_executable(testModelArray "ModelArray_test.cpp" "../src/ModelArray.cpp" "${MODEL_INCLUDE_DIR}/ModelArrayDetails.cpp") -target_include_directories(testModelArray PRIVATE "../src" "${MODEL_INCLUDE_DIR}") -target_link_libraries(testModelArray PRIVATE doctest::doctest Eigen3::Eigen) - set(MODEL_INCLUDE_DIR "../../core/src/discontinuousgalerkin") if(ENABLE_MPI) @@ -58,6 +30,7 @@ if(ENABLE_MPI) ) add_custom_target(generate_partition_files ALL DEPENDS partition_metadata_3.nc partition_metadata_2.nc) + add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp") target_compile_definitions(testRectGrid_MPI3 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") target_include_directories(testRectGrid_MPI3 PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") @@ -127,6 +100,7 @@ if(ENABLE_MPI) target_link_libraries(testXiosRead_MPI2 PRIVATE nextsimlib doctest::doctest) endif() else() + add_executable(testRectGrid "RectGrid_test.cpp") target_compile_definitions(testRectGrid PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") target_include_directories(testRectGrid PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule") @@ -141,51 +115,64 @@ else() target_compile_definitions(testConfigOutput PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\") target_include_directories(testConfigOutput PRIVATE ${MODEL_INCLUDE_DIR}) target_link_libraries(testConfigOutput PRIVATE nextsimlib doctest::doctest) -endif() -add_executable(testModelComponent "ModelComponent_test.cpp") -target_include_directories(testModelComponent PRIVATE ${MODEL_INCLUDE_DIR}) -target_link_libraries(testModelComponent PRIVATE nextsimlib doctest::doctest) + add_executable(testIterator "Iterator_test.cpp") + target_link_libraries(testIterator PRIVATE nextsimlib doctest::doctest) -add_executable(testModelArrayRef "ModelArrayRef_test.cpp") -target_include_directories(testModelArrayRef PRIVATE ${MODEL_INCLUDE_DIR}) -target_link_libraries(testModelArrayRef PRIVATE nextsimlib doctest::doctest) + add_executable(testCommandLineParser "CommandLineParser_test.cpp" "ArgV.cpp") + target_link_libraries(testCommandLineParser PRIVATE nextsimlib doctest::doctest) -# PrognosticData (and hopefully that alone) requires code from the physics tree -add_executable(testPrognosticData "PrognosticData_test.cpp" "DynamicsModuleForPDtest.cpp") -target_include_directories(testPrognosticData PRIVATE ${PHYSICS_INCLUDE_DIRS} ${MODEL_INCLUDE_DIR} "${ModulesRoot}/FreezingPointModule" "${ModulesRoot}/DynamicsModule") -target_link_libraries(testPrognosticData PRIVATE nextsimlib doctest::doctest) + add_executable(testConfigurator "Configurator_test.cpp" "ArgV.cpp") + target_link_libraries(testConfigurator PRIVATE nextsimlib doctest::doctest) -# PrognosticDataIO, which cannot currently be tested with MPI. -# TODO: itegrate with the above test. -if(ENABLE_MPI) - message(WARNING "testPrognosticDataIO has been temporarily disabled when running with MPI enabled") -else() + add_executable(testConfiguredModule "ConfiguredModule_test.cpp" "ArgV.cpp") + target_link_libraries(testConfiguredModule PRIVATE nextsimlib doctest::doctest) + + add_executable(testTimer "Timer_test.cpp") + target_link_libraries(testTimer PRIVATE nextsimlib doctest::doctest) + + add_executable(testScopedTimer "ScopedTimer_test.cpp" "../src/ScopedTimer.cpp") + target_link_libraries(testScopedTimer PRIVATE nextsimlib doctest::doctest) + + add_executable(testTimeClasses "Time_test.cpp") + target_link_libraries(testTimeClasses PRIVATE nextsimlib doctest::doctest) + + add_executable(testModelComponent "ModelComponent_test.cpp") + target_include_directories(testModelComponent PRIVATE ${MODEL_INCLUDE_DIR}) + target_link_libraries(testModelComponent PRIVATE nextsimlib doctest::doctest) + + add_executable(testModelArrayRef "ModelArrayRef_test.cpp") + target_include_directories(testModelArrayRef PRIVATE ${MODEL_INCLUDE_DIR}) + target_link_libraries(testModelArrayRef PRIVATE nextsimlib doctest::doctest) + + # PrognosticData (and hopefully that alone) requires code from the physics tree + add_executable(testPrognosticData "PrognosticData_test.cpp" "DynamicsModuleForPDtest.cpp") + target_include_directories(testPrognosticData PRIVATE ${PHYSICS_INCLUDE_DIRS} ${MODEL_INCLUDE_DIR} "${ModulesRoot}/FreezingPointModule" "${ModulesRoot}/DynamicsModule") + target_link_libraries(testPrognosticData PRIVATE nextsimlib doctest::doctest) + + # TODO: itegrate with the above test. add_executable(testPrognosticDataIO "PrognosticDataIO_test.cpp" "DynamicsModuleForPDtest.cpp") target_compile_definitions(testPrognosticDataIO PRIVATE ISDG=${isDG}) target_include_directories(testPrognosticDataIO PRIVATE ${PHYSICS_INCLUDE_DIRS} ${MODEL_INCLUDE_DIR} "${ModulesRoot}/FreezingPointModule" "${ModulesRoot}/DynamicsModule" "${ModulesRoot}/StructureModule") target_link_libraries(testPrognosticDataIO PRIVATE nextsimlib doctest::doctest) -endif() + add_executable(testMonthlyCubicBSpline + "MonthlyCubicBSpline_test.cpp" + ) + target_link_libraries(testMonthlyCubicBSpline LINK_PUBLIC Boost::boost doctest::doctest) -if(ENABLE_MPI) - message(WARNING "testConfigOutput has been temporarily disabled when running with MPI enabled") -else() - add_executable(testConfigOutput "ConfigOutput_test.cpp") - target_include_directories(testConfigOutput PRIVATE ${MODEL_INCLUDE_DIR}) - target_link_libraries(testConfigOutput PRIVATE nextsimlib doctest::doctest) -endif() + add_executable(testFileCallbackCloser + "FileCallbackCloser_test.cpp" + "../../core/src/FileCallbackCloser.cpp" + ) + target_link_libraries(testFileCallbackCloser PRIVATE doctest::doctest) -add_executable(testMonthlyCubicBSpline - "MonthlyCubicBSpline_test.cpp" - ) -target_link_libraries(testMonthlyCubicBSpline LINK_PUBLIC Boost::boost doctest::doctest) + set(MODEL_INCLUDE_DIR "./testmodelarraydetails") -add_executable(testFileCallbackCloser - "FileCallbackCloser_test.cpp" - "../../core/src/FileCallbackCloser.cpp" - ) -target_link_libraries(testFileCallbackCloser PRIVATE doctest::doctest) + add_executable(testModelArray "ModelArray_test.cpp" "../src/ModelArray.cpp" "${MODEL_INCLUDE_DIR}/ModelArrayDetails.cpp") + target_include_directories(testModelArray PRIVATE "../src" "${MODEL_INCLUDE_DIR}") + target_link_libraries(testModelArray PRIVATE doctest::doctest Eigen3::Eigen) -# The help config test needs access to the full binary -FILE(CREATE_LINK ${NEXTSIM_BINARY_PATH} ${CMAKE_CURRENT_BINARY_DIR}/nextsim SYMBOLIC) + # The help config test needs access to the full binary + FILE(CREATE_LINK ${NEXTSIM_BINARY_PATH} ${CMAKE_CURRENT_BINARY_DIR}/nextsim SYMBOLIC) +endif() diff --git a/physics/test/CMakeLists.txt b/physics/test/CMakeLists.txt index 908d82842..f5d4013f3 100644 --- a/physics/test/CMakeLists.txt +++ b/physics/test/CMakeLists.txt @@ -15,63 +15,71 @@ execute_process(COMMAND "${Python_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/topa include_directories(${COMMON_INCLUDE_DIRS}) -add_executable(testIceGrowth "IceGrowth_test.cpp") -target_include_directories(testIceGrowth PRIVATE "${ModulesRoot}/OceanBoundaryModule" "${CoreModulesRoot}/FreezingPointModule") -target_link_libraries(testIceGrowth PRIVATE nextsimlib doctest::doctest) - -add_executable(testThermoWinton "ThermoWintonTemperature_test.cpp") -target_include_directories(testThermoWinton PRIVATE "${ModulesRoot}/IceThermodynamicsModule" "${ModulesRoot}/OceanBoundaryModule") -target_link_libraries(testThermoWinton PRIVATE nextsimlib doctest::doctest) - -add_executable(testFEFluxes "FiniteElementFluxes_test.cpp") -target_include_directories(testFEFluxes PRIVATE "${ModulesRoot}/FluxCalculationModule" "${ModulesRoot}/OceanBoundaryModule" "${CoreModulesRoot}/FreezingPointModule") -target_link_libraries(testFEFluxes PRIVATE nextsimlib doctest::doctest) - -add_executable(testBIOHFluxes "BasicIceOceanFlux_test.cpp") -target_include_directories(testBIOHFluxes PRIVATE "${ModulesRoot}/IceOceanHeatFluxModule" "${ModulesRoot}/OceanBoundaryModule" "${CoreModulesRoot}/FreezingPointModule") -target_link_libraries(testBIOHFluxes PRIVATE nextsimlib doctest::doctest) - -add_executable(testSpecHum "SpecificHumidity_test.cpp") -target_include_directories(testSpecHum PRIVATE "${ModulesRoot}/FluxCalculationModule") -target_link_libraries(testSpecHum PRIVATE nextsimlib doctest::doctest) - -add_executable(testConstantOcn "ConstantOceanBoundary_test.cpp") -target_include_directories(testConstantOcn PRIVATE "${ModulesRoot}/OceanBoundaryModule") -target_link_libraries(testConstantOcn PRIVATE nextsimlib doctest::doctest) - -add_executable(testSlabOcn "SlabOcean_test.cpp") -target_include_directories(testSlabOcn PRIVATE "${CoreModulesRoot}/FreezingPointModule") -target_link_libraries(testSlabOcn PRIVATE nextsimlib doctest::doctest) - -add_executable(testThermoIce0 "ThermoIce0_test.cpp") -target_include_directories(testThermoIce0 PRIVATE "${ModulesRoot}/IceThermodynamicsModule") -target_link_libraries(testThermoIce0 PRIVATE nextsimlib doctest::doctest) - -add_executable(testIceMinima "IceMinima_test.cpp") -target_link_libraries(testIceMinima PRIVATE nextsimlib doctest::doctest) - -add_executable(testERA5Atm "ERA5Atm_test.cpp") -target_include_directories(testERA5Atm PRIVATE "${ModulesRoot}/AtmosphereBoundaryModule") -target_compile_definitions(testERA5Atm PRIVATE TEST_FILE_SOURCE=${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(testERA5Atm PRIVATE nextsimlib doctest::doctest) -file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/era5_test128x128.nc" - DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") - -add_executable(testTOPAZOcn "TOPAZOcn_test.cpp") -target_include_directories(testTOPAZOcn PRIVATE "${ModulesRoot}/OceanBoundaryModule") -target_compile_definitions(testTOPAZOcn PRIVATE TEST_FILE_SOURCE=${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(testTOPAZOcn PRIVATE nextsimlib doctest::doctest) -file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/topaz_test128x128.nc" - DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") - -add_executable(testUniformOcean "UniformOcean_test.cpp") -target_include_directories(testUniformOcean PRIVATE "${ModulesRoot}/OceanBoundaryModule") -target_link_libraries(testUniformOcean PRIVATE nextsimlib doctest::doctest) - -add_executable(testBenchmarkBoundaries "BenchmarkBoundaries_test.cpp") -target_include_directories(testBenchmarkBoundaries PRIVATE "${ModulesRoot}/AtmosphereBoundaryModule" "${ModulesRoot}/OceanBoundaryModule") -target_link_libraries(testBenchmarkBoundaries PRIVATE nextsimlib doctest::doctest) - -add_executable(testDamageHealing "DamageHealing_test.cpp") -target_include_directories(testDamageHealing PRIVATE "${ModulesRoot}/DamageHealingModule" "${ModulesRoot}/DamageHealingModule") -target_link_libraries(testDamageHealing PRIVATE nextsimlib doctest::doctest) +if(ENABLE_MPI) + add_executable(testERA5Atm_MPI1 "ERA5Atm_test.cpp" "../../core/test/MainMPI.cpp") + target_compile_definitions(testERA5Atm_MPI1 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\") + target_include_directories(testERA5Atm_MPI1 PRIVATE "${ModulesRoot}/AtmosphereBoundaryModule") + target_link_libraries(testERA5Atm_MPI1 PRIVATE nextsimlib doctest::doctest) + + add_executable(testTOPAZOcn_MPI1 "TOPAZOcn_test.cpp" "../../core/test/MainMPI.cpp") + target_include_directories(testTOPAZOcn_MPI1 PRIVATE "${ModulesRoot}/OceanBoundaryModule") + target_compile_definitions(testTOPAZOcn_MPI1 PRIVATE USE_MPI TEST_FILES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\") + target_link_libraries(testTOPAZOcn_MPI1 PRIVATE nextsimlib doctest::doctest) +else() + add_executable(testERA5Atm "ERA5Atm_test.cpp") + target_compile_definitions(testERA5Atm PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\") + target_include_directories(testERA5Atm PRIVATE "${ModulesRoot}/AtmosphereBoundaryModule") + target_link_libraries(testERA5Atm PRIVATE nextsimlib doctest::doctest) + + add_executable(testTOPAZOcn "TOPAZOcn_test.cpp") + target_include_directories(testTOPAZOcn PRIVATE "${ModulesRoot}/OceanBoundaryModule") + target_compile_definitions(testTOPAZOcn PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\") + target_link_libraries(testTOPAZOcn PRIVATE nextsimlib doctest::doctest) + + add_executable(testIceGrowth "IceGrowth_test.cpp") + target_include_directories(testIceGrowth PRIVATE "${ModulesRoot}/OceanBoundaryModule" "${CoreModulesRoot}/FreezingPointModule") + target_link_libraries(testIceGrowth PRIVATE nextsimlib doctest::doctest) + + add_executable(testThermoWinton "ThermoWintonTemperature_test.cpp") + target_include_directories(testThermoWinton PRIVATE "${ModulesRoot}/IceThermodynamicsModule" "${ModulesRoot}/OceanBoundaryModule") + target_link_libraries(testThermoWinton PRIVATE nextsimlib doctest::doctest) + + add_executable(testFEFluxes "FiniteElementFluxes_test.cpp") + target_include_directories(testFEFluxes PRIVATE "${ModulesRoot}/FluxCalculationModule" "${ModulesRoot}/OceanBoundaryModule" "${CoreModulesRoot}/FreezingPointModule") + target_link_libraries(testFEFluxes PRIVATE nextsimlib doctest::doctest) + + add_executable(testBIOHFluxes "BasicIceOceanFlux_test.cpp") + target_include_directories(testBIOHFluxes PRIVATE "${ModulesRoot}/IceOceanHeatFluxModule" "${ModulesRoot}/OceanBoundaryModule" "${CoreModulesRoot}/FreezingPointModule") + target_link_libraries(testBIOHFluxes PRIVATE nextsimlib doctest::doctest) + + add_executable(testSpecHum "SpecificHumidity_test.cpp") + target_include_directories(testSpecHum PRIVATE "${ModulesRoot}/FluxCalculationModule") + target_link_libraries(testSpecHum PRIVATE nextsimlib doctest::doctest) + + add_executable(testThermoIce0 "ThermoIce0_test.cpp") + target_include_directories(testThermoIce0 PRIVATE "${ModulesRoot}/IceThermodynamicsModule") + target_link_libraries(testThermoIce0 PRIVATE nextsimlib doctest::doctest) + + add_executable(testIceMinima "IceMinima_test.cpp") + target_link_libraries(testIceMinima PRIVATE nextsimlib doctest::doctest) + + add_executable(testConstantOcn "ConstantOceanBoundary_test.cpp") + target_include_directories(testConstantOcn PRIVATE "${ModulesRoot}/OceanBoundaryModule") + target_link_libraries(testConstantOcn PRIVATE nextsimlib doctest::doctest) + + add_executable(testSlabOcn "SlabOcean_test.cpp") + target_include_directories(testSlabOcn PRIVATE "${CoreModulesRoot}/FreezingPointModule") + target_link_libraries(testSlabOcn PRIVATE nextsimlib doctest::doctest) + + add_executable(testUniformOcean "UniformOcean_test.cpp") + target_include_directories(testUniformOcean PRIVATE "${ModulesRoot}/OceanBoundaryModule") + target_link_libraries(testUniformOcean PRIVATE nextsimlib doctest::doctest) + + add_executable(testBenchmarkBoundaries "BenchmarkBoundaries_test.cpp") + target_include_directories(testBenchmarkBoundaries PRIVATE "${ModulesRoot}/AtmosphereBoundaryModule" "${ModulesRoot}/OceanBoundaryModule") + target_link_libraries(testBenchmarkBoundaries PRIVATE nextsimlib doctest::doctest) + + add_executable(testDamageHealing "DamageHealing_test.cpp") + target_include_directories(testDamageHealing PRIVATE "${ModulesRoot}/DamageHealingModule" "${ModulesRoot}/DamageHealingModule") + target_link_libraries(testDamageHealing PRIVATE nextsimlib doctest::doctest) +endif() diff --git a/physics/test/ERA5Atm_test.cpp b/physics/test/ERA5Atm_test.cpp index 624f57465..d36dc5dff 100644 --- a/physics/test/ERA5Atm_test.cpp +++ b/physics/test/ERA5Atm_test.cpp @@ -5,8 +5,12 @@ * @author Tim Spain */ +#ifdef USE_MPI +#include +#else #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include +#endif #include "include/ERA5Atmosphere.hpp" @@ -18,10 +22,8 @@ #include #include -#define TO_STR(s) TO_STRI(s) -#define TO_STRI(s) #s -#ifndef TEST_FILE_SOURCE -#define TEST_FILE_SOURCE . +#ifndef TEST_FILES_DIR +#define TEST_FILES_DIR "." #endif extern template class Module::Module; @@ -29,24 +31,33 @@ extern template class Module::Module; namespace Nextsim { TEST_SUITE_BEGIN("ERA5Atmosphere"); +#ifdef USE_MPI +MPI_TEST_CASE("ERA5Atmosphere construction test", 1) +#else TEST_CASE("ERA5Atmosphere construction test") +#endif { - std::string filePath = "era5_test128x128.nc"; - std::string sourceDir = TO_STR(TEST_FILE_SOURCE); - // Copy the test file from the test source directory to the working directory - if (!std::filesystem::exists(filePath)) { - std::filesystem::copy(sourceDir + "/" + filePath, "."); - } + const std::string filePath = "era5_test128x128.nc"; + const std::string orig_file = std::string(TEST_FILES_DIR) + "/" + filePath; + std::filesystem::copy(orig_file, filePath, std::filesystem::copy_options::overwrite_existing); + // In the real model, the array sizes will have been set by the restart file by this point size_t nx = 128; size_t ny = 128; size_t nxvertex = nx + 1; size_t nyvertex = ny + 1; +#ifdef USE_MPI + ModelArray::setDimension(ModelArray::Dimension::X, nx, nx, 0); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nxvertex, nxvertex, 0); + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, nyvertex, nyvertex, 0); +#else ModelArray::setDimension(ModelArray::Dimension::X, nx); ModelArray::setDimension(ModelArray::Dimension::Y, ny); ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nxvertex); ModelArray::setDimension(ModelArray::Dimension::YVERTEX, nyvertex); +#endif ERA5Atmosphere e5; diff --git a/physics/test/TOPAZOcn_test.cpp b/physics/test/TOPAZOcn_test.cpp index 749c8a7b3..f9bbfc59a 100644 --- a/physics/test/TOPAZOcn_test.cpp +++ b/physics/test/TOPAZOcn_test.cpp @@ -5,8 +5,12 @@ * @author Tim Spain */ +#ifdef USE_MPI +#include +#else #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include +#endif #include "include/TOPAZOcean.hpp" @@ -16,23 +20,22 @@ #include -#define TO_STR(s) TO_STRI(s) -#define TO_STRI(s) #s -#ifndef TEST_FILE_SOURCE -#define TEST_FILE_SOURCE . +#ifndef TEST_FILES_DIR +#define TEST_FILES_DIR "." #endif namespace Nextsim { TEST_SUITE_BEGIN("TOPAZOcean"); +#ifdef USE_MPI +MPI_TEST_CASE("TOPAZOcean test", 1) +#else TEST_CASE("TOPAZOcean test") +#endif { - std::string filePath = "topaz_test128x128.nc"; - std::string sourceDir = TO_STR(TEST_FILE_SOURCE); - // Copy the test file from the test source directory to the working directory - if (!std::filesystem::exists(filePath)) { - std::filesystem::copy(sourceDir + "/" + filePath, "."); - } + const std::string filePath = "topaz_test128x128.nc"; + const std::string orig_file = std::string(TEST_FILES_DIR) + "/" + filePath; + std::filesystem::copy(orig_file, filePath, std::filesystem::copy_options::overwrite_existing); // In the real model, the array sizes will have been set by the restart file by this point size_t nx = 128; @@ -40,10 +43,17 @@ TEST_CASE("TOPAZOcean test") size_t nxvertex = nx + 1; size_t nyvertex = ny + 1; +#ifdef USE_MPI + ModelArray::setDimension(ModelArray::Dimension::X, nx, nx, 0); + ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nxvertex, nxvertex, 0); + ModelArray::setDimension(ModelArray::Dimension::Y, ny, ny, 0); + ModelArray::setDimension(ModelArray::Dimension::YVERTEX, nyvertex, nyvertex, 0); +#else ModelArray::setDimension(ModelArray::Dimension::X, nx); ModelArray::setDimension(ModelArray::Dimension::Y, ny); ModelArray::setDimension(ModelArray::Dimension::XVERTEX, nxvertex); ModelArray::setDimension(ModelArray::Dimension::YVERTEX, nyvertex); +#endif TOPAZOcean topaz; topaz.configure(); From 2d2bf24b701c4e00d7be0be48b0c3beca0ee9f82 Mon Sep 17 00:00:00 2001 From: melt Date: Mon, 24 Jun 2024 10:29:01 +0100 Subject: [PATCH 10/25] fix defaults in ModelArrayDetails `ModelArray::definedDimensions` were not set correctly. This commit sets default global size, local size and start position for each dimension --- .../ModelArrayDetails.cpp | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/core/src/discontinuousgalerkin/ModelArrayDetails.cpp b/core/src/discontinuousgalerkin/ModelArrayDetails.cpp index 39e86ad8e..a7cc7ed8e 100644 --- a/core/src/discontinuousgalerkin/ModelArrayDetails.cpp +++ b/core/src/discontinuousgalerkin/ModelArrayDetails.cpp @@ -28,17 +28,18 @@ namespace Nextsim { // clang-format off std::map ModelArray::definedDimensions = { - { ModelArray::Dimension::X, { "xdim", "x", 0 } }, - { ModelArray::Dimension::Y, { "ydim", "y", 0 } }, - { ModelArray::Dimension::Z, { "zdim", "z", 1 } }, - { ModelArray::Dimension::XVERTEX, { "xvertex", "xvertex", 1 } }, // defined as x + 1 - { ModelArray::Dimension::YVERTEX, { "yvertex", "yvertex", 1 } }, // defined as y + 1 - { ModelArray::Dimension::XCG, { "x_cg", "x_cg", CGDEGREE } }, - { ModelArray::Dimension::YCG, { "y_cg", "y_cg", CGDEGREE } }, + // set default global size, local size and start position for each dimension + { ModelArray::Dimension::X, { "xdim", "x", 0, 0, 0 } }, + { ModelArray::Dimension::Y, { "ydim", "y", 0, 0, 0 } }, + { ModelArray::Dimension::Z, { "zdim", "z", 1, 1, 0 } }, + { ModelArray::Dimension::XVERTEX, { "xvertex", "xvertex", 1, 1, 0 } }, // defined as x + 1 + { ModelArray::Dimension::YVERTEX, { "yvertex", "yvertex", 1, 1, 0 } }, // defined as y + 1 + { ModelArray::Dimension::XCG, { "x_cg", "x_cg", CGDEGREE, CGDEGREE, 0 } }, + { ModelArray::Dimension::YCG, { "y_cg", "y_cg", CGDEGREE, CGDEGREE, 0 } }, // The DG components are also included here to store the names - { ModelArray::Dimension::DG, { "dg_comp", "dg_comp", DGCOMP } }, - { ModelArray::Dimension::DGSTRESS, { "dgstress_comp", "dgstress_comp", DGSTRESSCOMP } }, - { ModelArray::Dimension::NCOORDS, { "ncoords", "ncoords", 2 } }, // It's a two dimensional model + { ModelArray::Dimension::DG, { "dg_comp", "dg_comp", DGCOMP, DGCOMP, 0 } }, + { ModelArray::Dimension::DGSTRESS, { "dgstress_comp", "dgstress_comp", DGSTRESSCOMP, DGSTRESSCOMP, 0 } }, + { ModelArray::Dimension::NCOORDS, { "ncoords", "ncoords", 2, 2, 0 } }, // It's a two dimensional model // clang-format on }; From 68ffbcdf28a7e9a95eea25323d6e48315386079e Mon Sep 17 00:00:00 2001 From: melt Date: Mon, 24 Jun 2024 10:30:29 +0100 Subject: [PATCH 11/25] fix dimension order of diagnostic output time is now the first dimension e.g., `var(time, ydim, xdim)` --- core/src/ParaGridIO.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/ParaGridIO.cpp b/core/src/ParaGridIO.cpp index c2b442c65..f67e30c87 100644 --- a/core/src/ParaGridIO.cpp +++ b/core/src/ParaGridIO.cpp @@ -414,10 +414,6 @@ void ParaGridIO::writeDiagnosticTime( count.push_back(dim.local_length); } - std::reverse(ncDims.begin(), ncDims.end()); - std::reverse(start.begin(), start.end()); - std::reverse(count.begin(), count.end()); - // Deal with VERTEX in each case // Add the time dimension for all types that are not VERTEX if (type != ModelArray::Type::VERTEX) { @@ -429,6 +425,10 @@ void ParaGridIO::writeDiagnosticTime( continue; } + std::reverse(ncDims.begin(), ncDims.end()); + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); + dimMap[type] = ncDims; startMap[type] = start; countMap[type] = count; From 93e65994e9a922c1af3ddec375085e2995ed4fa3 Mon Sep 17 00:00:00 2001 From: melt Date: Mon, 24 Jun 2024 12:18:42 +0100 Subject: [PATCH 12/25] update docstring --- core/src/include/ModelArray.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/include/ModelArray.hpp b/core/src/include/ModelArray.hpp index c26bcc9c4..f92aa9a27 100644 --- a/core/src/include/ModelArray.hpp +++ b/core/src/include/ModelArray.hpp @@ -234,7 +234,7 @@ class ModelArray { static size_t size(Type type) { return m_sz.at(type); } //! Returns the size of the data array of this object. size_t trueSize() const { return m_data.rows(); } - //! Returns the size of a dimension + //! Returns the local size of a dimension static size_t size(Dimension dim) { return definedDimensions.at(dim).local_length; } //! Returns a read-only pointer to the underlying data buffer. From 86408ed8d61dd16fd30f1e9f216b3b70da751c11 Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 24 Jul 2024 16:53:19 +0100 Subject: [PATCH 13/25] chore: convert length vars to camelCase --- core/src/ModelArray.cpp | 10 +++++----- core/src/ParaGridIO.cpp | 16 ++++++++-------- core/src/include/ModelArray.hpp | 14 +++++++------- .../StructureModule/include/ParametricGrid.hpp | 2 +- core/test/ConfigOutput_test.cpp | 2 +- core/test/ParaGrid_test.cpp | 14 +++++++------- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/core/src/ModelArray.cpp b/core/src/ModelArray.cpp index d3b228066..06d3b9b8b 100644 --- a/core/src/ModelArray.cpp +++ b/core/src/ModelArray.cpp @@ -188,7 +188,7 @@ void ModelArray::setDimensions(Type type, const MultiDim& newDims) { std::vector& dimSpecs = typeDimensions.at(type); for (size_t i = 0; i < dimSpecs.size(); ++i) { - definedDimensions.at(dimSpecs[i]).local_length = newDims[i]; + definedDimensions.at(dimSpecs[i]).localLength = newDims[i]; } validateMaps(); } @@ -208,8 +208,8 @@ void ModelArray::setDimension(Dimension dim, size_t global_length) #endif { #ifdef USE_MPI - definedDimensions.at(dim).global_length = global_length; - definedDimensions.at(dim).local_length = local_length; + definedDimensions.at(dim).globalLength = global_length; + definedDimensions.at(dim).localLength = local_length; definedDimensions.at(dim).start = start; #else // if MPI is not used then set the local_length to be the same as the global @@ -285,7 +285,7 @@ void ModelArray::DimensionMap::validate() std::vector& typeDims = entry.second; dims.resize(typeDims.size()); for (size_t i = 0; i < typeDims.size(); ++i) { - dims[i] = definedDimensions.at(typeDims[i]).local_length; + dims[i] = definedDimensions.at(typeDims[i]).localLength; } } } @@ -296,7 +296,7 @@ void ModelArray::SizeMap::validate() size_t size = 1; std::vector& typeDims = entry.second; for (size_t i = 0; i < typeDims.size(); ++i) { - size *= definedDimensions.at(typeDims[i]).local_length; + size *= definedDimensions.at(typeDims[i]).localLength; } m_sizes.at(entry.first) = size; } diff --git a/core/src/ParaGridIO.cpp b/core/src/ParaGridIO.cpp index f67e30c87..a08275e78 100644 --- a/core/src/ParaGridIO.cpp +++ b/core/src/ParaGridIO.cpp @@ -187,7 +187,7 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) for (ModelArray::Dimension dt : ModelArray::typeDimensions.at(type)) { auto dim = ModelArray::definedDimensions.at(dt); start.push_back(dim.start); - count.push_back(dim.local_length); + count.push_back(dim.localLength); } // dims are looped in [dg], x, y, [z] order so start and count // order must be reveresed to match order netcdf expects @@ -242,7 +242,7 @@ ModelState ParaGridIO::readForcingTimeStatic( = ModelArray::typeDimensions.at(ModelArray::Type::H); for (auto riter = dimensions.rbegin(); riter != dimensions.rend(); ++riter) { indexArray.push_back(0); - extentArray.push_back(ModelArray::definedDimensions.at(*riter).local_length); + extentArray.push_back(ModelArray::definedDimensions.at(*riter).localLength); } for (const std::string& varName : forcings) { @@ -282,7 +282,7 @@ void ParaGridIO::dumpModelState( for (auto entry : ModelArray::definedDimensions) { ModelArray::Dimension dim = entry.first; size_t dimSz = (dimCompMap.count(dim)) ? ModelArray::nComponents(dimCompMap.at(dim)) - : dimSz = entry.second.global_length; + : dimSz = entry.second.globalLength; ncFromMAMap[dim] = dataGroup.addDim(entry.second.name, dimSz); // TODO Do I need to add data, even if it is just integers 0...n-1? } @@ -323,7 +323,7 @@ void ParaGridIO::dumpModelState( for (ModelArray::Dimension dt : entry.second.typeDimensions.at(type)) { auto dim = entry.second.definedDimensions.at(dt); start.push_back(dim.start); - count.push_back(dim.local_length); + count.push_back(dim.localLength); } // dims are looped in [dg], x, y, [z] order so start and count // order must be reveresed to match order netcdf expects @@ -381,7 +381,7 @@ void ParaGridIO::writeDiagnosticTime( for (auto entry : ModelArray::definedDimensions) { ModelArray::Dimension dim = entry.first; size_t dimSz = (dimCompMap.count(dim)) ? ModelArray::nComponents(dimCompMap.at(dim)) - : dimSz = entry.second.global_length; + : dimSz = entry.second.globalLength; ncFromMAMap[dim] = (isNew) ? dataGroup.addDim(entry.second.name, dimSz) : dataGroup.getDim(entry.second.name); } @@ -411,7 +411,7 @@ void ParaGridIO::writeDiagnosticTime( auto dim = ModelArray::definedDimensions.at(dt); ncDims.push_back(ncFromMAMap.at(dt)); start.push_back(dim.start); - count.push_back(dim.local_length); + count.push_back(dim.localLength); } // Deal with VERTEX in each case @@ -445,9 +445,9 @@ void ParaGridIO::writeDiagnosticTime( maskIndexes = { 0, 0 }; maskExtents = { ModelArray::definedDimensions .at(ModelArray::typeDimensions.at(ModelArray::Type::H)[0]) - .local_length, + .localLength, ModelArray::definedDimensions.at(ModelArray::typeDimensions.at(ModelArray::Type::H)[1]) - .local_length }; + .localLength }; } // Put the time axis variable diff --git a/core/src/include/ModelArray.hpp b/core/src/include/ModelArray.hpp index f92aa9a27..e2c2f10ee 100644 --- a/core/src/include/ModelArray.hpp +++ b/core/src/include/ModelArray.hpp @@ -57,8 +57,8 @@ class ModelArray { struct DimensionSpec { std::string name; std::string altName; - size_t global_length; - size_t local_length; + size_t globalLength; + size_t localLength; size_t start; }; typedef std::map> TypeDimensions; @@ -235,7 +235,7 @@ class ModelArray { //! Returns the size of the data array of this object. size_t trueSize() const { return m_data.rows(); } //! Returns the local size of a dimension - static size_t size(Dimension dim) { return definedDimensions.at(dim).local_length; } + static size_t size(Dimension dim) { return definedDimensions.at(dim).localLength; } //! Returns a read-only pointer to the underlying data buffer. const double* getData() const { return m_data.data(); } @@ -288,7 +288,7 @@ class ModelArray { if (size() != trueSize()) { if (hasDoF(type)) { m_data.resize( - m_sz.at(type), definedDimensions.at(componentMap.at(type)).local_length); + m_sz.at(type), definedDimensions.at(componentMap.at(type)).localLength); } else { m_data.resize(m_sz.at(type), Eigen::NoChange); } @@ -451,8 +451,8 @@ class ModelArray { static void setNComponents(Type type, size_t nComp) { if (hasDoF(type)) { - definedDimensions.at(componentMap.at(type)).local_length = nComp; - definedDimensions.at(componentMap.at(type)).global_length = nComp; + definedDimensions.at(componentMap.at(type)).localLength = nComp; + definedDimensions.at(componentMap.at(type)).globalLength = nComp; } } @@ -545,7 +545,7 @@ class ModelArray { //! specified type of ModelArray. inline static size_t nComponents(const Type type) { - return (hasDoF(type)) ? definedDimensions.at(componentMap.at(type)).local_length : 1; + return (hasDoF(type)) ? definedDimensions.at(componentMap.at(type)).localLength : 1; } //! Returns whether this type of ModelArray has additional discontinuous //! Galerkin components. diff --git a/core/src/modules/StructureModule/include/ParametricGrid.hpp b/core/src/modules/StructureModule/include/ParametricGrid.hpp index 56d34227f..534430482 100644 --- a/core/src/modules/StructureModule/include/ParametricGrid.hpp +++ b/core/src/modules/StructureModule/include/ParametricGrid.hpp @@ -62,7 +62,7 @@ class ParametricGrid : public IStructure { int nIceLayers() const override { - return ModelArray::definedDimensions.at(ModelArray::Dimension::Z).local_length; + return ModelArray::definedDimensions.at(ModelArray::Dimension::Z).localLength; }; class IParaGridIO { diff --git a/core/test/ConfigOutput_test.cpp b/core/test/ConfigOutput_test.cpp index 914dc0e8e..30d839686 100644 --- a/core/test/ConfigOutput_test.cpp +++ b/core/test/ConfigOutput_test.cpp @@ -112,7 +112,7 @@ TEST_CASE("Test periodic output") auto dim_x = ModelArray::Dimension::X; auto start_x = ModelArray::definedDimensions.at(dim_x).start; - auto local_nx = ModelArray::definedDimensions.at(dim_x).local_length; + auto local_nx = ModelArray::definedDimensions.at(dim_x).localLength; for (size_t k = 0; k < nz; ++k) { for (size_t j = 0; j < ny; ++j) { diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index ce1e789e1..e7168a40c 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -61,7 +61,7 @@ void initialize_test_data(HField& hfield, DGField& dgfield, HField& mask){ mask.resize(); auto dim_x = ModelArray::Dimension::X; auto start_x = ModelArray::definedDimensions.at(dim_x).start; - auto local_nx = ModelArray::definedDimensions.at(dim_x).local_length; + auto local_nx = ModelArray::definedDimensions.at(dim_x).localLength; for (size_t j = 0; j < ny; ++j) { for (size_t i = 0; i < local_nx; ++i) { hfield(i, j) = j * yFactor + (i+start_x) * xFactor; @@ -75,7 +75,7 @@ void initialize_test_data(HField& hfield, DGField& dgfield, HField& mask){ void initialize_test_coordinates(VertexField& coordinates){ auto dim_x_vertex = ModelArray::Dimension::XVERTEX; - auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).local_length; + auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).localLength; auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; for (size_t i = 0; i < local_nx_vertex; ++i) { for (size_t j = 0; j < ny + 1; ++j) { @@ -158,9 +158,9 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") // the following will be set correctly with MPI ON and OFF auto dim_x = ModelArray::Dimension::X; auto start_x = ModelArray::definedDimensions.at(dim_x).start; - auto local_nx = ModelArray::definedDimensions.at(dim_x).local_length; + auto local_nx = ModelArray::definedDimensions.at(dim_x).localLength; auto dim_x_vertex = ModelArray::Dimension::XVERTEX; - auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).local_length; + auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).localLength; auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; HField x; @@ -343,9 +343,9 @@ TEST_CASE("Write a diagnostic ParaGrid file") // the following will be set correctly with MPI ON and OFF auto dim_x = ModelArray::Dimension::X; auto start_x = ModelArray::definedDimensions.at(dim_x).start; - auto local_nx = ModelArray::definedDimensions.at(dim_x).local_length; + auto local_nx = ModelArray::definedDimensions.at(dim_x).localLength; auto dim_x_vertex = ModelArray::Dimension::XVERTEX; - auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).local_length; + auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).localLength; auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; HField fractional(ModelArray::Type::H); @@ -604,7 +604,7 @@ TEST_CASE("Check if a file with the old dimension names can be read") ModelState ms = gridIn.getModelState(inputFilename); #endif - auto local_nx = ModelArray::definedDimensions.at(ModelArray::Dimension::X).local_length; + auto local_nx = ModelArray::definedDimensions.at(ModelArray::Dimension::X).localLength; REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == local_nx); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[1] == ny); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[2] == NZLevels::get()); From 3f9fdcd070b65bf3c15a469dcab0d9a6d30778d6 Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 24 Jul 2024 16:58:36 +0100 Subject: [PATCH 14/25] chore: merge 2 ifdefs into one --- core/src/ModelArray.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/ModelArray.cpp b/core/src/ModelArray.cpp index 06d3b9b8b..182656e66 100644 --- a/core/src/ModelArray.cpp +++ b/core/src/ModelArray.cpp @@ -203,15 +203,13 @@ void ModelArray::setNComponents(std::map cMap) #ifdef USE_MPI void ModelArray::setDimension( Dimension dim, size_t global_length, size_t local_length, size_t start) -#else -void ModelArray::setDimension(Dimension dim, size_t global_length) -#endif { -#ifdef USE_MPI definedDimensions.at(dim).globalLength = global_length; definedDimensions.at(dim).localLength = local_length; definedDimensions.at(dim).start = start; #else +void ModelArray::setDimension(Dimension dim, size_t global_length) +{ // if MPI is not used then set the local_length to be the same as the global definedDimensions.at(dim).global_length = global_length; definedDimensions.at(dim).local_length = global_length; From 6a13d4dd27e52d795d2407dfd7c548fe8de9dec0 Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 24 Jul 2024 17:42:35 +0100 Subject: [PATCH 15/25] chore: rename vars to use camelCase --- core/src/ModelArray.cpp | 14 +++---- core/src/ParaGridIO.cpp | 14 +++---- core/src/include/ModelArray.hpp | 4 +- core/test/ConfigOutput_test.cpp | 18 ++++---- core/test/ParaGrid_test.cpp | 74 ++++++++++++++++----------------- 5 files changed, 62 insertions(+), 62 deletions(-) diff --git a/core/src/ModelArray.cpp b/core/src/ModelArray.cpp index 182656e66..160a8b601 100644 --- a/core/src/ModelArray.cpp +++ b/core/src/ModelArray.cpp @@ -202,17 +202,17 @@ void ModelArray::setNComponents(std::map cMap) #ifdef USE_MPI void ModelArray::setDimension( - Dimension dim, size_t global_length, size_t local_length, size_t start) + Dimension dim, size_t globalLength, size_t localLength, size_t start) { - definedDimensions.at(dim).globalLength = global_length; - definedDimensions.at(dim).localLength = local_length; + definedDimensions.at(dim).globalLength = globalLength; + definedDimensions.at(dim).localLength = localLength; definedDimensions.at(dim).start = start; #else -void ModelArray::setDimension(Dimension dim, size_t global_length) +void ModelArray::setDimension(Dimension dim, size_t globalLength) { - // if MPI is not used then set the local_length to be the same as the global - definedDimensions.at(dim).global_length = global_length; - definedDimensions.at(dim).local_length = global_length; + // if MPI is not used then set the localLength to be the same as the global + definedDimensions.at(dim).globalLength = globalLength; + definedDimensions.at(dim).localLength = globalLength; definedDimensions.at(dim).start = 0; #endif validateMaps(); diff --git a/core/src/ParaGridIO.cpp b/core/src/ParaGridIO.cpp index a08275e78..98149daeb 100644 --- a/core/src/ParaGridIO.cpp +++ b/core/src/ParaGridIO.cpp @@ -131,25 +131,25 @@ ModelState ParaGridIO::getModelState(const std::string& filePath) } else { #ifdef USE_MPI auto dimName = dim.getName(); - size_t local_length = 0; + size_t localLength = 0; size_t start = 0; if (dimType == ModelArray::Dimension::X) { - local_length = metadata.localExtentX; + localLength = metadata.localExtentX; start = metadata.localCornerX; } else if (dimType == ModelArray::Dimension::Y) { - local_length = metadata.localExtentY; + localLength = metadata.localExtentY; start = metadata.localCornerY; } else if (dimType == ModelArray::Dimension::XVERTEX) { - local_length = metadata.localExtentX + 1; + localLength = metadata.localExtentX + 1; start = metadata.localCornerX; } else if (dimType == ModelArray::Dimension::YVERTEX) { - local_length = metadata.localExtentY + 1; + localLength = metadata.localExtentY + 1; start = metadata.localCornerY; } else { - local_length = dim.getSize(); + localLength = dim.getSize(); start = 0; } - ModelArray::setDimension(dimType, dim.getSize(), local_length, start); + ModelArray::setDimension(dimType, dim.getSize(), localLength, start); #else ModelArray::setDimension(dimType, dim.getSize()); #endif diff --git a/core/src/include/ModelArray.hpp b/core/src/include/ModelArray.hpp index e2c2f10ee..fa6df62b5 100644 --- a/core/src/include/ModelArray.hpp +++ b/core/src/include/ModelArray.hpp @@ -276,9 +276,9 @@ class ModelArray { * @param length The new length of the dimension. */ #ifdef USE_MPI - static void setDimension(Dimension dim, size_t global_length, size_t local_length, size_t size); + static void setDimension(Dimension dim, size_t globalLength, size_t localLength, size_t size); #else - static void setDimension(Dimension dim, size_t global_length); + static void setDimension(Dimension dim, size_t globalLength); #endif //! Conditionally updates the size of the object data buffer to match the diff --git a/core/test/ConfigOutput_test.cpp b/core/test/ConfigOutput_test.cpp index 30d839686..36a65f820 100644 --- a/core/test/ConfigOutput_test.cpp +++ b/core/test/ConfigOutput_test.cpp @@ -110,22 +110,22 @@ TEST_CASE("Test periodic output") IDiagnosticOutput& ido = Module::getImplementation(); tryConfigure(ido); - auto dim_x = ModelArray::Dimension::X; - auto start_x = ModelArray::definedDimensions.at(dim_x).start; - auto local_nx = ModelArray::definedDimensions.at(dim_x).localLength; + auto dimX = ModelArray::Dimension::X; + auto startX = ModelArray::definedDimensions.at(dimX).start; + auto localNX = ModelArray::definedDimensions.at(dimX).localLength; for (size_t k = 0; k < nz; ++k) { for (size_t j = 0; j < ny; ++j) { - for (size_t i = 0; i < local_nx; ++i) { - tice(i, j, k) = 0.1 * k + 0.4 + 0.01 * (j * nx + (i + start_x)); + for (size_t i = 0; i < localNX; ++i) { + tice(i, j, k) = 0.1 * k + 0.4 + 0.01 * (j * nx + (i + startX)); } } } for (size_t j = 0; j < ny; ++j) { - for (size_t i = 0; i < local_nx; ++i) { - hice(i, j) = 0 + 0.01 * (j * nx + (i + start_x)); - cice(i, j) = 0.1 + 0.01 * (j * nx + (i + start_x)); - hsnow(i, j) = 0.2 + 0.01 * (j * nx + (i + start_x)); + for (size_t i = 0; i < localNX; ++i) { + hice(i, j) = 0 + 0.01 * (j * nx + (i + startX)); + cice(i, j) = 0.1 + 0.01 * (j * nx + (i + startX)); + hsnow(i, j) = 0.2 + 0.01 * (j * nx + (i + startX)); } } std::vector diagFiles; diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index e7168a40c..964fe2d96 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -59,13 +59,13 @@ void initialize_test_data(HField& hfield, DGField& dgfield, HField& mask){ hfield.resize(); dgfield.resize(); mask.resize(); - auto dim_x = ModelArray::Dimension::X; - auto start_x = ModelArray::definedDimensions.at(dim_x).start; - auto local_nx = ModelArray::definedDimensions.at(dim_x).localLength; + auto dimX = ModelArray::Dimension::X; + auto startX = ModelArray::definedDimensions.at(dimX).start; + auto localNX = ModelArray::definedDimensions.at(dimX).localLength; for (size_t j = 0; j < ny; ++j) { - for (size_t i = 0; i < local_nx; ++i) { - hfield(i, j) = j * yFactor + (i+start_x) * xFactor; - mask(i, j) = ((i + start_x) - nx / 2) * ((i + start_x) - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * ny) ? 0 : 1; + for (size_t i = 0; i < localNX; ++i) { + hfield(i, j) = j * yFactor + (i+startX) * xFactor; + mask(i, j) = ((i + startX) - nx / 2) * ((i + startX) - nx / 2) + (j - ny / 2) * (j - ny / 2) > (nx * ny) ? 0 : 1; for (size_t d = 0; d < DG; ++d) { dgfield.components({ i, j })[d] = hfield(i, j) + d; } @@ -74,12 +74,12 @@ void initialize_test_data(HField& hfield, DGField& dgfield, HField& mask){ }; void initialize_test_coordinates(VertexField& coordinates){ - auto dim_x_vertex = ModelArray::Dimension::XVERTEX; - auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).localLength; - auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; - for (size_t i = 0; i < local_nx_vertex; ++i) { + auto dimXVertex = ModelArray::Dimension::XVERTEX; + auto localNXVertex = ModelArray::definedDimensions.at(dimXVertex).localLength; + auto startXVertex = ModelArray::definedDimensions.at(dimXVertex).start; + for (size_t i = 0; i < localNXVertex; ++i) { for (size_t j = 0; j < ny + 1; ++j) { - double x = (i + start_x_vertex) - 0.5 - float(nx) / 2; + double x = (i + startXVertex) - 0.5 - float(nx) / 2; double y = j - 0.5 - float(ny) / 2; coordinates.components({ i, j })[0] = x * scale; coordinates.components({ i, j })[1] = y * scale; @@ -156,12 +156,12 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") // MPI domain is only split in x-direction for this test // the following will be set correctly with MPI ON and OFF - auto dim_x = ModelArray::Dimension::X; - auto start_x = ModelArray::definedDimensions.at(dim_x).start; - auto local_nx = ModelArray::definedDimensions.at(dim_x).localLength; - auto dim_x_vertex = ModelArray::Dimension::XVERTEX; - auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).localLength; - auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; + auto dimX = ModelArray::Dimension::X; + auto startX = ModelArray::definedDimensions.at(dimX).start; + auto localNX = ModelArray::definedDimensions.at(dimX).localLength; + auto dimXVertex = ModelArray::Dimension::XVERTEX; + auto localNXVertex = ModelArray::definedDimensions.at(dimXVertex).localLength; + auto startXVertex = ModelArray::definedDimensions.at(dimXVertex).start; HField x; HField y; @@ -170,8 +170,8 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") // Element coordinates for (size_t j = 0; j < ny; ++j) { double yy = scale * (j - float(ny) / 2); - for (size_t i = 0; i < local_nx; ++i) { - double xx = scale * ((i + start_x) - float(nx) / 2); + for (size_t i = 0; i < localNX; ++i) { + double xx = scale * ((i + startX) - float(nx) / 2); x(i, j) = xx; y(i, j) = yy; } @@ -211,9 +211,9 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") metadata.setMpiMetadata(test_comm); metadata.globalExtentX = nx; metadata.globalExtentY = ny; - metadata.localCornerX = start_x; + metadata.localCornerX = startX; metadata.localCornerY = 0; - metadata.localExtentX = local_nx; + metadata.localExtentX = localNX; metadata.localExtentY = ny; #endif grid.dumpModelState(state, metadata, filename, true); @@ -250,7 +250,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") ModelState ms = gridIn.getModelState(filename); #endif - REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == local_nx); + REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == localNX); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[1] == ny); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[2] == NZLevels::get()); @@ -260,13 +260,13 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") REQUIRE(ModelArray::nDimensions(ModelArray::Type::Z) == 3); REQUIRE(ticeRef.getType() == ModelArray::Type::Z); REQUIRE(ticeRef.nDimensions() == 3); - REQUIRE(ticeRef.dimensions()[0] == local_nx); + REQUIRE(ticeRef.dimensions()[0] == localNX); REQUIRE(ticeRef.dimensions()[1] == ny); REQUIRE(ticeRef.dimensions()[2] == NZLevels::get()); ModelArray& hiceRef = ms.data.at(hiceName); REQUIRE(hiceRef.nDimensions() == 2); - REQUIRE(hiceRef.dimensions()[0] == local_nx); + REQUIRE(hiceRef.dimensions()[0] == localNX); REQUIRE(hiceRef.dimensions()[1] == ny); REQUIRE(ModelArray::nComponents(ModelArray::Type::DG) == DG); REQUIRE(hiceRef.nComponents() == DG); @@ -277,7 +277,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") ModelArray& coordRef = ms.data.at(coordsName); REQUIRE(coordRef.nDimensions() == 2); REQUIRE(coordRef.nComponents() == 2); - REQUIRE(coordRef.dimensions()[0] == local_nx_vertex); + REQUIRE(coordRef.dimensions()[0] == localNXVertex); REQUIRE(coordRef.dimensions()[1] == ny + 1); REQUIRE(coordRef.components({ 3, 8 })[0] - coordRef.components({ 2, 8 })[0] == scale); REQUIRE(coordRef.components({ 3, 8 })[1] - coordRef.components({ 3, 7 })[1] == scale); @@ -341,12 +341,12 @@ TEST_CASE("Write a diagnostic ParaGrid file") // MPI domain is only split in x-direction for this test // the following will be set correctly with MPI ON and OFF - auto dim_x = ModelArray::Dimension::X; - auto start_x = ModelArray::definedDimensions.at(dim_x).start; - auto local_nx = ModelArray::definedDimensions.at(dim_x).localLength; - auto dim_x_vertex = ModelArray::Dimension::XVERTEX; - auto local_nx_vertex = ModelArray::definedDimensions.at(dim_x_vertex).localLength; - auto start_x_vertex = ModelArray::definedDimensions.at(dim_x_vertex).start; + auto dimX = ModelArray::Dimension::X; + auto startX = ModelArray::definedDimensions.at(dimX).start; + auto localNX = ModelArray::definedDimensions.at(dimX).localLength; + auto dimXVertex = ModelArray::Dimension::XVERTEX; + auto localNXVertex = ModelArray::definedDimensions.at(dimXVertex).localLength; + auto startXVertex = ModelArray::definedDimensions.at(dimXVertex).start; HField fractional(ModelArray::Type::H); DGField fractionalDG(ModelArray::Type::DG); @@ -368,8 +368,8 @@ TEST_CASE("Write a diagnostic ParaGrid file") // Element coordinates for (size_t j = 0; j < ny; ++j) { double yy = scale * (j - float(ny) / 2); - for (size_t i = 0; i < local_nx; ++i) { - double xx = scale * ((i + start_x) - float(nx) / 2); + for (size_t i = 0; i < localNX; ++i) { + double xx = scale * ((i + startX) - float(nx) / 2); x(i, j) = xx; y(i, j) = yy; } @@ -407,9 +407,9 @@ TEST_CASE("Write a diagnostic ParaGrid file") metadata.setMpiMetadata(test_comm); metadata.globalExtentX = nx; metadata.globalExtentY = ny; - metadata.localCornerX = start_x; + metadata.localCornerX = startX; metadata.localCornerY = 0; - metadata.localExtentX = local_nx; + metadata.localExtentX = localNX; metadata.localExtentY = ny; #endif @@ -604,8 +604,8 @@ TEST_CASE("Check if a file with the old dimension names can be read") ModelState ms = gridIn.getModelState(inputFilename); #endif - auto local_nx = ModelArray::definedDimensions.at(ModelArray::Dimension::X).localLength; - REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == local_nx); + auto localNX = ModelArray::definedDimensions.at(ModelArray::Dimension::X).localLength; + REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[0] == localNX); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[1] == ny); REQUIRE(ModelArray::dimensions(ModelArray::Type::Z)[2] == NZLevels::get()); } From 5d6ee09b0c6975ada9c644b0343839b05c373c7e Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 24 Jul 2024 17:46:20 +0100 Subject: [PATCH 16/25] chore: rename func to use camelCase --- core/test/ParaGrid_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index 964fe2d96..2aeadd1cc 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -55,7 +55,7 @@ namespace Nextsim { size_t c = 0; -void initialize_test_data(HField& hfield, DGField& dgfield, HField& mask){ +void initializeTestData(HField& hfield, DGField& dgfield, HField& mask){ hfield.resize(); dgfield.resize(); mask.resize(); @@ -133,7 +133,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") HField fractional(ModelArray::Type::H); DGField fractionalDG(ModelArray::Type::DG); HField mask(ModelArray::Type::H); - initialize_test_data(fractional, fractionalDG, mask); + initializeTestData(fractional, fractionalDG, mask); DGField hice = fractionalDG + 10; DGField cice = fractionalDG + 20; @@ -351,7 +351,7 @@ TEST_CASE("Write a diagnostic ParaGrid file") HField fractional(ModelArray::Type::H); DGField fractionalDG(ModelArray::Type::DG); HField mask(ModelArray::Type::H); - initialize_test_data(fractional, fractionalDG, mask); + initializeTestData(fractional, fractionalDG, mask); REQUIRE(fractional.nDimensions() == 2); From 6136e32d67e5fed2ff8f2b3b650fb2adc0a3b11a Mon Sep 17 00:00:00 2001 From: melt Date: Thu, 15 Aug 2024 10:27:11 +0100 Subject: [PATCH 17/25] chore: rename func + vars to use camelCase --- core/test/ParaGrid_test.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/core/test/ParaGrid_test.cpp b/core/test/ParaGrid_test.cpp index 2aeadd1cc..867e5bae0 100644 --- a/core/test/ParaGrid_test.cpp +++ b/core/test/ParaGrid_test.cpp @@ -32,12 +32,12 @@ #include #include -const std::string test_files_dir = TEST_FILES_DIR; -const std::string filename = test_files_dir + "/paraGrid_test.nc"; +const std::string testFilesDir = TEST_FILES_DIR; +const std::string filename = testFilesDir + "/paraGrid_test.nc"; const std::string diagFile = "paraGrid_diag.nc"; -const std::string date_string = "2000-01-01T00:00:00Z"; +const std::string dateString = "2000-01-01T00:00:00Z"; #ifdef USE_MPI -const std::string partition_filename = test_files_dir + "/partition_metadata_2.nc"; +const std::string partitionFilename = testFilesDir + "/partition_metadata_2.nc"; #endif static const int DG = 3; @@ -73,7 +73,7 @@ void initializeTestData(HField& hfield, DGField& dgfield, HField& mask){ } }; -void initialize_test_coordinates(VertexField& coordinates){ +void initializeTestCoordinates(VertexField& coordinates){ auto dimXVertex = ModelArray::Dimension::XVERTEX; auto localNXVertex = ModelArray::definedDimensions.at(dimXVertex).localLength; auto startXVertex = ModelArray::definedDimensions.at(dimXVertex).start; @@ -149,7 +149,7 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") } VertexField coordinates(ModelArray::Type::VERTEX); - initialize_test_coordinates(coordinates); + initializeTestCoordinates(coordinates); REQUIRE(coordinates.components({ 3, 8 })[0] - coordinates.components({ 2, 8 })[0] == scale); REQUIRE(coordinates.components({ 3, 8 })[1] - coordinates.components({ 3, 7 })[1] == scale); @@ -243,8 +243,8 @@ TEST_CASE("Write and read a ModelState-based ParaGrid restart file") gridIn.setIO(readIO); #ifdef USE_MPI - ModelMetadata metadataIn(partition_filename, test_comm); - metadataIn.setTime(TimePoint(date_string)); + ModelMetadata metadataIn(partitionFilename, test_comm); + metadataIn.setTime(TimePoint(dateString)); ModelState ms = gridIn.getModelState(filename, metadataIn); #else ModelState ms = gridIn.getModelState(filename); @@ -359,7 +359,7 @@ TEST_CASE("Write a diagnostic ParaGrid file") DGField cice = fractionalDG + 20; VertexField coordinates(ModelArray::Type::VERTEX); - initialize_test_coordinates(coordinates); + initializeTestCoordinates(coordinates); HField x; HField y; @@ -533,8 +533,8 @@ TEST_CASE("Check an exception is thrown for an invalid file name") // MD5 hash of the current output of $ date std::string longRandomFilename("a44f5cc1f7934a8ae8dd03a95308745d.nc"); #ifdef USE_MPI - ModelMetadata metadataIn(partition_filename, test_comm); - metadataIn.setTime(TimePoint(date_string)); + ModelMetadata metadataIn(partitionFilename, test_comm); + metadataIn.setTime(TimePoint(dateString)); REQUIRE_THROWS(state = gridIn.getModelState(longRandomFilename, metadataIn)); #else REQUIRE_THROWS(state = gridIn.getModelState(longRandomFilename)); @@ -598,7 +598,7 @@ TEST_CASE("Check if a file with the old dimension names can be read") metadata.localCornerY = 0; metadata.localExtentX = 1; metadata.localExtentY = ny; - metadata.setTime(TimePoint(date_string)); + metadata.setTime(TimePoint(dateString)); ModelState ms = gridIn.getModelState(inputFilename, metadata); #else ModelState ms = gridIn.getModelState(inputFilename); From 43eac7f1319dcf84d9816a973032d8da8da8b78a Mon Sep 17 00:00:00 2001 From: melt Date: Tue, 20 Aug 2024 13:09:11 +0200 Subject: [PATCH 18/25] bugfix: add missing include to ParaGridIO --- core/src/include/ParaGridIO.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/include/ParaGridIO.hpp b/core/src/include/ParaGridIO.hpp index 55fdf7d1b..25b102927 100644 --- a/core/src/include/ParaGridIO.hpp +++ b/core/src/include/ParaGridIO.hpp @@ -10,6 +10,9 @@ #include "StructureModule/include/ParametricGrid.hpp" +#ifdef USE_MPI +#include "ParallelNetcdfFile.hpp" +#endif #include #include #include From ce28dbdd3f3ea8c1bc542bbb217d85c65d48da2f Mon Sep 17 00:00:00 2001 From: melt Date: Tue, 20 Aug 2024 13:21:14 +0200 Subject: [PATCH 19/25] chore: only build serial dynamics tests when mpi is off --- dynamics/test/CMakeLists.txt | 56 +++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/dynamics/test/CMakeLists.txt b/dynamics/test/CMakeLists.txt index d1211227e..5ee75b1f5 100644 --- a/dynamics/test/CMakeLists.txt +++ b/dynamics/test/CMakeLists.txt @@ -6,47 +6,49 @@ set(CoreDir "../../core/src/") include_directories(${INCLUDE_DIR}) -add_executable(testDGModelArray +if(NOT ENABLE_MPI) + add_executable(testDGModelArray "DGModelArray_test.cpp" "${CoreDir}/ModelArray.cpp" "${CoreDir}/${ModelArrayStructure}/ModelArrayDetails.cpp" ) -target_include_directories(testDGModelArray PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") -target_link_libraries(testDGModelArray LINK_PUBLIC doctest::doctest Eigen3::Eigen) + target_include_directories(testDGModelArray PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") + target_link_libraries(testDGModelArray LINK_PUBLIC doctest::doctest Eigen3::Eigen) -add_executable(testCGModelArray + add_executable(testCGModelArray "CGModelArray_test.cpp" "${CoreDir}/ModelArray.cpp" "${CoreDir}/${ModelArrayStructure}/ModelArrayDetails.cpp" ) -target_include_directories(testCGModelArray PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") -target_link_libraries(testCGModelArray LINK_PUBLIC doctest::doctest Eigen3::Eigen) + target_include_directories(testCGModelArray PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") + target_link_libraries(testCGModelArray LINK_PUBLIC doctest::doctest Eigen3::Eigen) -file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/25km_NH.smesh" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/25km_NH.smesh" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") -add_executable(testParametricMesh + add_executable(testParametricMesh "ParametricMesh_test.cpp" "FakeSmeshData.cpp" "${SRC_DIR}/ParametricMesh.cpp" "${CoreDir}/ModelArray.cpp" "${CoreDir}/${ModelArrayStructure}/ModelArrayDetails.cpp" ) -target_include_directories(testParametricMesh PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") -target_link_libraries(testParametricMesh LINK_PUBLIC doctest::doctest Eigen3::Eigen) - - -add_executable(testAdvection Advection_test.cpp - "${SRC_DIR}/Interpolations.cpp" - "${SRC_DIR}/DGTransport.cpp" - "${SRC_DIR}/ParametricMap.cpp" - ) -target_include_directories(testAdvection PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") -target_link_libraries(testAdvection LINK_PUBLIC doctest::doctest Eigen3::Eigen) - -add_executable(testAdvectionPeriodicBC AdvectionPeriodicBC_test.cpp - "${SRC_DIR}/DGTransport.cpp" - "${SRC_DIR}/ParametricMap.cpp" - "${SRC_DIR}/Interpolations.cpp" - ) -target_include_directories(testAdvectionPeriodicBC PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") -target_link_libraries(testAdvectionPeriodicBC LINK_PUBLIC doctest::doctest Eigen3::Eigen) + target_include_directories(testParametricMesh PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") + target_link_libraries(testParametricMesh LINK_PUBLIC doctest::doctest Eigen3::Eigen) + + + add_executable(testAdvection Advection_test.cpp + "${SRC_DIR}/Interpolations.cpp" + "${SRC_DIR}/DGTransport.cpp" + "${SRC_DIR}/ParametricMap.cpp" + ) + target_include_directories(testAdvection PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") + target_link_libraries(testAdvection LINK_PUBLIC doctest::doctest Eigen3::Eigen) + + add_executable(testAdvectionPeriodicBC AdvectionPeriodicBC_test.cpp + "${SRC_DIR}/DGTransport.cpp" + "${SRC_DIR}/ParametricMap.cpp" + "${SRC_DIR}/Interpolations.cpp" + ) + target_include_directories(testAdvectionPeriodicBC PRIVATE "${CoreDir}" "${SRC_DIR}" "${CoreDir}/${ModelArrayStructure}") + target_link_libraries(testAdvectionPeriodicBC LINK_PUBLIC doctest::doctest Eigen3::Eigen) +endif() From 1a45dfc16c245e434b957596c36c6a4cb8a9ed26 Mon Sep 17 00:00:00 2001 From: melt Date: Tue, 20 Aug 2024 14:34:15 +0200 Subject: [PATCH 20/25] feat: add setter to DimensionSpec struct --- core/src/ModelArray.cpp | 12 +++--------- core/src/include/ModelArray.hpp | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/core/src/ModelArray.cpp b/core/src/ModelArray.cpp index 160a8b601..21e092db7 100644 --- a/core/src/ModelArray.cpp +++ b/core/src/ModelArray.cpp @@ -201,19 +201,13 @@ void ModelArray::setNComponents(std::map cMap) } #ifdef USE_MPI -void ModelArray::setDimension( - Dimension dim, size_t globalLength, size_t localLength, size_t start) +void ModelArray::setDimension(Dimension dim, size_t globalLength, size_t localLength, size_t start) { - definedDimensions.at(dim).globalLength = globalLength; - definedDimensions.at(dim).localLength = localLength; - definedDimensions.at(dim).start = start; + definedDimensions.at(dim).setLengths(globalLength, localLength, start); #else void ModelArray::setDimension(Dimension dim, size_t globalLength) { - // if MPI is not used then set the localLength to be the same as the global - definedDimensions.at(dim).globalLength = globalLength; - definedDimensions.at(dim).localLength = globalLength; - definedDimensions.at(dim).start = 0; + definedDimensions.at(dim).setLengths(globalLength); #endif validateMaps(); } diff --git a/core/src/include/ModelArray.hpp b/core/src/include/ModelArray.hpp index fa6df62b5..06cc860e8 100644 --- a/core/src/include/ModelArray.hpp +++ b/core/src/include/ModelArray.hpp @@ -60,7 +60,24 @@ class ModelArray { size_t globalLength; size_t localLength; size_t start; +#ifdef USE_MPI + void setLengths(size_t globalLength, size_t localLength, size_t start) + { + this->globalLength = globalLength; + this->localLength = localLength; + this->start = start; + } +#else + void setLengths(size_t length) + { + // if MPI is not used then localLength and globalLength are set to the same value + this->globalLength = length; + this->localLength = length; + this->start = 0; + } +#endif }; + typedef std::map> TypeDimensions; //! The dimensions that make up each defined type. Defined in ModelArrayDetails.cpp From d87c6891521fad4a3f4e5bf67baf56c78241ef0b Mon Sep 17 00:00:00 2001 From: melt Date: Tue, 20 Aug 2024 15:31:27 +0200 Subject: [PATCH 21/25] bugfix: set all dimensions in XiosWrite_test --- core/test/XiosWrite_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/test/XiosWrite_test.cpp b/core/test/XiosWrite_test.cpp index 4f829be14..1e866c6ab 100644 --- a/core/test/XiosWrite_test.cpp +++ b/core/test/XiosWrite_test.cpp @@ -112,9 +112,9 @@ MPI_TEST_CASE("TestXiosWrite", 2) // --- Tests for writing to file Module::setImplementation("Nextsim::ParametricGrid"); - ModelArray::setDimension(ModelArray::Dimension::X, n1); - ModelArray::setDimension(ModelArray::Dimension::Y, n2); - ModelArray::setDimension(ModelArray::Dimension::Z, n3); + ModelArray::setDimension(ModelArray::Dimension::X, n1, n1, 0); + ModelArray::setDimension(ModelArray::Dimension::Y, n2, n2, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, n3, n3, 0); // Create some fake data to test writing methods HField field_2D(ModelArray::Type::H); field_2D.resize(); From 6027a8dcb88114c492917509ea0b687214fb29f4 Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 21 Aug 2024 09:58:23 +0200 Subject: [PATCH 22/25] chore: rename nc dump files to cdl --- core/test/CMakeLists.txt | 6 +++--- ...artition_metadata_2.nc.dump => partition_metadata_2.cdl} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename core/test/{partition_metadata_2.nc.dump => partition_metadata_2.cdl} (100%) diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 127014586..8227dc38e 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -19,14 +19,14 @@ include_directories(${COMMON_INCLUDE_DIRS}) set(MODEL_INCLUDE_DIR "../../core/src/discontinuousgalerkin") if(ENABLE_MPI) -# Generate partition file needed for MPI test from its dump +# Generate partition files needed for MPI test from respective cdl files add_custom_command(OUTPUT partition_metadata_3.nc COMMAND ncgen -b -o partition_metadata_3.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3.cdl DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3.cdl ) add_custom_command(OUTPUT partition_metadata_2.nc - COMMAND ncgen -b -o partition_metadata_2.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.nc.dump - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.nc.dump + COMMAND ncgen -b -o partition_metadata_2.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.cdl + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.cdl ) add_custom_target(generate_partition_files ALL DEPENDS partition_metadata_3.nc partition_metadata_2.nc) diff --git a/core/test/partition_metadata_2.nc.dump b/core/test/partition_metadata_2.cdl similarity index 100% rename from core/test/partition_metadata_2.nc.dump rename to core/test/partition_metadata_2.cdl From b9e2c091fa9b54b8319b0455bfb438f9d894eb42 Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 21 Aug 2024 10:04:39 +0200 Subject: [PATCH 23/25] feat: modify finitevolume to match dg initialization --- core/src/finitevolume/ModelArrayDetails.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/finitevolume/ModelArrayDetails.cpp b/core/src/finitevolume/ModelArrayDetails.cpp index e68110472..1ea91f6eb 100644 --- a/core/src/finitevolume/ModelArrayDetails.cpp +++ b/core/src/finitevolume/ModelArrayDetails.cpp @@ -15,11 +15,11 @@ namespace Nextsim { std::map ModelArray::definedDimensions = { - { ModelArray::Dimension::X, { "xdim", "x", 0 } }, - { ModelArray::Dimension::Y, { "ydim", "y", 0 } }, - { ModelArray::Dimension::Z, { "zdim", "z", 1 } }, - { ModelArray::Dimension::XVERTEX, { "xvertex", "xvertex", 1 } }, // defined as x + 1 - { ModelArray::Dimension::YVERTEX, { "yvertex", "yvertex", 1 } }, // defined as y + 1 + { ModelArray::Dimension::X, { "xdim", "x", 0, 0, 0 } }, + { ModelArray::Dimension::Y, { "ydim", "y", 0, 0, 0 } }, + { ModelArray::Dimension::Z, { "zdim", "z", 1, 1, 0 } }, + { ModelArray::Dimension::XVERTEX, { "xvertex", "xvertex", 1, 1, 0 } }, // defined as x + 1 + { ModelArray::Dimension::YVERTEX, { "yvertex", "yvertex", 1, 1, 0 } }, // defined as y + 1 }; ModelArray::TypeDimensions ModelArray::typeDimensions = { From 1ec385ad5959e97640031fb0e756774078d458cf Mon Sep 17 00:00:00 2001 From: melt Date: Wed, 21 Aug 2024 10:14:57 +0200 Subject: [PATCH 24/25] feat: modify testmodelarraydetails to match dg initialization --- core/test/testmodelarraydetails/ModelArrayDetails.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/test/testmodelarraydetails/ModelArrayDetails.cpp b/core/test/testmodelarraydetails/ModelArrayDetails.cpp index 2f59e3dbf..4fbb02ba1 100644 --- a/core/test/testmodelarraydetails/ModelArrayDetails.cpp +++ b/core/test/testmodelarraydetails/ModelArrayDetails.cpp @@ -9,10 +9,10 @@ namespace Nextsim { std::map ModelArray::definedDimensions = { - { ModelArray::Dimension::X, { "x", "x", 1 } }, - { ModelArray::Dimension::Y, { "y", "y", 1 } }, - { ModelArray::Dimension::Z, { "z", "z", 1 } }, - { ModelArray::Dimension::U, { "u", "u", 1 } }, + { ModelArray::Dimension::X, { "x", "x", 1, 1, 0 } }, + { ModelArray::Dimension::Y, { "y", "y", 1, 1, 0 } }, + { ModelArray::Dimension::Z, { "z", "z", 1, 1, 0 } }, + { ModelArray::Dimension::U, { "u", "u", 1, 1, 0 } }, }; ModelArray::TypeDimensions ModelArray::typeDimensions = { From 4088afad3157632ab9c5abf4edae1b9441c4a875 Mon Sep 17 00:00:00 2001 From: Joe Wallwork Date: Wed, 21 Aug 2024 09:57:32 +0100 Subject: [PATCH 25/25] Updated setDimension API in XIOS read test --- core/test/XiosRead_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/test/XiosRead_test.cpp b/core/test/XiosRead_test.cpp index 038687b83..1641902f6 100644 --- a/core/test/XiosRead_test.cpp +++ b/core/test/XiosRead_test.cpp @@ -116,9 +116,9 @@ MPI_TEST_CASE("TestXiosRead", 2) // --- Tests for reading to file Module::setImplementation("Nextsim::ParametricGrid"); - ModelArray::setDimension(ModelArray::Dimension::X, n1); - ModelArray::setDimension(ModelArray::Dimension::Y, n2); - ModelArray::setDimension(ModelArray::Dimension::Z, n3); + ModelArray::setDimension(ModelArray::Dimension::X, n1, n1, 0); + ModelArray::setDimension(ModelArray::Dimension::Y, n2, n2, 0); + ModelArray::setDimension(ModelArray::Dimension::Z, n3, n3, 0); // Create some fake data to test writing methods HField field_2D(ModelArray::Type::H); field_2D.resize();