MPI parallelization of thermodynamics (#331)

## Background 
This PR is part of #120, where the basis of strategy for MPI
parallelisation is described.

## Change description
As all thermodynamics operations are local to a grid cell all required
MPI communication, is handled by NetCDF4 library. Therefore the only
required steps are:

- [x] Initialize and finalize MPI stack
- [x] Read decomposition metadata on each rank
- [x] Read and write the necessary part of the grid on each rank
(depends on #330)
- [x] Tests for parallel I/O 

**NOTE** PR #432 should be merged into this branch before it is merged
into develop

To run `run_simple_example.sh` you will need to generate the following
netcdf file

```
$ ncdump partition.nc 
netcdf partition {
dimensions:
        P = 1 ;
        L = 1 ;
        globalX = 30 ;
        globalY = 30 ;

group: bounding_boxes {
  variables:
        int global_x(P) ;
        int global_y(P) ;
        int local_extent_x(P) ;
        int local_extent_y(P) ;
  data:

   global_x = 0 ;

   global_y = 0 ;

   local_extent_x = 30 ;

   local_extent_y = 30 ;
  } // group bounding_boxes
}

```

CMakeLists.txt

@@ -50,6 +50,12 @@ set_property(TARGET doctest::doctest PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${D
 
 find_package(Eigen3 3.4 REQUIRED)
 
+option(ENABLE_MPI "Enable distributed-memory parallelization with MPI" OFF)
+if(ENABLE_MPI)
+  find_package(MPI REQUIRED COMPONENTS C CXX)
+endif()
+
+
 # To add netCDF to a target:
 # target_include_directories(target PUBLIC ${netCDF_INCLUDE_DIR})
 # target_link_directories(target PUBLIC ${netCDF_LIB_DIR})
@@ -89,6 +95,10 @@ foreach(compo ${CodeComponents})
 endforeach()
 
 add_library(nextsimlib SHARED ${NextsimSources})
+target_compile_definitions(nextsimlib
+  PRIVATE
+    $<$<BOOL:${ENABLE_MPI}>:USE_MPI>
+)
 target_include_directories(nextsimlib
     PRIVATE
     "${CMAKE_CURRENT_SOURCE_DIR}"
@@ -97,9 +107,16 @@ target_include_directories(nextsimlib
     "${netCDF_INCLUDE_DIR}"
     )
 target_link_directories(nextsimlib PUBLIC "${netCDF_LIB_DIR}")
-target_link_libraries(nextsimlib PUBLIC Boost::program_options Boost::log "${NSDG_NetCDF_Library}" Eigen3::Eigen)
+target_link_libraries(nextsimlib PUBLIC
+    Boost::program_options Boost::log "${NSDG_NetCDF_Library}" Eigen3::Eigen
+    $<$<BOOL:${ENABLE_MPI}>:MPI::MPI_C> $<$<BOOL:${ENABLE_MPI}>:MPI::MPI_CXX>
+    )
 
 add_executable(nextsim ./core/src/main.cpp)
+target_compile_definitions(nextsim
+  PRIVATE
+    $<$<BOOL:${ENABLE_MPI}>:USE_MPI>
+)
 target_include_directories(nextsim PRIVATE
     "${CMAKE_CURRENT_SOURCE_DIR}"
     "${NextsimIncludeDirs}"

core/src/CMakeLists.txt

@@ -26,6 +26,10 @@ set(BaseSources
     "${ModelArrayStructure}/ModelArrayDetails.cpp"
     )
 
+set(ParallelNetCDFSources
+    "${CMAKE_CURRENT_SOURCE_DIR}/ParallelNetcdfFile.cpp"
+    )
+
 list(TRANSFORM BaseSources PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
 
 set(ModuleDir "${CMAKE_CURRENT_SOURCE_DIR}/modules")
@@ -66,8 +70,25 @@ set(NextsimSources
     "${NextsimSources}"
     "${BaseSources}"
     "${ModuleSources}"
+    "${ParallelNetCDFSources}"
     PARENT_SCOPE)
 
+if(ENABLE_MPI)
+    set(NextsimSources
+        "${NextsimSources}"
+        "${BaseSources}"
+        "${ModuleSources}"
+        "${ParallelNetCDFSources}"
+        PARENT_SCOPE)
+else()
+    set(NextsimSources
+        "${NextsimSources}"
+        "${BaseSources}"
+        "${ModuleSources}"
+        PARENT_SCOPE)
+endif()
+
+
 set(NextsimIncludeDirs
     "${NextsimIncludeDirs}"
     "${ModuleDir}"

core/src/Model.cpp

@@ -2,6 +2,7 @@
  * @file Model.cpp
  * @date 12 Aug 2021
  * @author Tim Spain <timothy.spain@nersc.no>
+ * @author Kacper Kornet <kk562@cam.ac.uk>
  */
 
 #include "include/Model.hpp"
@@ -26,15 +27,26 @@ const std::string Model::restartOptionName = "model.init_file";
 template <>
 const std::map<int, std::string> Configured<Model>::keyMap = {
     { Model::RESTARTFILE_KEY, Model::restartOptionName },
+#ifdef USE_MPI
+    { Model::PARTITIONFILE_KEY, "model.partition_file" },
+#endif
     { Model::STARTTIME_KEY, "model.start" },
     { Model::STOPTIME_KEY, "model.stop" },
     { Model::RUNLENGTH_KEY, "model.run_length" },
     { Model::TIMESTEP_KEY, "model.time_step" },
     { Model::MISSINGVALUE_KEY, "model.missing_value" },
 };
 
+#ifdef USE_MPI
+Model::Model(MPI_Comm comm)
+#else
 Model::Model()
+#endif
 {
+#ifdef USE_MPI
+    m_etadata.setMpiMetadata(comm);
+#endif
+
     iterator.setIterant(&modelStep);
 
     finalFileName = "restart.nc";
@@ -78,7 +90,17 @@ void Model::configure()
     modelStep.init();
     modelStep.setInitFile(initialFileName);
 
+#ifdef USE_MPI
+    std::string partitionFile
+        = Configured::getConfiguration(keyMap.at(PARTITIONFILE_KEY), std::string("partition.nc"));
+#endif
+
+#ifdef USE_MPI
+    ModelState initialState(
+        StructureFactory::stateFromFile(initialFileName, partitionFile, m_etadata));
+#else
     ModelState initialState(StructureFactory::stateFromFile(initialFileName));
+#endif
     modelStep.setData(pData);
     modelStep.setMetadata(m_etadata);
     pData.setData(initialState.data);
@@ -115,6 +137,10 @@ Model::HelpMap& Model::getHelpText(HelpMap& map, bool getAll)
             "The file path to the restart file to use for the run." },
         { keyMap.at(MISSINGVALUE_KEY), ConfigType::NUMERIC, { "-∞", "∞" }, "-2³⁰⁰", "",
             "Missing data indicator used for input and output." },
+#ifdef USE_MPI
+        { keyMap.at(PARTITIONFILE_KEY), ConfigType::STRING, {}, "", "",
+            "The file path to the file describing MPI domain decomposition to use for the run." },
+#endif
     };
 
     return map;

core/src/ModelMetadata.cpp

@@ -16,4 +16,13 @@ const std::string& ModelMetadata::structureName() const
     return Module::getImplementation<IStructure>().structureType();
 }
 
+#ifdef USE_MPI
+void ModelMetadata::setMpiMetadata(MPI_Comm comm)
+{
+    mpiComm = comm;
+    MPI_Comm_size(mpiComm, &mpiSize);
+    MPI_Comm_rank(mpiComm, &mpiMyRank);
+}
+#endif
+
 } /* namespace Nextsim */

core/src/ParallelNetcdfFile.cpp

@@ -0,0 +1,46 @@
+#include <ncCheck.h>
+#include <netcdf.h>
+#include <netcdf_par.h>
+
+#include "include/ParallelNetcdfFile.hpp"
+
+using namespace netCDF;
+
+// Not sure why it is needed but let's replicate netCDF::ncFile::open
+// in this respect
+extern int g_ncid;
+
+NcFilePar::NcFilePar(
+    const std::string& filePath, const FileMode fMode, MPI_Comm comm, MPI_Info mpiInfo)
+{
+    open_par(filePath, fMode, comm, mpiInfo);
+}
+
+void NcFilePar::open_par(
+    const std::string& filePath, const FileMode fMode, MPI_Comm comm, MPI_Info mpiInfo)
+{
+    if (!nullObject)
+        close();
+
+    switch (fMode) {
+    case NcFile::write:
+        ncCheck(nc_open_par(filePath.c_str(), NC_WRITE, comm, mpiInfo, &myId), __FILE__, __LINE__);
+        break;
+    case NcFile::read:
+        ncCheck(
+            nc_open_par(filePath.c_str(), NC_NOWRITE, comm, mpiInfo, &myId), __FILE__, __LINE__);
+        break;
+    case NcFile::newFile:
+        ncCheck(nc_create_par(filePath.c_str(), NC_NETCDF4 | NC_NOCLOBBER, comm, mpiInfo, &myId),
+            __FILE__, __LINE__);
+        break;
+    case NcFile::replace:
+        ncCheck(nc_create_par(filePath.c_str(), NC_NETCDF4 | NC_CLOBBER, comm, mpiInfo, &myId),
+            __FILE__, __LINE__);
+        break;
+    }
+
+    g_ncid = myId;
+
+    nullObject = false;
+}