Skip to content

Commit

Permalink
Extended checkpointing functionality when combining preInlet along wi…
Browse files Browse the repository at this point in the history 
…th interiorViscosity or solidify_mechanics
  • Loading branch information
@asterioukonstantinos
asterioukonstantinos committed Jan 6, 2023
1 parent 3189621 commit 6fead36
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 49 deletions.
25 changes: 13 additions & 12 deletions core/hemoCell.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,6 @@ void HemoCell::iterate() {
if(iter %cellfields->particleVelocityUpdateTimescale == 0) {
// #### 3 #### IBM interpolation
cellfields->interpolateFluidVelocity();

// ### 4 ### sync the particles
cellfields->syncEnvelopes();
}
Expand Down Expand Up @@ -482,18 +481,20 @@ void HemoCell::initializeLattice(MultiBlockManagement3D const & management) {
totalNodes += preInlet->getNumberOfNodes();
totalNodes += cellsInBoundingBox(management.getBoundingBox());

preInlet->nProcs = global::mpi().getSize()*(preInlet->getNumberOfNodes()/(T)totalNodes);
if (preInlet->nProcs == 0) {
preInlet->nProcs = 1;
try { // Look for block management info in the config file
plint preInlet_pABx = (*cfg)["preInlet"]["parameters"]["pABx"].read<plint>();
plint preInlet_pABy = (*cfg)["preInlet"]["parameters"]["pABy"].read<plint>();
plint preInlet_pABz = (*cfg)["preInlet"]["parameters"]["pABz"].read<plint>();
preInlet->nProcs = preInlet_pABx*preInlet_pABy*preInlet_pABz;
}
catch (const std::invalid_argument& e) {
preInlet->nProcs = global::mpi().getSize()*(preInlet->getNumberOfNodes()/(T)totalNodes);
if (preInlet->nProcs == 0) {
preInlet->nProcs = 1;
}
}

// JON addition: Try to read in number of processors allocated to preinlet from config file.
// TODO: Do we still need this?
// Just continue with value computed above if reading it from XML throws an exception because it does not exist
try { preInlet->nProcs = (*cfg)["preInlet"]["parameters"]["nProcs"].read<int>(); }
catch (const std::invalid_argument& e) {}

int nProcs = global::mpi().getSize()-preInlet->nProcs;
int nProcs = global::mpi().getSize() - preInlet->nProcs;

//Assign processors to PreInlet or Domain
unsigned int currentPreInlet = 0;
Expand Down Expand Up @@ -692,4 +693,4 @@ void HemoCell::sanityCheck() {


sanityCheckDone = true;
}
}
32 changes: 18 additions & 14 deletions helper/bindingField.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
#include "hemocell.h"
#include "palabos3D.h"
#include "palabos3D.hh"
#include "preInlet.h"

namespace hemo {
bindingFieldHelper::bindingFieldHelper(HemoCellFields * cellFields_) : cellFields(*cellFields_) {
Expand All @@ -36,26 +37,26 @@ namespace hemo {

//Create bindingfield with same properties as fluid field underlying the particleField.
multiBindingField = new plb::MultiScalarField3D<bool>(
MultiBlockManagement3D (
*cellFields.hemocell.lattice->getSparseBlockStructure().clone(),
cellFields.hemocell.lattice->getMultiBlockManagement().getThreadAttribution().clone(),
cellFields.hemocell.lattice->getMultiBlockManagement().getEnvelopeWidth(),
cellFields.hemocell.lattice->getMultiBlockManagement().getRefinementLevel()),
MultiBlockManagement3D (
*cellFields.hemocell.domain_lattice->getSparseBlockStructure().clone(),
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getThreadAttribution().clone(),
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getEnvelopeWidth(),
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getRefinementLevel()),
defaultMultiBlockPolicy3D().getBlockCommunicator(),
defaultMultiBlockPolicy3D().getCombinedStatistics(),
defaultMultiBlockPolicy3D().getMultiScalarAccess<bool>(),
0);
multiBindingField->periodicity().toggle(0,cellFields.hemocell.lattice->periodicity().get(0));
multiBindingField->periodicity().toggle(1,cellFields.hemocell.lattice->periodicity().get(1));
multiBindingField->periodicity().toggle(2,cellFields.hemocell.lattice->periodicity().get(2));
multiBindingField->periodicity().toggle(0,cellFields.hemocell.domain_lattice->periodicity().get(0));
multiBindingField->periodicity().toggle(1,cellFields.hemocell.domain_lattice->periodicity().get(1));
multiBindingField->periodicity().toggle(2,cellFields.hemocell.domain_lattice->periodicity().get(2));

multiBindingField->initialize();

//Make sure each particleField has access to its local scalarField
for (const plint & bId : multiBindingField->getLocalInfo().getBlocks()) {
HemoCellParticleField & pf = cellFields.immersedParticles->getComponent(bId);
HemoCellParticleField & pf = cellFields.domain_immersedParticles->getComponent(bId);
pf.bindingField = &multiBindingField->getComponent(bId);
}

}

bindingFieldHelper::~bindingFieldHelper() {
Expand All @@ -67,15 +68,17 @@ namespace hemo {
pcout << "(BindingField) Checkpoint called while global.enableSolidifyMechanics is not enabled, still checkpointing but this should not happen" << endl;
return;
}

std::string & outDir = hemo::global.checkpointDirectory;
mkpath(outDir.c_str(), 0777);

if (global::mpi().isMainProcessor()) {
renameFileToDotOld(outDir + "bindingSites.dat");
renameFileToDotOld(outDir + "bindingSites.plb");
}

plb::parallelIO::save(*multiBindingField, outDir + "bindingSites", true);

}

void bindingFieldHelper::restore(HemoCellFields & cellFields) {
Expand Down Expand Up @@ -119,8 +122,8 @@ namespace hemo {
}

void bindingFieldHelper::refillBindingSites() {
for (const plint & bId : cellFields.immersedParticles->getLocalInfo().getBlocks()) {
HemoCellParticleField & pf = cellFields.immersedParticles->getComponent(bId);
for (const plint & bId : cellFields.domain_immersedParticles->getLocalInfo().getBlocks()) {
HemoCellParticleField & pf = cellFields.domain_immersedParticles->getComponent(bId);
ScalarField3D<bool> & bf = *pf.bindingField;
Box3D domain = bf.getBoundingBox();
for (int x = domain.x0; x <= domain.x1 ; x++) {
Expand All @@ -134,4 +137,5 @@ namespace hemo {
}
}
}

}
22 changes: 11 additions & 11 deletions helper/interiorViscosity.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,23 +31,23 @@ namespace hemo {
//Create bindingfield with same properties as fluid field underlying the particleField.
multiInteriorViscosityField = new plb::MultiScalarField3D<T>(
MultiBlockManagement3D (
*cellFields.hemocell.lattice->getSparseBlockStructure().clone(),
cellFields.hemocell.lattice->getMultiBlockManagement().getThreadAttribution().clone(),
cellFields.hemocell.lattice->getMultiBlockManagement().getEnvelopeWidth(),
cellFields.hemocell.lattice->getMultiBlockManagement().getRefinementLevel()),
*cellFields.hemocell.domain_lattice->getSparseBlockStructure().clone(),
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getThreadAttribution().clone(),
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getEnvelopeWidth(),
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getRefinementLevel()),
defaultMultiBlockPolicy3D().getBlockCommunicator(),
defaultMultiBlockPolicy3D().getCombinedStatistics(),
defaultMultiBlockPolicy3D().getMultiScalarAccess<T>(),
0);
multiInteriorViscosityField->periodicity().toggle(0,cellFields.hemocell.lattice->periodicity().get(0));
multiInteriorViscosityField->periodicity().toggle(1,cellFields.hemocell.lattice->periodicity().get(1));
multiInteriorViscosityField->periodicity().toggle(2,cellFields.hemocell.lattice->periodicity().get(2));
multiInteriorViscosityField->periodicity().toggle(0,cellFields.hemocell.domain_lattice->periodicity().get(0));
multiInteriorViscosityField->periodicity().toggle(1,cellFields.hemocell.domain_lattice->periodicity().get(1));
multiInteriorViscosityField->periodicity().toggle(2,cellFields.hemocell.domain_lattice->periodicity().get(2));

multiInteriorViscosityField->initialize();

//Make sure each particleField has access to its local scalarField
for (const plint & bId : multiInteriorViscosityField->getLocalInfo().getBlocks()) {
HemoCellParticleField & pf = cellFields.immersedParticles->getComponent(bId);
HemoCellParticleField & pf = cellFields.domain_immersedParticles->getComponent(bId);
pf.interiorViscosityField = &multiInteriorViscosityField->getComponent(bId);
}
}
Expand Down Expand Up @@ -119,8 +119,8 @@ namespace hemo {
}

void InteriorViscosityHelper::refillBindingSites() {
for (const plint & bId : cellFields.immersedParticles->getLocalInfo().getBlocks()) {
HemoCellParticleField & pf = cellFields.immersedParticles->getComponent(bId);
for (const plint & bId : cellFields.domain_immersedParticles->getLocalInfo().getBlocks()) {
HemoCellParticleField & pf = cellFields.domain_immersedParticles->getComponent(bId);
ScalarField3D<T> & bf = *pf.interiorViscosityField;
Box3D domain = bf.getBoundingBox();
for (int x = domain.x0; x <= domain.x1 ; x++) {
Expand All @@ -139,4 +139,4 @@ namespace hemo {
}
}
}
}
}
16 changes: 4 additions & 12 deletions helper/preInlet.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,12 +359,7 @@ void PreInlet::applyPreInletVelocityBoundary() {
int z_o = domain.getNz();
int y_o = domain.getNy();
int tag;

plb::Array<T,3> receiver;
plb::Array<T,3> vel;
std::vector<plb::Array<T,3>> buffer;
std::vector<MPI_Request> requests;

for (int bId : hemocell->lattice->getLocalInfo().getBlocks()) {
Box3D bulk = hemocell->lattice->getMultiBlockManagement().getBulk(bId);
if (!intersect(domain,bulk,result)) { continue; }
Expand All @@ -382,25 +377,22 @@ void PreInlet::applyPreInletVelocityBoundary() {
int dest = hemocell->domain_lattice_management->getThreadAttribution().getMpiProcess(hemocell->domain_lattice_management->getSparseBlockStructure().locate(x+loc.x,y+loc.y,z+loc.z));

// send velocity from preInlet boundary
buffer.push_back(vel);
requests.push_back(MPI_Request());
MPI_Isend(&buffer.back()[0],3*sizeof(T),MPI_CHAR,dest,tag,MPI_COMM_WORLD,&requests.back());
MPI_Send(&vel[0],3*sizeof(T),MPI_CHAR,dest,tag,MPI_COMM_WORLD);
} else {
Box3D point(x,x,y,y,z,z);
int source = hemocell->preinlet_lattice_management->getThreadAttribution().getMpiProcess(hemocell->preinlet_lattice_management->getSparseBlockStructure().locate(x+loc.x,y+loc.y,z+loc.z));

// receive velocity from preInlet
MPI_Recv(&receiver,3*sizeof(T),MPI_CHAR,source,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Recv(&vel[0],3*sizeof(T),MPI_CHAR,source,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);

// update velocity lattice main domain
setBoundaryVelocity(hemocell->lattice->getComponent(bId),point,receiver);
setBoundaryVelocity(hemocell->lattice->getComponent(bId),point,vel);
}
}
}
}
}
}
MPI_Waitall(requests.size(), &requests[0], MPI_STATUS_IGNORE);
global.statistics.getCurrent().stop();
}

Expand Down Expand Up @@ -996,4 +988,4 @@ void PreInlet::createBoundary() {
PreInlet::CreatePreInletBoundingBox * PreInlet::CreatePreInletBoundingBox::clone() const { return new PreInlet::CreatePreInletBoundingBox(*this);}
PreInlet::CreateDrivingForceFunctional * PreInlet::CreateDrivingForceFunctional::clone() const { return new PreInlet::CreateDrivingForceFunctional(*this);}
PreInlet::FillFlagMatrix * PreInlet::FillFlagMatrix::clone() const { return new PreInlet::FillFlagMatrix(*this);}
}
}

0 comments on commit 6fead36

Please sign in to comment.