OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adding MPI wrapper class

Browse Source
This commit is contained in:
Mark Berrill
2020-01-28 08:51:32 -05:00
parent acb2d30454
commit d1f714a82e
125 changed files with 8530 additions and 2541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
tests/TestBubble.cpp
View File

@@ -7,7 +7,7 @@
#include "analysis/pmmc.h"
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "IO/Mesh.h"
#include "IO/Writer.h"
@@ -32,14 +32,15 @@ int main(int argc, char **argv)
// Initialize MPI
int provided_thread_support = -1;
MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support);
MPI_Comm comm;
MPI_Comm_dup(MPI_COMM_WORLD,&comm);
int rank = comm_rank(comm);
int nprocs = comm_size(comm);
if ( rank==0 && provided_thread_support<MPI_THREAD_MULTIPLE )
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
auto comm = Utilities::MPI( MPI_COMM_WORLD ).dup();
int rank = comm.getRank();
int nprocs = comm.getSize();
if ( rank==0 && provided_thread_support<MPI_THREAD_MULTIPLE )
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
@@ -119,7 +120,7 @@ int main(int argc, char **argv)
int jproc = rank_info.jy;
int kproc = rank_info.kz;
MPI_Barrier(comm);
comm.barrier();
// **************************************************************
// **************************************************************
double Ps = -(das-dbs)/(das+dbs);
@@ -162,7 +163,7 @@ int main(int argc, char **argv)
// Mask that excludes the solid phase
Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,pBC);
MPI_Barrier(comm);
comm.barrier();
Nx+=2; Ny+=2; Nz += 2;
@@ -432,7 +433,7 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
MPI_Barrier(comm);
comm.barrier();
starttime = MPI_Wtime();
//.........................................
//...........................................................................
@@ -517,7 +518,7 @@ int main(int argc, char **argv)
ScaLBL_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
ScaLBL_CopyToDevice(Den,cDen,2*N*sizeof(double));
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
}
//*************************************************************************
@@ -529,7 +530,7 @@ int main(int argc, char **argv)
ScaLBL_Comm.SendHalo(Phi);
ScaLBL_Comm.RecvHalo(Phi);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
if (rank==0 && pBC){
@@ -560,7 +561,7 @@ int main(int argc, char **argv)
ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
ScaLBL_CopyToHost(Phase.data(),Phi,N*sizeof(double));
ScaLBL_CopyToHost(Press.data(),Pressure,N*sizeof(double));
MPI_Barrier(comm);
comm.barrier();
//...........................................................................
int timestep = 0;
@@ -591,7 +592,7 @@ int main(int argc, char **argv)
//*************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
// Swap the distributions for momentum transport
//*************************************************************************
@@ -599,7 +600,7 @@ int main(int argc, char **argv)
//*************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
// Wait for communications to complete and unpack the distributions
ScaLBL_Comm.RecvD3Q19(f_even, f_odd);
@@ -616,7 +617,7 @@ int main(int argc, char **argv)
ScaLBL_D3Q7_Swap(ID, B_even, B_odd, Nx, Ny, Nz);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
//*************************************************************************
// Wait for communication and unpack the D3Q7 distributions
@@ -633,7 +634,7 @@ int main(int argc, char **argv)
//*************************************************************************
// ScaLBL_ComputePhaseField(ID, Phi, Copy, Den, N);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
ScaLBL_ComputePhaseField(ID, Phi, Den, N);
//*************************************************************************
@@ -659,7 +660,7 @@ int main(int argc, char **argv)
//...................................................................................
MPI_Barrier(comm);
comm.barrier();
// Timestep completed!
timestep++;
@@ -807,7 +808,7 @@ int main(int argc, char **argv)
//...........................................................................
}
//...........................................................................
MPI_Barrier(comm);
comm.barrier();
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
@@ -827,7 +828,7 @@ int main(int argc, char **argv)
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Barrier(comm);
comm.barrier();
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
// See McClure, Prins and Miller (2013)
@@ -950,7 +951,7 @@ int main(int argc, char **argv)
//************************************************************************/
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
comm.barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
@@ -989,9 +990,8 @@ int main(int argc, char **argv)
PROFILE_SAVE("TestBubble");
// ****************************************************
MPI_Barrier(comm);
comm.barrier();
} // Limit scope so variables that contain communicators will free before MPI_Finialize
MPI_Comm_free(&comm);
MPI_Finalize();
return 0;
}