OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "Moving more MPI calls to the wrapper"

Browse Source 
This reverts commit 0f91767b6c.
This commit is contained in:
JamesEMcclure 2020-03-17 21:23:18 -04:00
parent fa61d19095
commit 9f5b44dfe4
53 changed files with 2678 additions and 2360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
IO/netcdf.cpp
View File

@ -119,7 +119,7 @@ std::string VariableTypeName( VariableType type )
int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm )
{
int fid = 0;
if ( comm.isNull() ) {
if ( comm == MPI_COMM_NULL ) {
if ( mode == READ ) {
int err = nc_open( filename.c_str(), NC_NOWRITE, &fid );
CHECK_NC_ERR( err );

2
StackTrace/ErrorHandlers.h
View File

@ -6,7 +6,7 @@
#include <functional>
#include "common/MPI.h"
#include "mpi.h"
namespace StackTrace

2
StackTrace/Utilities.cpp
View File

@ -14,7 +14,7 @@
#include <typeinfo>
#ifdef USE_MPI
#include "common/MPI.h"
#include "mpi.h"
#endif
#ifdef USE_TIMER

7
analysis/TwoPhase.cpp
View File

@ -890,14 +890,14 @@ void TwoPhase::ComponentAverages()
RecvBuffer.resize(BLOB_AVG_COUNT,NumberComponents_NWP);
/* for (int b=0; b<NumberComponents_NWP; b++){
Dm->Comm.barrier();
Dm->Comm.sumReduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT);
MPI_Barrier(Dm->Comm);
MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm);
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_NWP(idx,b)=RecvBuffer(idx);
}
*/
Dm->Comm.barrier();
Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP);
// Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT);
// MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm);
if (Dm->rank()==0){
printf("rescaling... \n");
@ -994,6 +994,7 @@ void TwoPhase::ComponentAverages()
// reduce the wetting phase averages
for (int b=0; b<NumberComponents_WP; b++){
Dm->Comm.barrier();
// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm);
Dm->Comm.sumReduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT);
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_WP(idx,b)=RecvBuffer(idx);
}

132
analysis/morphology.cpp
View File

@ -201,24 +201,42 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
//......................................................................................
Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag);
MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
@ -285,7 +303,7 @@ double morph_open()
fillHalo<char> fillChar(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1);
GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
count = 0.f;
for (int k=1; k<Nz-1; k++){
@ -298,7 +316,7 @@ double morph_open()
}
}
}
countGlobal = Dm->Comm.sumReduce( count );
MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
return countGlobal;
}
*/
@ -488,42 +506,42 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
//......................................................................................
Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag);
MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
@ -599,7 +617,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
Dm->Comm.barrier();
MPI_Barrier(Dm->Comm);
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){

216
common/Communication.h
View File

@ -134,7 +134,7 @@ void InitializeRanks( const int rank, const int nprocx, const int nprocy, const
//***************************************************************************************
inline void CommunicateSendRecvCounts( const Utilities::MPI& comm, int sendtag, int recvtag,
inline void CommunicateSendRecvCounts( const Utilities::MPI& Communicator, int sendtag, int recvtag,
int rank_x, int rank_y, int rank_z,
int rank_X, int rank_Y, int rank_Z,
int rank_xy, int rank_XY, int rank_xY, int rank_Xy,
@ -152,53 +152,54 @@ inline void CommunicateSendRecvCounts( const Utilities::MPI& comm, int sendtag,
int& recvCount_yz, int& recvCount_YZ, int& recvCount_yZ, int& recvCount_Yz )
{
MPI_Request req1[18], req2[18];
req1[0] = comm.Isend(&sendCount_x,1,rank_x,sendtag+0);
req2[0] = comm.Irecv(&recvCount_X,1,rank_X,recvtag+0);
req1[1] = comm.Isend(&sendCount_X,1,rank_X,sendtag+1);
req2[1] = comm.Irecv(&recvCount_x,1,rank_x,recvtag+1);
req1[2] = comm.Isend(&sendCount_y,1,rank_y,sendtag+2);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_Y,recvtag+2);
req1[3] = comm.Isend(&sendCount_Y,1,rank_Y,sendtag+3);
req2[3] = comm.Irecv(&recvCount_y,1,rank_y,recvtag+3);
req1[4] = comm.Isend(&sendCount_z,1,rank_z,sendtag+4);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_Z,recvtag+4);
req1[5] = comm.Isend(&sendCount_Z,1,rank_Z,sendtag+5);
req2[5] = comm.Irecv(&recvCount_z,1,rank_z,recvtag+5);
MPI_Status stat1[18],stat2[18];
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator.getCommunicator(),&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator.getCommunicator(),&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator.getCommunicator(),&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator.getCommunicator(),&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator.getCommunicator(),&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator.getCommunicator(),&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator.getCommunicator(),&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator.getCommunicator(),&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator.getCommunicator(),&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator.getCommunicator(),&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator.getCommunicator(),&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator.getCommunicator(),&req2[5]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_xy,sendtag+6);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_XY,recvtag+6);
req1[7] = comm.Isend(&sendCount_XY,1,rank_XY,sendtag+7);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_xy,recvtag+7);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_Xy,sendtag+8);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_xY,recvtag+8);
req1[9] = comm.Isend(&sendCount_xY,1,rank_xY,sendtag+9);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_Xy,recvtag+9);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator.getCommunicator(),&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator.getCommunicator(),&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator.getCommunicator(),&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator.getCommunicator(),&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator.getCommunicator(),&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator.getCommunicator(),&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator.getCommunicator(),&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator.getCommunicator(),&req2[9]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_xz,sendtag+10);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_XZ,recvtag+10);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_XZ,sendtag+11);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_xz,recvtag+11);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_Xz,sendtag+12);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_xZ,recvtag+12);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_xZ,sendtag+13);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_Xz,recvtag+13);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator.getCommunicator(),&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator.getCommunicator(),&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator.getCommunicator(),&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator.getCommunicator(),&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator.getCommunicator(),&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator.getCommunicator(),&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator.getCommunicator(),&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator.getCommunicator(),&req2[13]);
req1[14] = comm.Isend(&sendCount_yz,1,rank_yz,sendtag+14);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_YZ,recvtag+14);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_YZ,sendtag+15);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_yz,recvtag+15);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_Yz,sendtag+16);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_yZ,recvtag+16);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_yZ,sendtag+17);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_Yz,recvtag+17);
comm.waitAll( 18, req1 );
comm.waitAll( 18, req2 );
comm.barrier();
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator.getCommunicator(),&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator.getCommunicator(),&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator.getCommunicator(),&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator.getCommunicator(),&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator.getCommunicator(),&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator.getCommunicator(),&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator.getCommunicator(),&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator.getCommunicator(),&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
Communicator.barrier();
}
//***************************************************************************************
inline void CommunicateRecvLists( const Utilities::MPI& comm, int sendtag, int recvtag,
inline void CommunicateRecvLists( const Utilities::MPI& Communicator, int sendtag, int recvtag,
int *sendList_x, int *sendList_y, int *sendList_z, int *sendList_X, int *sendList_Y, int *sendList_Z,
int *sendList_xy, int *sendList_XY, int *sendList_xY, int *sendList_Xy,
int *sendList_xz, int *sendList_XZ, int *sendList_xZ, int *sendList_Xz,
@ -219,52 +220,53 @@ inline void CommunicateRecvLists( const Utilities::MPI& comm, int sendtag, int r
int rank_Xy, int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz, int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz)
{
MPI_Request req1[18], req2[18];
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_z,recvtag);
MPI_Status stat1[18],stat2[18];
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator.getCommunicator(),&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator.getCommunicator(),&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator.getCommunicator(),&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator.getCommunicator(),&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator.getCommunicator(),&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator.getCommunicator(),&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator.getCommunicator(),&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator.getCommunicator(),&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator.getCommunicator(),&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator.getCommunicator(),&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator.getCommunicator(),&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator.getCommunicator(),&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator.getCommunicator(),&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator.getCommunicator(),&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator.getCommunicator(),&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator.getCommunicator(),&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator.getCommunicator(),&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator.getCommunicator(),&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator.getCommunicator(),&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator.getCommunicator(),&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator.getCommunicator(),&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator.getCommunicator(),&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator.getCommunicator(),&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator.getCommunicator(),&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator.getCommunicator(),&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator.getCommunicator(),&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator.getCommunicator(),&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator.getCommunicator(),&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz,recvtag);
comm.waitAll( 18, req1 );
comm.waitAll( 18, req2 );
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator.getCommunicator(),&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator.getCommunicator(),&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator.getCommunicator(),&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator.getCommunicator(),&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator.getCommunicator(),&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator.getCommunicator(),&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator.getCommunicator(),&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator.getCommunicator(),&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
}
//***************************************************************************************
inline void CommunicateMeshHalo(DoubleArray &Mesh, const Utilities::MPI& comm,
inline void CommunicateMeshHalo(DoubleArray &Mesh, const Utilities::MPI& Communicator,
double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy,
double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz,
@ -314,24 +316,42 @@ inline void CommunicateMeshHalo(DoubleArray &Mesh, const Utilities::MPI& comm,
PackMeshData(sendList_yZ, sendCount_yZ ,sendbuf_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData);
//......................................................................................
comm.sendrecv(sendbuf_x,sendCount_x,rank_x,sendtag,recvbuf_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendbuf_X,sendCount_X,rank_X,sendtag,recvbuf_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendbuf_y,sendCount_y,rank_y,sendtag,recvbuf_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendbuf_Y,sendCount_Y,rank_Y,sendtag,recvbuf_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendbuf_z,sendCount_z,rank_z,sendtag,recvbuf_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendbuf_Z,sendCount_Z,rank_Z,sendtag,recvbuf_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendbuf_xy,sendCount_xy,rank_xy,sendtag,recvbuf_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendbuf_XY,sendCount_XY,rank_XY,sendtag,recvbuf_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag,recvbuf_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendbuf_xY,sendCount_xY,rank_xY,sendtag,recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendbuf_xz,sendCount_xz,rank_xz,sendtag,recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag,recvbuf_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag,recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag,recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendbuf_yz,sendCount_yz,rank_yz,sendtag,recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag,recvbuf_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag,recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag,recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_DOUBLE,rank_x,sendtag,
recvbuf_X,recvCount_X,MPI_DOUBLE,rank_X,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_DOUBLE,rank_X,sendtag,
recvbuf_x,recvCount_x,MPI_DOUBLE,rank_x,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_DOUBLE,rank_y,sendtag,
recvbuf_Y,recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,
recvbuf_y,recvCount_y,MPI_DOUBLE,rank_y,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_DOUBLE,rank_z,sendtag,
recvbuf_Z,recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,
recvbuf_z,recvCount_z,MPI_DOUBLE,rank_z,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,
recvbuf_XY,recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,
recvbuf_xy,recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,
recvbuf_xY,recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,
recvbuf_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,
recvbuf_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,
recvbuf_xz,recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,
recvbuf_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,
recvbuf_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,
recvbuf_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,
recvbuf_yz,recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,
recvbuf_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,
recvbuf_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData);

206
common/Domain.cpp
View File

@ -837,44 +837,44 @@ void Domain::CommInit()
sendBuf_YZ = new int [sendCount_YZ];
sendBuf_XZ = new int [sendCount_XZ];
//......................................................................................
req1[0] = Comm.Isend(&sendCount_x,1,rank_x(),sendtag+0);
req2[0] = Comm.Irecv(&recvCount_X,1,rank_X(),recvtag+0);
req1[1] = Comm.Isend(&sendCount_X,1,rank_X(),sendtag+1);
req2[1] = Comm.Irecv(&recvCount_x,1,rank_x(),recvtag+1);
req1[2] = Comm.Isend(&sendCount_y,1,rank_y(),sendtag+2);
req2[2] = Comm.Irecv(&recvCount_Y,1,rank_Y(),recvtag+2);
req1[3] = Comm.Isend(&sendCount_Y,1,rank_Y(),sendtag+3);
req2[3] = Comm.Irecv(&recvCount_y,1,rank_y(),recvtag+3);
req1[4] = Comm.Isend(&sendCount_z,1,rank_z(),sendtag+4);
req2[4] = Comm.Irecv(&recvCount_Z,1,rank_Z(),recvtag+4);
req1[5] = Comm.Isend(&sendCount_Z,1,rank_Z(),sendtag+5);
req2[5] = Comm.Irecv(&recvCount_z,1,rank_z(),recvtag+5);
req1[6] = Comm.Isend(&sendCount_xy,1,rank_xy(),sendtag+6);
req2[6] = Comm.Irecv(&recvCount_XY,1,rank_XY(),recvtag+6);
req1[7] = Comm.Isend(&sendCount_XY,1,rank_XY(),sendtag+7);
req2[7] = Comm.Irecv(&recvCount_xy,1,rank_xy(),recvtag+7);
req1[8] = Comm.Isend(&sendCount_Xy,1,rank_Xy(),sendtag+8);
req2[8] = Comm.Irecv(&recvCount_xY,1,rank_xY(),recvtag+8);
req1[9] = Comm.Isend(&sendCount_xY,1,rank_xY(),sendtag+9);
req2[9] = Comm.Irecv(&recvCount_Xy,1,rank_Xy(),recvtag+9);
req1[10] = Comm.Isend(&sendCount_xz,1,rank_xz(),sendtag+10);
req2[10] = Comm.Irecv(&recvCount_XZ,1,rank_XZ(),recvtag+10);
req1[11] = Comm.Isend(&sendCount_XZ,1,rank_XZ(),sendtag+11);
req2[11] = Comm.Irecv(&recvCount_xz,1,rank_xz(),recvtag+11);
req1[12] = Comm.Isend(&sendCount_Xz,1,rank_Xz(),sendtag+12);
req2[12] = Comm.Irecv(&recvCount_xZ,1,rank_xZ(),recvtag+12);
req1[13] = Comm.Isend(&sendCount_xZ,1,rank_xZ(),sendtag+13);
req2[13] = Comm.Irecv(&recvCount_Xz,1,rank_Xz(),recvtag+13);
req1[14] = Comm.Isend(&sendCount_yz,1,rank_yz(),sendtag+14);
req2[14] = Comm.Irecv(&recvCount_YZ,1,rank_YZ(),recvtag+14);
req1[15] = Comm.Isend(&sendCount_YZ,1,rank_YZ(),sendtag+15);
req2[15] = Comm.Irecv(&recvCount_yz,1,rank_yz(),recvtag+15);
req1[16] = Comm.Isend(&sendCount_Yz,1,rank_Yz(),sendtag+16);
req2[16] = Comm.Irecv(&recvCount_yZ,1,rank_yZ(),recvtag+16);
req1[17] = Comm.Isend(&sendCount_yZ,1,rank_yZ(),sendtag+17);
req2[17] = Comm.Irecv(&recvCount_Yz,1,rank_Yz(),recvtag+17);
Comm.waitAll(18,req1);
Comm.waitAll(18,req2);
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x(),sendtag+0,Comm.getCommunicator(),&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X(),recvtag+0,Comm.getCommunicator(),&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X(),sendtag+1,Comm.getCommunicator(),&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x(),recvtag+1,Comm.getCommunicator(),&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y(),sendtag+2,Comm.getCommunicator(),&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y(),recvtag+2,Comm.getCommunicator(),&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y(),sendtag+3,Comm.getCommunicator(),&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y(),recvtag+3,Comm.getCommunicator(),&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z(),sendtag+4,Comm.getCommunicator(),&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z(),recvtag+4,Comm.getCommunicator(),&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z(),sendtag+5,Comm.getCommunicator(),&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z(),recvtag+5,Comm.getCommunicator(),&req2[5]);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy(),sendtag+6,Comm.getCommunicator(),&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY(),recvtag+6,Comm.getCommunicator(),&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY(),sendtag+7,Comm.getCommunicator(),&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy(),recvtag+7,Comm.getCommunicator(),&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy(),sendtag+8,Comm.getCommunicator(),&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY(),recvtag+8,Comm.getCommunicator(),&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY(),sendtag+9,Comm.getCommunicator(),&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy(),recvtag+9,Comm.getCommunicator(),&req2[9]);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz(),sendtag+10,Comm.getCommunicator(),&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ(),recvtag+10,Comm.getCommunicator(),&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ(),sendtag+11,Comm.getCommunicator(),&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz(),recvtag+11,Comm.getCommunicator(),&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz(),sendtag+12,Comm.getCommunicator(),&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ(),recvtag+12,Comm.getCommunicator(),&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ(),sendtag+13,Comm.getCommunicator(),&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz(),recvtag+13,Comm.getCommunicator(),&req2[13]);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz(),sendtag+14,Comm.getCommunicator(),&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ(),recvtag+14,Comm.getCommunicator(),&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ(),sendtag+15,Comm.getCommunicator(),&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz(),recvtag+15,Comm.getCommunicator(),&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz(),sendtag+16,Comm.getCommunicator(),&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ(),recvtag+16,Comm.getCommunicator(),&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ(),sendtag+17,Comm.getCommunicator(),&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz(),recvtag+17,Comm.getCommunicator(),&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
Comm.barrier();
//......................................................................................
// recv buffers
@ -897,44 +897,44 @@ void Domain::CommInit()
recvList_YZ = new int [recvCount_YZ];
recvList_XZ = new int [recvCount_XZ];
//......................................................................................
req1[0] = Comm.Isend(sendList_x,sendCount_x,rank_x(),sendtag);
req2[0] = Comm.Irecv(recvList_X,recvCount_X,rank_X(),recvtag);
req1[1] = Comm.Isend(sendList_X,sendCount_X,rank_X(),sendtag);
req2[1] = Comm.Irecv(recvList_x,recvCount_x,rank_x(),recvtag);
req1[2] = Comm.Isend(sendList_y,sendCount_y,rank_y(),sendtag);
req2[2] = Comm.Irecv(recvList_Y,recvCount_Y,rank_Y(),recvtag);
req1[3] = Comm.Isend(sendList_Y,sendCount_Y,rank_Y(),sendtag);
req2[3] = Comm.Irecv(recvList_y,recvCount_y,rank_y(),recvtag);
req1[4] = Comm.Isend(sendList_z,sendCount_z,rank_z(),sendtag);
req2[4] = Comm.Irecv(recvList_Z,recvCount_Z,rank_Z(),recvtag);
req1[5] = Comm.Isend(sendList_Z,sendCount_Z,rank_Z(),sendtag);
req2[5] = Comm.Irecv(recvList_z,recvCount_z,rank_z(),recvtag);
req1[6] = Comm.Isend(sendList_xy,sendCount_xy,rank_xy(),sendtag);
req2[6] = Comm.Irecv(recvList_XY,recvCount_XY,rank_XY(),recvtag);
req1[7] = Comm.Isend(sendList_XY,sendCount_XY,rank_XY(),sendtag);
req2[7] = Comm.Irecv(recvList_xy,recvCount_xy,rank_xy(),recvtag);
req1[8] = Comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy(),sendtag);
req2[8] = Comm.Irecv(recvList_xY,recvCount_xY,rank_xY(),recvtag);
req1[9] = Comm.Isend(sendList_xY,sendCount_xY,rank_xY(),sendtag);
req2[9] = Comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy(),recvtag);
req1[10] = Comm.Isend(sendList_xz,sendCount_xz,rank_xz(),sendtag);
req2[10] = Comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ(),recvtag);
req1[11] = Comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ(),sendtag);
req2[11] = Comm.Irecv(recvList_xz,recvCount_xz,rank_xz(),recvtag);
req1[12] = Comm.Isend(sendList_Xz,sendCount_Xz,rank_Xz(),sendtag);
req2[12] = Comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ(),recvtag);
req1[13] = Comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ(),sendtag);
req2[13] = Comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz(),recvtag);
req1[14] = Comm.Isend(sendList_yz,sendCount_yz,rank_yz(),sendtag);
req2[14] = Comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ(),recvtag);
req1[15] = Comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ(),sendtag);
req2[15] = Comm.Irecv(recvList_yz,recvCount_yz,rank_yz(),recvtag);
req1[16] = Comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz(),sendtag);
req2[16] = Comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ(),recvtag);
req1[17] = Comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ(),sendtag);
req2[17] = Comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz(),recvtag);
Comm.waitAll(18,req1);
Comm.waitAll(18,req2);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x(),sendtag,Comm.getCommunicator(),&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X(),recvtag,Comm.getCommunicator(),&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X(),sendtag,Comm.getCommunicator(),&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x(),recvtag,Comm.getCommunicator(),&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y(),sendtag,Comm.getCommunicator(),&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y(),recvtag,Comm.getCommunicator(),&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y(),sendtag,Comm.getCommunicator(),&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y(),recvtag,Comm.getCommunicator(),&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z(),sendtag,Comm.getCommunicator(),&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z(),recvtag,Comm.getCommunicator(),&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z(),sendtag,Comm.getCommunicator(),&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z(),recvtag,Comm.getCommunicator(),&req2[5]);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy(),sendtag,Comm.getCommunicator(),&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY(),recvtag,Comm.getCommunicator(),&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY(),sendtag,Comm.getCommunicator(),&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy(),recvtag,Comm.getCommunicator(),&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy(),sendtag,Comm.getCommunicator(),&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY(),recvtag,Comm.getCommunicator(),&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY(),sendtag,Comm.getCommunicator(),&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy(),recvtag,Comm.getCommunicator(),&req2[9]);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz(),sendtag,Comm.getCommunicator(),&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ(),recvtag,Comm.getCommunicator(),&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ(),sendtag,Comm.getCommunicator(),&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz(),recvtag,Comm.getCommunicator(),&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz(),sendtag,Comm.getCommunicator(),&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ(),recvtag,Comm.getCommunicator(),&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ(),sendtag,Comm.getCommunicator(),&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz(),recvtag,Comm.getCommunicator(),&req2[13]);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz(),sendtag,Comm.getCommunicator(),&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ(),recvtag,Comm.getCommunicator(),&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ(),sendtag,Comm.getCommunicator(),&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz(),recvtag,Comm.getCommunicator(),&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz(),sendtag,Comm.getCommunicator(),&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ(),recvtag,Comm.getCommunicator(),&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ(),sendtag,Comm.getCommunicator(),&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz(),recvtag,Comm.getCommunicator(),&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -1124,24 +1124,42 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh)
PackMeshData(sendList_yZ, sendCount_yZ ,sendData_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendData_YZ, MeshData);
//......................................................................................
Comm.sendrecv(sendData_x,sendCount_x,rank_x(),sendtag,recvData_X,recvCount_X,rank_X(),recvtag);
Comm.sendrecv(sendData_X,sendCount_X,rank_X(),sendtag,recvData_x,recvCount_x,rank_x(),recvtag);
Comm.sendrecv(sendData_y,sendCount_y,rank_y(),sendtag,recvData_Y,recvCount_Y,rank_Y(),recvtag);
Comm.sendrecv(sendData_Y,sendCount_Y,rank_Y(),sendtag,recvData_y,recvCount_y,rank_y(),recvtag);
Comm.sendrecv(sendData_z,sendCount_z,rank_z(),sendtag,recvData_Z,recvCount_Z,rank_Z(),recvtag);
Comm.sendrecv(sendData_Z,sendCount_Z,rank_Z(),sendtag,recvData_z,recvCount_z,rank_z(),recvtag);
Comm.sendrecv(sendData_xy,sendCount_xy,rank_xy(),sendtag,recvData_XY,recvCount_XY,rank_XY(),recvtag);
Comm.sendrecv(sendData_XY,sendCount_XY,rank_XY(),sendtag,recvData_xy,recvCount_xy,rank_xy(),recvtag);
Comm.sendrecv(sendData_Xy,sendCount_Xy,rank_Xy(),sendtag,recvData_xY,recvCount_xY,rank_xY(),recvtag);
Comm.sendrecv(sendData_xY,sendCount_xY,rank_xY(),sendtag,recvData_Xy,recvCount_Xy,rank_Xy(),recvtag);
Comm.sendrecv(sendData_xz,sendCount_xz,rank_xz(),sendtag,recvData_XZ,recvCount_XZ,rank_XZ(),recvtag);
Comm.sendrecv(sendData_XZ,sendCount_XZ,rank_XZ(),sendtag,recvData_xz,recvCount_xz,rank_xz(),recvtag);
Comm.sendrecv(sendData_Xz,sendCount_Xz,rank_Xz(),sendtag,recvData_xZ,recvCount_xZ,rank_xZ(),recvtag);
Comm.sendrecv(sendData_xZ,sendCount_xZ,rank_xZ(),sendtag,recvData_Xz,recvCount_Xz,rank_Xz(),recvtag);
Comm.sendrecv(sendData_yz,sendCount_yz,rank_yz(),sendtag,recvData_YZ,recvCount_YZ,rank_YZ(),recvtag);
Comm.sendrecv(sendData_YZ,sendCount_YZ,rank_YZ(),sendtag,recvData_yz,recvCount_yz,rank_yz(),recvtag);
Comm.sendrecv(sendData_Yz,sendCount_Yz,rank_Yz(),sendtag,recvData_yZ,recvCount_yZ,rank_yZ(),recvtag);
Comm.sendrecv(sendData_yZ,sendCount_yZ,rank_yZ(),sendtag,recvData_Yz,recvCount_Yz,rank_Yz(),recvtag);
MPI_Sendrecv(sendData_x,sendCount_x,MPI_DOUBLE,rank_x(),sendtag,
recvData_X,recvCount_X,MPI_DOUBLE,rank_X(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_X,sendCount_X,MPI_DOUBLE,rank_X(),sendtag,
recvData_x,recvCount_x,MPI_DOUBLE,rank_x(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_y,sendCount_y,MPI_DOUBLE,rank_y(),sendtag,
recvData_Y,recvCount_Y,MPI_DOUBLE,rank_Y(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Y,sendCount_Y,MPI_DOUBLE,rank_Y(),sendtag,
recvData_y,recvCount_y,MPI_DOUBLE,rank_y(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_z,sendCount_z,MPI_DOUBLE,rank_z(),sendtag,
recvData_Z,recvCount_Z,MPI_DOUBLE,rank_Z(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Z,sendCount_Z,MPI_DOUBLE,rank_Z(),sendtag,
recvData_z,recvCount_z,MPI_DOUBLE,rank_z(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_xy,sendCount_xy,MPI_DOUBLE,rank_xy(),sendtag,
recvData_XY,recvCount_XY,MPI_DOUBLE,rank_XY(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_XY,sendCount_XY,MPI_DOUBLE,rank_XY(),sendtag,
recvData_xy,recvCount_xy,MPI_DOUBLE,rank_xy(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy(),sendtag,
recvData_xY,recvCount_xY,MPI_DOUBLE,rank_xY(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_xY,sendCount_xY,MPI_DOUBLE,rank_xY(),sendtag,
recvData_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_xz,sendCount_xz,MPI_DOUBLE,rank_xz(),sendtag,
recvData_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ(),sendtag,
recvData_xz,recvCount_xz,MPI_DOUBLE,rank_xz(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz(),sendtag,
recvData_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ(),sendtag,
recvData_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_yz,sendCount_yz,MPI_DOUBLE,rank_yz(),sendtag,
recvData_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ(),sendtag,
recvData_yz,recvCount_yz,MPI_DOUBLE,rank_yz(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz(),sendtag,
recvData_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendData_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ(),sendtag,
recvData_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz(),recvtag,Comm.getCommunicator(),MPI_STATUS_IGNORE);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvData_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvData_X, MeshData);

3
common/Domain.h
View File

@ -120,6 +120,8 @@ public: // Public variables (need to create accessors instead)
int BoundaryCondition;
MPI_Group Group; // Group of processors associated with this domain
//**********************************
// MPI ranks for all 18 neighbors
//**********************************
@ -189,6 +191,7 @@ private:
//......................................................................................
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z;
int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ;

33
common/MPI.I
View File

@ -579,39 +579,6 @@ inline MPI_Request MPI_CLASS::Irecv(
}
/************************************************************************
* sendrecv *
************************************************************************/
#if defined( USE_MPI ) || defined( USE_EXT_MPI )
template<>
void MPI_CLASS::sendrecv<char>( const char*, int, int, int, char*, int, int, int ) const;
template<>
void MPI_CLASS::sendrecv<int>( const int*, int, int, int, int*, int, int, int ) const;
template<>
void MPI_CLASS::sendrecv<float>( const float*, int, int, int, float*, int, int, int ) const;
template<>
void MPI_CLASS::sendrecv<double>( const double*, int, int, int, double*, int, int, int ) const;
template<class TYPE>
void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag,
TYPE *recvbuf, int recvcount, int source, int recvtag ) const
{
ERROR( "Not implimented" );
}
#else
template<class TYPE>
void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag,
TYPE *recvbuf, int recvcount, int source, int recvtag ) const
{
ASSERT( dest == 0 );
ASSERT( source == 0 );
ASSERT( sendcount == recvcount );
ASSERT( sendtag == recvtag );
memcpy( recvbuf, sendbuf, sendcount * sizeof( TYPE ) );
}
#endif
/************************************************************************
* allGather *
************************************************************************/

48
common/MPI.cpp
View File

@ -2805,54 +2805,6 @@ MPI_Request MPI_CLASS::IrecvBytes(
}
/************************************************************************
* sendrecv *
************************************************************************/
#if defined( USE_MPI ) || defined( USE_EXT_MPI )
template<>
void MPI_CLASS::sendrecv<char>( const char* sendbuf, int sendcount, int dest, int sendtag,
char* recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<char>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_CHAR, dest, sendtag,
recvbuf, recvcount, MPI_CHAR, source, recvtag,
communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<char>", profile_level );
}
template<>
void MPI_CLASS::sendrecv<int>( const int* sendbuf, int sendcount, int dest, int sendtag,
int* recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<int>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_INT, dest, sendtag,
recvbuf, recvcount, MPI_INT, source, recvtag,
communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<int>", profile_level );
}
template<>
void MPI_CLASS::sendrecv<float>( const float* sendbuf, int sendcount, int dest, int sendtag,
float* recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<float>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_FLOAT, dest, sendtag,
recvbuf, recvcount, MPI_FLOAT, source, recvtag,
communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<float>", profile_level );
}
template<>
void MPI_CLASS::sendrecv<double>( const double* sendbuf, int sendcount, int dest, int sendtag,
double* recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<double>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_DOUBLE, dest, sendtag,
recvbuf, recvcount, MPI_DOUBLE, source, recvtag,
communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<double>", profile_level );
}
#endif
/************************************************************************
* allGather *
* Note: these specializations are only called when using MPI. *

7
common/MPI.h
View File

@ -792,13 +792,6 @@ public: // Member functions
void *buf, const int N_bytes, const int send_proc, const int tag ) const;
/*!
* @brief This function sends and recieves data using a blocking call
*/
template<class type>
void sendrecv( const type *sendbuf, int sendcount, int dest, int sendtag, type *recvbuf, int recvcount, int source, int recvtag ) const;
/*!
* Each processor sends every other processor a single value.
* @param[in] x Input value for allGather

1
common/ScaLBL.h
View File

@ -201,6 +201,7 @@ private:
int sendtag,recvtag;
// Give the object it's own MPI communicator
RankInfoStruct rank_info;
MPI_Group Group; // Group of processors associated with this domain
Utilities::MPI MPI_COMM_SCALBL; // MPI Communicator for this domain
MPI_Request req1[18],req2[18];
//......................................................................................

2
common/Utilities.cpp
View File

@ -8,7 +8,7 @@
#endif
#ifdef USE_MPI
#include "common/MPI.h"
#include "mpi.h"
#endif
#include <algorithm>

538
cpu/exe/lb2_Color_mpi.cpp
View File

@ -36,11 +36,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -54,6 +58,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -110,30 +115,31 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
@ -163,7 +169,7 @@ int main(int argc, char **argv)
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -445,7 +451,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@ -582,7 +588,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -593,46 +599,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
comm.Send(&sendCount_x,1,rank_X,sendtag);
comm.Recv(&recvCount_X,1,rank_x,recvtag);
comm.Send(&sendCount_X,1,rank_x,sendtag);
comm.Recv(&recvCount_x,1,rank_X,recvtag);
comm.Send(&sendCount_y,1,rank_Y,sendtag);
comm.Recv(&recvCount_Y,1,rank_y,recvtag);
comm.Send(&sendCount_Y,1,rank_y,sendtag);
comm.Recv(&recvCount_y,1,rank_Y,recvtag);
comm.Send(&sendCount_z,1,rank_Z,sendtag);
comm.Recv(&recvCount_Z,1,rank_z,recvtag);
comm.Send(&sendCount_Z,1,rank_z,sendtag);
comm.Recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xy,1,rank_XY,sendtag);
comm.Recv(&recvCount_XY,1,rank_xy,recvtag);
comm.Send(&sendCount_XY,1,rank_xy,sendtag);
comm.Recv(&recvCount_xy,1,rank_XY,recvtag);
comm.Send(&sendCount_Xy,1,rank_xY,sendtag);
comm.Recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.Send(&sendCount_xY,1,rank_Xy,sendtag);
comm.Recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xz,1,rank_XZ,sendtag);
comm.Recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.Send(&sendCount_XZ,1,rank_xz,sendtag);
comm.Recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.Send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.Send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_yz,1,rank_YZ,sendtag);
comm.Recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.Send(&sendCount_YZ,1,rank_yz,sendtag);
comm.Recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.Send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.Send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
@ -663,48 +669,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -840,24 +846,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -890,7 +914,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
*/ //......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -970,49 +994,48 @@ int main(int argc, char **argv)
PackValues(sendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
UnpackValues(recvList_x, recvCount_x,recvbuf_x, Phi, N);
@ -1041,8 +1064,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
//.........................................
sendtag = recvtag = 5;
@ -1135,42 +1158,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//*************************************************************************
@ -1188,8 +1211,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1270,23 +1293,23 @@ int main(int argc, char **argv)
//...................................................................................
//...................................................................................
// Send all the D3Q7 distributions
req1[0] = comm.Isend(recvbuf_x, 2*recvCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(sendbuf_X, 2*sendCount_X,rank_x,recvtag);
req1[1] = comm.Isend(recvbuf_X, 2*recvCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(sendbuf_x, 2*sendCount_x,rank_X,recvtag);
req1[2] = comm.Isend(recvbuf_y, 2*recvCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y, 2*sendCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(recvbuf_Y, 2*recvCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(sendbuf_y, 2*sendCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(recvbuf_z, 2*recvCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z, 2*sendCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(recvbuf_Z, 2*recvCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(sendbuf_z, 2*sendCount_z,rank_Z,recvtag);
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
comm.waitAll(6,req1);
comm.waitAll(6,req2);
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
//...................................................................................
//...................................................................................
UnpackDenD3Q7(sendList_x,sendCount_x,sendbuf_x,2,Den,N);
@ -1322,49 +1345,48 @@ int main(int argc, char **argv)
PackValues(sendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
//...................................................................................
// Send / Recv all the phase indcator field values
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
UnpackValues(recvList_x, recvCount_x,recvbuf_x, Phi, N);
@ -1387,14 +1409,14 @@ int main(int argc, char **argv)
UnpackValues(recvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
timestep++;
//...................................................................
}
//************************************************************************/
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -1437,7 +1459,7 @@ int main(int argc, char **argv)
fwrite(ColorGrad,8,3*N,COLORGRAD);
fclose(COLORGRAD);
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

711
cpu/exe/lb2_Color_wia_mpi_bubble.cpp
View File

@ -62,7 +62,7 @@ inline void UnpackMeshData(int *list, int count, double *recvbuf, DoubleArray &V
}
}
//***************************************************************************************
inline void CommunicateMeshHalo(DoubleArray &MeshData, const Utilities::MPI& Communicator,
inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy,
double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz,
@ -111,24 +111,42 @@ inline void CommunicateMeshHalo(DoubleArray &MeshData, const Utilities::MPI& Com
PackMeshData(sendList_yZ, sendCount_yZ ,sendbuf_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData);
//......................................................................................
comm.sendrecv(sendbuf_x,sendCount_x,rank_x,sendtag,recvbuf_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendbuf_X,sendCount_X,rank_X,sendtag,recvbuf_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendbuf_y,sendCount_y,rank_y,sendtag,recvbuf_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendbuf_Y,sendCount_Y,rank_Y,sendtag,recvbuf_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendbuf_z,sendCount_z,rank_z,sendtag,recvbuf_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendbuf_Z,sendCount_Z,rank_Z,sendtag,recvbuf_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendbuf_xy,sendCount_xy,rank_xy,sendtag,recvbuf_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendbuf_XY,sendCount_XY,rank_XY,sendtag,recvbuf_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag,recvbuf_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendbuf_xY,sendCount_xY,rank_xY,sendtag,recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendbuf_xz,sendCount_xz,rank_xz,sendtag,recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag,recvbuf_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag,recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag,recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendbuf_yz,sendCount_yz,rank_yz,sendtag,recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag,recvbuf_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag,recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag,recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
recvbuf_X,recvCount_X,MPI_CHAR,rank_X,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
recvbuf_x,recvCount_x,MPI_CHAR,rank_x,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
recvbuf_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
recvbuf_y,recvCount_y,MPI_CHAR,rank_y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
recvbuf_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
recvbuf_z,recvCount_z,MPI_CHAR,rank_z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
recvbuf_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
recvbuf_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
recvbuf_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
recvbuf_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
recvbuf_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
recvbuf_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
recvbuf_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
recvbuf_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
recvbuf_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
recvbuf_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
recvbuf_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
recvbuf_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,Communicator,MPI_STATUS_IGNORE);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData);
@ -154,11 +172,15 @@ inline void CommunicateMeshHalo(DoubleArray &MeshData, const Utilities::MPI& Com
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -172,6 +194,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -260,39 +283,38 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&xIntPos,1,0);
comm.bcast(&wp_saturation,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&Restart,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&xIntPos,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
comm.bcast(&Nz,1,0);
// comm.bcast(&nBlocks,1,0);
// comm.bcast(&nthreads,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
double Ps = -(das-dbs)/(das+dbs);
@ -324,7 +346,7 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -663,14 +685,14 @@ int main(int argc, char **argv)
//.......................................................................
if (rank == 0) printf("Reading the sphere packing \n");
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
comm.barrier();
MPI_Barrier(comm);
// Broadcast the sphere packing to all processes
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//.......................................................................
// sprintf(LocalRankString,"%05d",rank);
@ -703,7 +725,7 @@ int main(int argc, char **argv)
}
}
sum_local = 1.0*sum;
porosity = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = porosity*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -837,7 +859,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -848,48 +870,89 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
req1[0] = comm.Isend(&sendCount_x,1,rank_x,sendtag);
req2[0] = comm.Irecv(&recvCount_X,1,rank_X,recvtag);
req1[1] = comm.Isend(&sendCount_X,1,rank_X,sendtag);
req2[1] = comm.Irecv(&recvCount_x,1,rank_x,recvtag);
req1[2] = comm.Isend(&sendCount_y,1,rank_y,sendtag);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_Y,recvtag);
req1[3] = comm.Isend(&sendCount_Y,1,rank_Y,sendtag);
req2[3] = comm.Irecv(&recvCount_y,1,rank_y,recvtag);
req1[4] = comm.Isend(&sendCount_z,1,rank_z,sendtag);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_Z,recvtag);
req1[5] = comm.Isend(&sendCount_Z,1,rank_Z,sendtag);
req2[5] = comm.Irecv(&recvCount_z,1,rank_z,recvtag);
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_xy,sendtag);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_XY,recvtag);
req1[7] = comm.Isend(&sendCount_XY,1,rank_XY,sendtag);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_xy,recvtag);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_Xy,sendtag);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_xY,recvtag);
req1[9] = comm.Isend(&sendCount_xY,1,rank_xY,sendtag);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_Xy,recvtag);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_xz,sendtag);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_XZ,recvtag);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_XZ,sendtag);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_xz,recvtag);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_Xz,sendtag);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_xZ,recvtag);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_xZ,sendtag);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_Xz,recvtag);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(&sendCount_yz,1,rank_yz,sendtag);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_YZ,recvtag);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_YZ,sendtag);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_yz,recvtag);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_Yz,sendtag);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_yZ,recvtag);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_yZ,sendtag);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_Yz,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
*/ //**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@ -919,48 +982,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendList_Xz,endCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -1075,7 +1138,7 @@ int main(int argc, char **argv)
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
dvc_AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
//......................................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
dvc_CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
dvc_CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
@ -1182,24 +1245,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_x,sendtag,recvID_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_X,sendtag,recvID_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_y,sendtag,recvID_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_Y,sendtag,recvID_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_z,sendtag,recvID_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_Z,sendtag,recvID_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_xy,sendtag,recvID_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_XY,sendtag,recvID_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_Xy,sendtag,recvID_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_xY,sendtag,recvID_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_xz,sendtag,recvID_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_XZ,sendtag,recvID_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_Xz,sendtag,recvID_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_xZ,sendtag,recvID_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_yz,sendtag,recvID_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_YZ,sendtag,recvID_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_Yz,sendtag,recvID_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_yZ,sendtag,recvID_Yz,recvCount_Yz,rank_Yz,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1267,7 +1348,7 @@ int main(int argc, char **argv)
recvMeshData_YZ = new double [recvCount_YZ];
recvMeshData_XZ = new double [recvCount_XZ];
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1447,8 +1528,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
//.........................................
//...........................................................................
// MAIN VARIABLES INITIALIZED HERE
@ -1519,7 +1600,7 @@ int main(int argc, char **argv)
dvc_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
dvc_CopyToDevice(Den,cDen,2*N*sizeof(double));
dvc_Barrier();
comm.barrier();
MPI_Barrier(comm);
}
// Pack the buffers (zeros out the halo region)
dvc_PackDenD3Q7(dvcRecvList_x,recvCount_x,recvbuf_x,2,Den,N);
@ -1558,48 +1639,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
dvc_Barrier();
//...................................................................................
//...................................................................................
@ -1645,7 +1726,7 @@ int main(int argc, char **argv)
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double));
dvc_CopyToHost(Press.data,Pressure,N*sizeof(double));
dvc_CopyToHost(Vel,Velocity,3*N*sizeof(double));
comm.barrier();
MPI_Barrier(comm);
//...........................................................................
timestep=0;
@ -1741,42 +1822,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
//...................................................................................
//*************************************************************************
@ -1793,8 +1874,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
@ -1877,23 +1958,23 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the D3Q7 distributions
req1[0] = comm.Isend(recvbuf_x,2*recvCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(sendbuf_X,2*sendCount_X,rank_X,recvtag);
req1[1] = comm.Isend(recvbuf_X,2*recvCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(sendbuf_x,2*sendCount_x,rank_x,recvtag);
req1[2] = comm.Isend(recvbuf_y,2*recvCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y,2*sendCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(recvbuf_Y,2*recvCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(sendbuf_y,2*sendCount_y,rank_y,recvtag);
req1[4] = comm.Isend(recvbuf_z,2*recvCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z,2*sendCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(recvbuf_Z,2*recvCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(sendbuf_z,2*sendCount_z,rank_z,recvtag);
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
comm.waitAll(6,req1);
comm.waitAll(6,req2);
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
//...................................................................................
//...................................................................................
dvc_UnpackDenD3Q7(dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
@ -1932,48 +2013,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
dvc_Barrier();
//...................................................................................
//...................................................................................
@ -2003,7 +2084,7 @@ int main(int argc, char **argv)
dvc_UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
timestep++;
@ -2283,27 +2364,27 @@ int main(int argc, char **argv)
//...........................................................................
}
//...........................................................................
comm.barrier();
nwp_volume_global = comm.sumReduce( nwp_volume );
awn_global = comm.sumReduce( awn );
ans_global = comm.sumReduce( ans );
aws_global = comm.sumReduce( aws );
lwns_global = comm.sumReduce( lwns );
As_global = comm.sumReduce( As );
Jwn_global = comm.sumReduce( Jwn );
efawns_global = comm.sumReduce( efawns );
MPI_Barrier(comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
// Phase averages
vol_w_global = comm.sumReduce( vol_w );
vol_n_global = comm.sumReduce( vol_n );
paw_global = comm.sumReduce( paw );
pan_global = comm.sumReduce( pan );
vaw_global(0) = comm.sumReduce( vaw(0) );
van_global(0) = comm.sumReduce( van(0) );
vawn_global(0) = comm.sumReduce( vawn(0) );
Gwn_global(0) = comm.sumReduce( Gwn(0) );
Gns_global(0) = comm.sumReduce( Gns(0) );
Gws_global(0) = comm.sumReduce( Gws(0) );
comm.barrier();
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Barrier(comm);
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
// See McClure, Prins and Miller (2013)
@ -2370,8 +2451,8 @@ int main(int argc, char **argv)
}
//************************************************************************/
dvc_Barrier();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
@ -2408,7 +2489,7 @@ int main(int argc, char **argv)
*/ //************************************************************************/
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

348
gpu/exe/lb1_MRT_mpi.cpp
View File

@ -6,7 +6,7 @@
#include <iostream>
#include <fstream>
#include <string.h>
#include "common/MPI.h"
#include <mpi.h>
#include <stdlib.h>
using namespace std;
@ -64,11 +64,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -82,6 +86,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//**********************************
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@ -131,23 +136,24 @@ int main(int argc, char **argv)
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
double rlx_setA = 1.f/tau;
@ -170,7 +176,7 @@ int main(int argc, char **argv)
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -451,7 +457,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@ -588,7 +594,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -599,46 +605,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
comm.send(&sendCount_x,1,rank_X,sendtag);
comm.recv(&recvCount_X,1,rank_x,recvtag);
comm.send(&sendCount_X,1,rank_x,sendtag);
comm.recv(&recvCount_x,1,rank_X,recvtag);
comm.send(&sendCount_y,1,rank_Y,sendtag);
comm.recv(&recvCount_Y,1,rank_y,recvtag);
comm.send(&sendCount_Y,1,rank_y,sendtag);
comm.recv(&recvCount_y,1,rank_Y,recvtag);
comm.send(&sendCount_z,1,rank_Z,sendtag);
comm.recv(&recvCount_Z,1,rank_z,recvtag);
comm.send(&sendCount_Z,1,rank_z,sendtag);
comm.recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xy,1,rank_XY,sendtag);
comm.recv(&recvCount_XY,1,rank_xy,recvtag);
comm.send(&sendCount_XY,1,rank_xy,sendtag);
comm.recv(&recvCount_xy,1,rank_XY,recvtag);
comm.send(&sendCount_Xy,1,rank_xY,sendtag);
comm.recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.send(&sendCount_xY,1,rank_Xy,sendtag);
comm.recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xz,1,rank_XZ,sendtag);
comm.recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.send(&sendCount_XZ,1,rank_xz,sendtag);
comm.recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_yz,1,rank_YZ,sendtag);
comm.recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.send(&sendCount_YZ,1,rank_yz,sendtag);
comm.recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
@ -669,48 +675,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
@ -909,24 +915,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -959,7 +983,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
//......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -999,8 +1023,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@ -1112,48 +1136,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1236,7 +1260,7 @@ int main(int argc, char **argv)
//*****************************************************************************
//*****************************************************************************
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
timestep++;
//...................................................................
@ -1245,8 +1269,8 @@ int main(int argc, char **argv)
// cudaThreadSynchronize();
dvc_Barrier();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -1280,7 +1304,7 @@ int main(int argc, char **argv)
// dvc_CopyToDevice(velocity, vel, 3*dist_mem_size, dvc_CopyToDeviceDeviceToHost);
//..............................................................................
// cudaThreadSynchronize();
// comm.barrier();
// MPI_Barrier(comm);
//............................................................
//....Write the z-velocity to test poiseuille flow............
// double vz,vz_avg;
@ -1309,7 +1333,7 @@ int main(int argc, char **argv)
// free (velocity); free(id);
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

352
gpu/exe/lb1_MRT_mpi.cu
View File

@ -1,10 +1,8 @@
#include "common/MPI.h"
#include <stdio.h>
#include <iostream>
#include <fstream>
#include <cuda.h>
#include <mpi.h>
inline void PackID(int *list, int count, char *sendbuf, char *ID){
// Fill in the phase ID values from neighboring processors
@ -555,11 +553,15 @@ void Write_Out(double *array, int Nx, int Ny, int Nz){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -573,6 +575,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//**********************************
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@ -622,21 +625,24 @@ int main(int argc, char **argv)
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&iterMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&iterMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
double rlx_setA = 1.f/tau;
@ -659,7 +665,7 @@ int main(int argc, char **argv)
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -940,7 +946,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@ -1077,7 +1083,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -1088,46 +1094,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
comm.send(&sendCount_x,1,rank_X,sendtag);
comm.recv(&recvCount_X,1,rank_x,recvtag);
comm.send(&sendCount_X,1,rank_x,sendtag);
comm.recv(&recvCount_x,1,rank_X,recvtag);
comm.send(&sendCount_y,1,rank_Y,sendtag);
comm.recv(&recvCount_Y,1,rank_y,recvtag);
comm.send(&sendCount_Y,1,rank_y,sendtag);
comm.recv(&recvCount_y,1,rank_Y,recvtag);
comm.send(&sendCount_z,1,rank_Z,sendtag);
comm.recv(&recvCount_Z,1,rank_z,recvtag);
comm.send(&sendCount_Z,1,rank_z,sendtag);
comm.recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xy,1,rank_XY,sendtag);
comm.recv(&recvCount_XY,1,rank_xy,recvtag);
comm.send(&sendCount_XY,1,rank_xy,sendtag);
comm.recv(&recvCount_xy,1,rank_XY,recvtag);
comm.send(&sendCount_Xy,1,rank_xY,sendtag);
comm.recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.send(&sendCount_xY,1,rank_Xy,sendtag);
comm.recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xz,1,rank_XZ,sendtag);
comm.recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.send(&sendCount_XZ,1,rank_xz,sendtag);
comm.recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_yz,1,rank_YZ,sendtag);
comm.recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.send(&sendCount_YZ,1,rank_yz,sendtag);
comm.recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
//**********************************************************************************
//recvCount_x = sendCount_x;
//recvCount_X = sendCount_X;
@ -1151,7 +1157,7 @@ int main(int argc, char **argv)
//......................................................................................
// Use MPI to fill in the appropriate values
// int tag = 5;
// Mcomm.sendrecv(sendCount_x,1,rank_x,tag,sendCount_X,1);
// MPI_Sendrecv(sendCount_x,1,MPI_INT,rank_x,tag,sendCount_X,1,MPI_INT,comm,req);
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@ -1181,48 +1187,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
@ -1421,24 +1427,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1471,7 +1495,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
//......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1511,8 +1535,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@ -1609,48 +1633,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1734,7 +1758,7 @@ int main(int argc, char **argv)
//*****************************************************************************
//*****************************************************************************
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
iter++;
//...................................................................
@ -1742,8 +1766,8 @@ int main(int argc, char **argv)
//************************************************************************/
cudaThreadSynchronize();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*iter)/cputime/1000000 << " MLUPS" << endl;
@ -1778,7 +1802,7 @@ int main(int argc, char **argv)
cudaMemcpy(velocity, vel, 3*dist_mem_size, cudaMemcpyDeviceToHost);
//..............................................................................
cudaThreadSynchronize();
comm.barrier();
MPI_Barrier(comm);
//............................................................
//....Write the z-velocity to test poiseuille flow............
double vz,vz_avg;
@ -1807,7 +1831,7 @@ int main(int argc, char **argv)
free (velocity); free(id);
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

65
gpu/exe/lb2_Color.cu
View File

@ -1,4 +1,6 @@
#include "common/MPI.h"
#ifdef useMPI
#include <mpi.h>
#endif
#include <stdio.h>
#include <stdlib.h>
@ -60,10 +62,18 @@ int main(int argc, char *argv[])
{
//********** Initialize MPI ****************
int numprocs,rank;
#ifdef useMPI
MPI_Status stat;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int numprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_size(comm,&numprocs);
MPI_Comm_rank(comm,&rank);
#else
MPI_Comm comm = MPI_COMM_WORLD;
numprocs = 1;
rank = 0;
#endif
//******************************************
if (rank == 0){
@ -113,31 +123,32 @@ int main(int argc, char *argv[])
input >> tol; // error tolerance
//.............................................................
}
#ifdef useMPI
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
#endif
double rlxA = 1.f/tau;
double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA);
@ -232,7 +243,11 @@ int main(int argc, char *argv[])
if (k==4) k=Nz-5;
}
}
comm.bcast(&id[0],N,0);
#ifdef useMPI //............................................................
MPI_Barrier(comm);
MPI_Bcast(&id[0],N,MPI_CHAR,0,comm);
MPI_Barrier(comm);
#endif
if (rank == 0) printf("Domain set.\n");
//...........................................................................

541
gpu/exe/lb2_Color_mpi.cpp
View File

@ -2,7 +2,7 @@
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include "common/MPI.h"
#include <mpi.h>
using namespace std;
@ -98,11 +98,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -116,6 +120,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -172,30 +177,31 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
@ -225,7 +231,7 @@ int main(int argc, char **argv)
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -507,7 +513,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@ -644,7 +650,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -655,46 +661,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
comm.Send(&sendCount_x,1,rank_X,sendtag);
comm.Recv(&recvCount_X,1,rank_x,recvtag);
comm.Send(&sendCount_X,1,rank_x,sendtag);
comm.Recv(&recvCount_x,1,rank_X,recvtag);
comm.Send(&sendCount_y,1,rank_Y,sendtag);
comm.Recv(&recvCount_Y,1,rank_y,recvtag);
comm.Send(&sendCount_Y,1,rank_y,sendtag);
comm.Recv(&recvCount_y,1,rank_Y,recvtag);
comm.Send(&sendCount_z,1,rank_Z,sendtag);
comm.Recv(&recvCount_Z,1,rank_z,recvtag);
comm.Send(&sendCount_Z,1,rank_z,sendtag);
comm.Recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xy,1,rank_XY,sendtag);
comm.Recv(&recvCount_XY,1,rank_xy,recvtag);
comm.Send(&sendCount_XY,1,rank_xy,sendtag);
comm.Recv(&recvCount_xy,1,rank_XY,recvtag);
comm.Send(&sendCount_Xy,1,rank_xY,sendtag);
comm.Recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.Send(&sendCount_xY,1,rank_Xy,sendtag);
comm.Recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xz,1,rank_XZ,sendtag);
comm.Recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.Send(&sendCount_XZ,1,rank_xz,sendtag);
comm.Recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.Send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.Send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_yz,1,rank_YZ,sendtag);
comm.Recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.Send(&sendCount_YZ,1,rank_yz,sendtag);
comm.Recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.Send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.Send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
@ -725,48 +731,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -987,24 +993,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1037,7 +1061,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
*/ //......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1102,49 +1126,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@ -1184,8 +1207,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@ -1286,42 +1309,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//*************************************************************************
@ -1340,8 +1363,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1423,23 +1446,23 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the D3Q7 distributions
req1[0] = comm.Isend(recvbuf_x, 2*recvCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(sendbuf_X, 2*sendCount_X,rank_x,recvtag);
req1[1] = comm.Isend(recvbuf_X, 2*recvCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(sendbuf_x, 2*sendCount_x,rank_X,recvtag);
req1[2] = comm.Isend(recvbuf_y, 2*recvCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y, 2*sendCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(recvbuf_Y, 2*recvCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(sendbuf_y, 2*sendCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(recvbuf_z, 2*recvCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z, 2*sendCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(recvbuf_Z, 2*recvCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(sendbuf_z, 2*sendCount_z,rank_Z,recvtag);
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
comm.waitAll(6,req1);
comm.waitAll(6,req2);
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
//...................................................................................
//...................................................................................
dvc_UnpackDenD3Q7(faceGrid,packThreads,dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
@ -1484,48 +1507,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@ -1554,7 +1577,7 @@ int main(int argc, char **argv)
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
timestep++;
@ -1564,8 +1587,8 @@ int main(int argc, char **argv)
// cudaThreadSynchronize();
dvc_Barrier();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -1593,7 +1616,7 @@ int main(int argc, char **argv)
//************************************************************************/
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

621
gpu/exe/lb2_Color_pBC_wia_mpi.cpp
View File

@ -2,7 +2,7 @@
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include "common/MPI.h"
#include <mpi.h>
#include "pmmc.h"
#include "Domain.h"
@ -101,11 +101,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@ -119,6 +123,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -198,35 +203,35 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
double Ps = -(das-dbs)/(das+dbs);
@ -258,7 +263,7 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@ -556,14 +561,14 @@ int main(int argc, char **argv)
//.......................................................................
if (rank == 0) printf("Reading the sphere packing \n");
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
comm.barrier();
MPI_Barrier(comm);
// Broadcast the sphere packing to all processes
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//.......................................................................
// sprintf(LocalRankString,"%05d",rank);
@ -713,7 +718,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@ -724,49 +729,89 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
req1[0] = comm.Isend(&sendCount_x,1,rank_X,sendtag);
req2[0] = comm.Irecv(&recvCount_X,1,rank_x,recvtag);
req1[1] = comm.Isend(&sendCount_X,1,rank_x,sendtag);
req2[1] = comm.Irecv(&recvCount_x,1,rank_X,recvtag);
req1[2] = comm.Isend(&sendCount_y,1,rank_Y,sendtag);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_y,recvtag);
req1[3] = comm.Isend(&sendCount_Y,1,rank_y,sendtag);
req2[3] = comm.Irecv(&recvCount_y,1,rank_Y,recvtag);
req1[4] = comm.Isend(&sendCount_z,1,rank_Z,sendtag);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_z,recvtag);
req1[5] = comm.Isend(&sendCount_Z,1,rank_z,sendtag);
req2[5] = comm.Irecv(&recvCount_z,1,rank_Z,recvtag);
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_XY,sendtag);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_xy,recvtag);
req1[7] = comm.Isend(&sendCount_XY,1,rank_xy,sendtag);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_XY,recvtag);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_xY,sendtag);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_Xy,recvtag);
req1[9] = comm.Isend(&sendCount_xY,1,rank_Xy,sendtag);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_XZ,sendtag);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_xz,recvtag);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_xz,sendtag);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_XZ,recvtag);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_xZ,sendtag);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_Xz,recvtag);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_Xz,sendtag);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(&sendCount_yz,1,rank_YZ,sendtag);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_yz,recvtag);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_yz,sendtag);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_YZ,recvtag);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_yZ,sendtag);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_Yz,recvtag);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_Yz,sendtag);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//**********************************************************************************
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
*/ //**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@ -796,48 +841,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@ -1058,24 +1103,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1108,7 +1171,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
*/ //......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@ -1259,48 +1322,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@ -1340,8 +1403,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
//.........................................
sendtag = recvtag = 5;
@ -1431,42 +1494,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//*************************************************************************
@ -1484,8 +1547,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1567,23 +1630,23 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the D3Q7 distributions
req1[0] = comm.Isend(recvbuf_x,2*recvCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(sendbuf_X,2*sendCount_X,rank_x,recvtag);
req1[1] = comm.Isend(recvbuf_X,2*recvCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(sendbuf_x,2*sendCount_x,rank_X,recvtag);
req1[2] = comm.Isend(recvbuf_y,2*recvCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y,2*sendCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(recvbuf_Y,2*recvCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(sendbuf_y,2*sendCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(recvbuf_z,2*recvCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z,2*sendCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(recvbuf_Z,2*recvCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(sendbuf_z,2*sendCount_z,rank_Z,recvtag);
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
comm.waitAll(6,req1);
comm.waitAll(6,req2);
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
//...................................................................................
//...................................................................................
dvc_UnpackDenD3Q7(faceGrid,packThreads,dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
@ -1622,48 +1685,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@ -1692,7 +1755,7 @@ int main(int argc, char **argv)
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
timestep++;
//...................................................................
@ -1703,7 +1766,7 @@ int main(int argc, char **argv)
//...........................................................................
dvc_Barrier();
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double));
comm.barrier();
MPI_Barrier(comm);
//...........................................................................
// Compute areas using porous medium marching cubes algorithm
// McClure, Adalsteinsson, et al. (2007)
@ -1872,15 +1935,15 @@ int main(int argc, char **argv)
//*******************************************************************
}
//...........................................................................
comm.barrier();
nwp_volume_global = comm.sumReduce( nwp_volume );
awn_global = comm.sumReduce( awn );
ans_global = comm.sumReduce( ans );
aws_global = comm.sumReduce( aws );
lwns_global = comm.sumReduce( lwns );
As_global = comm.sumReduce( As );
MPI_Barrier(comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
comm.barrier();
MPI_Barrier(comm);
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
// See McClure, Prins and Miller (2013)
@ -1909,8 +1972,8 @@ int main(int argc, char **argv)
}
//************************************************************************/
dvc_Barrier();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
@ -1946,7 +2009,7 @@ int main(int argc, char **argv)
*/ //************************************************************************/
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

8
models/ColorModel.cpp
View File

@ -192,12 +192,12 @@ void ScaLBL_ColorModel::ReadInput(){
}
else if (domain_db->keyExists( "GridFile" )){
// Read the local domain data
auto input_id = readMicroCT( *domain_db, comm );
auto input_id = readMicroCT( *domain_db, MPI_COMM_WORLD );
// Fill the halo (assuming GCW of 1)
array<int,3> size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) };
ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz };
ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 );
fillHalo<signed char> fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 );
fillHalo<signed char> fill( MPI_COMM_WORLD, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 );
Array<signed char> id_view;
id_view.viewRaw( size1, Mask->id );
fill.copy( input_id, id_view );
@ -652,7 +652,7 @@ void ScaLBL_ColorModel::Run(){
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
@ -991,7 +991,7 @@ void ScaLBL_ColorModel::Run(){
//************************************************************************
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

4
models/DFHModel.cpp
View File

@ -487,7 +487,7 @@ void ScaLBL_DFHModel::Run(){
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
@ -583,7 +583,7 @@ void ScaLBL_DFHModel::Run(){
//************************************************************************
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

4
models/MRTModel.cpp
View File

@ -227,7 +227,7 @@ void ScaLBL_MRTModel::Run(){
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax);
if (rank==0) printf("********************************************************\n");
timestep=0;
@ -325,7 +325,7 @@ void ScaLBL_MRTModel::Run(){
}
}
//************************************************************************/
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

22
tests/BlobAnalyzeParallel.cpp
View File

@ -138,16 +138,16 @@ int main(int argc, char **argv)
}
comm.barrier();
// Computational domain
comm.bcast(&nx,1,0);
comm.bcast(&ny,1,0);
comm.bcast(&nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&nx,1,MPI_INT,0,comm);
MPI_Bcast(&ny,1,MPI_INT,0,comm);
MPI_Bcast(&nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -291,7 +291,7 @@ int main(int argc, char **argv)
}
Dm.CommInit(); // Initialize communications for domains
sum_global = comm.sumReduce( sum );
MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = sum_global/Dm.Volume;
if (rank==0) printf("Porosity = %f \n",porosity);

54
tests/GenerateSphereTest.cpp
View File

@ -213,24 +213,42 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
//......................................................................................
Dm.Comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag);
Dm.Comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag);
Dm.Comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag);
Dm.Comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag);
Dm.Comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag);
Dm.Comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag);
Dm.Comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag);
Dm.Comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag);
Dm.Comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag);
Dm.Comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag);
Dm.Comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag);
Dm.Comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag);
Dm.Comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag);
Dm.Comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag);
Dm.Comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag);
Dm.Comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag);
Dm.Comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag);
Dm.Comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag);
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);

28
tests/TestBlobAnalyze.cpp
View File

@ -190,16 +190,16 @@ int main(int argc, char **argv)
}
comm.barrier();
// Computational domain
comm.bcast(&nx,1,0);
comm.bcast(&ny,1,0);
comm.bcast(&nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&nx,1,MPI_INT,0,comm);
MPI_Bcast(&ny,1,MPI_INT,0,comm);
MPI_Bcast(&nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -255,10 +255,10 @@ int main(int argc, char **argv)
comm.barrier();
// Broadcast the sphere packing to all processes
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
//...........................................................................
comm.barrier();
//.......................................................................

41
tests/TestBubble.cpp
View File

@ -45,6 +45,7 @@ int main(int argc, char **argv)
int nprocx,nprocy,nprocz;
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -433,7 +434,7 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
//...........................................................................
// MAIN VARIABLES INITIALIZED HERE
@ -808,25 +809,25 @@ int main(int argc, char **argv)
}
//...........................................................................
comm.barrier();
nwp_volume_global = comm.sumReduce( nwp_volume );
awn_global = comm.sumReduce( awn );
ans_global = comm.sumReduce( ans );
aws_global = comm.sumReduce( aws );
lwns_global = comm.sumReduce( lwns );
As_global = comm.sumReduce( As );
Jwn_global = comm.sumReduce( Jwn );
efawns_global = comm.sumReduce( efawns );
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
// Phase averages
vol_w_global = comm.sumReduce( vol_w );
vol_n_global = comm.sumReduce( vol_n );
paw_global = comm.sumReduce( paw );
pan_global = comm.sumReduce( pan );
vaw_global(0) = comm.sumReduce( vaw(0) );
van_global(0) = comm.sumReduce( van(0) );
vawn_global(0) = comm.sumReduce( vawn(0) );
Gwn_global(0) = comm.sumReduce( Gwn(0) );
Gns_global(0) = comm.sumReduce( Gns(0) );
Gws_global(0) = comm.sumReduce( Gws(0) );
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
comm.barrier();
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
@ -951,7 +952,7 @@ int main(int argc, char **argv)
//************************************************************************/
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

4
tests/TestBubbleDFH.cpp
View File

@ -387,7 +387,7 @@ int main(int argc, char **argv)
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
err = 1.0;
@ -487,7 +487,7 @@ int main(int argc, char **argv)
//************************************************************************
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

20
tests/TestColorGrad.cpp
View File

@ -114,16 +114,16 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
//.................................................
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
// **************************************************************

4
tests/TestCommD3Q19.cpp
View File

@ -378,7 +378,7 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
@ -403,7 +403,7 @@ int main(int argc, char **argv)
//...................................................................
}
//************************************************************************/
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;

4
tests/TestForceD3Q19.cpp
View File

@ -450,7 +450,7 @@ int main (int argc, char **argv)
for (int i=0; i<nprocs; i++) {
if ( rank==i )
printf("%i of %i: Testing force term \n",rank,nprocs);
comm.barrier();
MPI_Barrier(MPI_COMM_WORLD);
}
// Create a memory leak for valgrind to find
@ -540,6 +540,6 @@ int main (int argc, char **argv)
printf("Fy = %f; Computed vy=%f \n",Fy,vel[Np+0]);
printf("Fz = %f; Computed vz=%f \n",Fz,vel[2*Np+0]);
comm.barrier();
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
}

4
tests/TestForceMoments.cpp
View File

@ -210,7 +210,7 @@ int main(int argc, char **argv)
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
/************ MAIN ITERATION LOOP (timing communications)***************************************/
//ScaLBL_Comm->SendD3Q19(dist, &dist[10*Np]);
@ -244,7 +244,7 @@ int main(int argc, char **argv)
//************************************************************************/
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;

28
tests/TestMRT.cpp
View File

@ -580,16 +580,16 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
//.................................................
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
// **************************************************************
@ -668,7 +668,7 @@ int main(int argc, char **argv)
}
}
comm.barrier();
sum = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = sum*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -731,7 +731,7 @@ int main(int argc, char **argv)
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
while (timestep < timesteps) {
@ -752,7 +752,7 @@ int main(int argc, char **argv)
}
//************************************************************************/
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@ -795,7 +795,7 @@ int main(int argc, char **argv)
}
}
}
sum = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
double PoreVel = sum*iVol_global;
if (rank==0) printf("Velocity = %f \n",PoreVel);

1
tests/TestMicroCTReader.cpp
View File

@ -62,6 +62,7 @@ int main(int argc, char **argv)
int N_errors = ut.NumFailGlobal();
// Close MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return N_errors;
}

2
tests/TestMomentsD3Q19.cpp
View File

@ -539,7 +539,7 @@ int main (int argc, char **argv)
error=count;
// Finished
comm.barrier();
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return error;
}

2
tests/TestNetcdf.cpp
View File

@ -116,7 +116,7 @@ int main(int argc, char **argv)
PROFILE_SAVE("TestNetcdf");
// Close MPI
comm.barrier();
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return N_errors;
}

4
tests/TestSegDist.cpp
View File

@ -100,10 +100,10 @@ int main(int argc, char **argv)
comm.barrier();
if (rank==0) printf("Initialized! Converting to Signed Distance function \n");
double t1 = Utilities::MPI::time();
double t1 = MPI_Wtime();
DoubleArray Distance(nx,ny,nz);
CalcDist(Distance,id,Dm,{false,false,false});
double t2 = Utilities::MPI::time();
double t2 = MPI_Wtime();
if (rank==0)
printf("Total time: %f seconds \n",t2-t1);

30
tests/lb2_CMT_wia.cpp
View File

@ -292,18 +292,18 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,2*sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X,2*recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X,2*sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x,2*recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y,2*sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,2*recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,2*sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,2*recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z,2*sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,2*recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,2*sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,2*recvCount_z,rank_z,recvtag);
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
*/ //...................................................................................
ScaLBL_D3Q7_Swap(ID, &packed_even[0], &packed_odd[0], Nx, Ny, Nz);
@ -311,8 +311,8 @@ int main(int argc, char **argv)
/* //...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(6,req1);
comm.waitAll(6,req2);
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -358,7 +358,7 @@ int main(int argc, char **argv)
fclose(PHASE);
// Close MPI
comm.barrier();
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return 0;
}

427
tests/lb2_Color_blob_wia_mpi.cpp
View File

@ -114,6 +114,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -206,36 +207,36 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
//.................................................
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&phi_s,1,0);
comm.bcast(&wp_saturation,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&Restart,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
// comm.bcast(&nBlocks,1,0);
// comm.bcast(&nthreads,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -398,10 +399,10 @@ int main(int argc, char **argv)
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
comm.barrier();
// Broadcast the sphere packing to all processes
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
//...........................................................................
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
@ -417,7 +418,7 @@ int main(int argc, char **argv)
D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx;
printf("Sauter Mean Diameter (computed from sphere packing) = %f \n ",D);
}
comm.bcast(&D,1,0);
MPI_Bcast(&D,1,MPI_DOUBLE,0,comm);
//.......................................................................
// sprintf(LocalRankString,"%05d",rank);
@ -477,7 +478,7 @@ int main(int argc, char **argv)
id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0;
//.........................................................
sum_local = 1.0*sum;
porosity = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = porosity*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -885,24 +886,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_x,sendtag,recvID_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_X,sendtag,recvID_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_y,sendtag,recvID_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_Y,sendtag,recvID_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_z,sendtag,recvID_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_Z,sendtag,recvID_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_xy,sendtag,recvID_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_XY,sendtag,recvID_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_Xy,sendtag,recvID_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_xY,sendtag,recvID_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_xz,sendtag,recvID_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_XZ,sendtag,recvID_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_Xz,sendtag,recvID_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_xZ,sendtag,recvID_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_yz,sendtag,recvID_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_YZ,sendtag,recvID_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_Yz,sendtag,recvID_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_yZ,sendtag,recvID_Yz,recvCount_Yz,rank_Yz,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@ -1361,48 +1380,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
ScaLBL_DeviceBarrier();
//...................................................................................
//...................................................................................
@ -1478,7 +1497,7 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
sendtag = recvtag = 5;
@ -1574,42 +1593,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x, 5*sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X, 5*recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X, 5*sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x, 5*recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y, 5*sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, 5*recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, 5*sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, 5*recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z, 5*sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, 5*recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, 5*sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, 5*recvCount_z,rank_z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
//...................................................................................
//*************************************************************************
@ -1629,8 +1648,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
@ -1724,18 +1743,18 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x, 2*sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvbuf_X, 2*recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendbuf_X, 2*sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvbuf_x, 2*recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendbuf_y, 2*sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, 2*recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, 2*sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, 2*recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendbuf_z, 2*sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, 2*recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, 2*sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, 2*recvCount_z,rank_z,recvtag);
MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
//...................................................................................
ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz);
@ -1743,8 +1762,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(6,req1);
comm.waitAll(6,req2);
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@ -1805,48 +1824,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_x,sendtag,comm,&req1[0]);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag,comm,&req2[0]);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_X,sendtag,comm,&req1[1]);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag,comm,&req2[1]);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_y,sendtag,comm,&req1[2]);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag,comm,&req2[2]);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag,comm,&req1[3]);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag,comm,&req2[3]);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_z,sendtag,comm,&req1[4]);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag,comm,&req2[4]);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag,comm,&req1[5]);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag,comm,&req1[6]);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag,comm,&req2[6]);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag,comm,&req1[7]);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag,comm,&req2[7]);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag,comm,&req1[8]);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag,comm,&req2[8]);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag,comm,&req1[9]);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag,comm,&req1[10]);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag,comm,&req2[10]);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag,comm,&req1[11]);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag,comm,&req2[11]);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag,comm,&req1[12]);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag,comm,&req2[12]);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag,comm,&req1[13]);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag,comm,&req1[14]);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag,comm,&req2[14]);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag,comm,&req1[15]);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag,comm,&req2[15]);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag,comm,&req1[16]);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag,comm,&req2[16]);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag,comm,&req1[17]);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag,comm,&req2[17]);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
ScaLBL_DeviceBarrier();
//...................................................................................
//...................................................................................
@ -2423,28 +2442,28 @@ int main(int argc, char **argv)
//...........................................................................
comm.barrier();
nwp_volume_global = comm.sumReduce( nwp_volume );
awn_global = comm.sumReduce( awn );
ans_global = comm.sumReduce( ans );
aws_global = comm.sumReduce( aws );
lwns_global = comm.sumReduce( lwns );
As_global = comm.sumReduce( As );
Jwn_global = comm.sumReduce( Jwn );
Kwn_global = comm.sumReduce( Kwn );
efawns_global = comm.sumReduce( efawns );
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
// Phase averages
vol_w_global = comm.sumReduce( vol_w );
vol_n_global = comm.sumReduce( vol_n );
paw_global = comm.sumReduce( paw );
pan_global = comm.sumReduce( pan );
vaw_global(0) = comm.sumReduce( vaw(0) );
van_global(0) = comm.sumReduce( van(0) );
vawn_global(0) = comm.sumReduce( vawn(0) );
Gwn_global(0) = comm.sumReduce( Gwn(0) );
Gns_global(0) = comm.sumReduce( Gns(0) );
Gws_global(0) = comm.sumReduce( Gws(0) );
trawn_global = comm.sumReduce( trawn );
trJwn_global = comm.sumReduce( trJwn );
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
comm.barrier();
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
@ -2670,7 +2689,7 @@ int main(int argc, char **argv)
//************************************************************************/
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

48
tests/lbpm_BGK_simulator.cpp
View File

@ -97,28 +97,28 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
//.................................................
comm.bcast(&tau,1,0);
//comm.bcast(&pBC,1,0);
//comm.bcast(&Restart,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
//MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
// MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
//comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -249,7 +249,7 @@ int main(int argc, char **argv)
}
}
}
sum = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = sum*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -331,7 +331,7 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
double D32,Fo,Re,velocity,err1D,mag_force,vel_prev;
@ -410,7 +410,7 @@ int main(int argc, char **argv)
//************************************************************************/
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

61
tests/lbpm_color_macro_simulator.cpp
View File

@ -39,6 +39,9 @@ int main(int argc, char **argv)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
printf("Running Color LBM \n");
@ -169,32 +172,32 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
//.................................................
comm.bcast(&tauA,1,0);
comm.bcast(&tauB,1,0);
comm.bcast(&rhoA,1,0);
comm.bcast(&rhoB,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&BoundaryCondition,1,0);
comm.bcast(&InitialCondition,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&RESTART_INTERVAL,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&tauA,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tauB,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&rhoA,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&rhoB,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm);
MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&RESTART_INTERVAL,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
flux = 0.f;
@ -319,7 +322,7 @@ int main(int argc, char **argv)
timestep=0;
}
}
comm.bcast(&timestep,1,0);
MPI_Bcast(&timestep,1,MPI_INT,0,comm);
FILE *RESTART = fopen(LocalRestartFile,"rb");
if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx");
readID=fread(id,1,N,RESTART);
@ -358,7 +361,7 @@ int main(int argc, char **argv)
}
}
}
sum - comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = sum*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
//.........................................................
@ -534,7 +537,7 @@ int main(int argc, char **argv)
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
err = 1.0;
@ -634,7 +637,7 @@ int main(int argc, char **argv)
//************************************************************************
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

34
tests/lbpm_disc_pp.cpp
View File

@ -9,7 +9,7 @@
#include "analysis/pmmc.h"
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/MPI.h"
#include "common/MPI.h" // This includes mpi.h
#include "common/SpherePack.h"
/*
@ -147,6 +147,8 @@ int main(int argc, char **argv)
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
int depth;
@ -187,16 +189,16 @@ int main(int argc, char **argv)
comm.barrier();
//.................................................
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&ndiscs,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&ndiscs,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -273,9 +275,9 @@ int main(int argc, char **argv)
if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad);
comm.barrier();
// Broadcast the sphere packing to all processes
comm.bcast(cx,ndiscs,0);
comm.bcast(cy,ndiscs,0);
comm.bcast(rad,ndiscs,0);
MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm);
//...........................................................................
comm.barrier();
if (rank == 0){
@ -344,7 +346,7 @@ int main(int argc, char **argv)
}
}
sum_local = 1.0*sum;
porosity = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = porosity*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -360,7 +362,7 @@ int main(int argc, char **argv)
}
}
}
pore_vol = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
//.........................................................
// don't perform computations at the eight corners

22
tests/lbpm_inkbottle_pp.cpp
View File

@ -81,16 +81,16 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -197,7 +197,7 @@ int main(int argc, char **argv)
}
}
}
pore_vol = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
//.........................................................
// don't perform computations at the eight corners

35
tests/lbpm_juanes_bench_disc_pp.cpp
View File

@ -9,7 +9,7 @@
#include "analysis/pmmc.h"
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/MPI.h"
#include "common/MPI.h" // This includes mpi.h
#include "common/SpherePack.h"
/*
@ -147,6 +147,9 @@ int main(int argc, char **argv)
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -190,16 +193,16 @@ int main(int argc, char **argv)
comm.barrier();
//.................................................
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&ndiscs,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&ndiscs,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -289,9 +292,9 @@ int main(int argc, char **argv)
if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad);
comm.barrier();
// Broadcast the sphere packing to all processes
comm.bcast(cx,ndiscs,0);
comm.bcast(cy,ndiscs,0);
comm.bcast(rad,ndiscs,0);
MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm);
//...........................................................................
comm.barrier();
/* if (rank == 0){
@ -433,7 +436,7 @@ int main(int argc, char **argv)
}
}
sum_local = 1.0*sum;
porosity = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = porosity*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -449,7 +452,7 @@ int main(int argc, char **argv)
}
}
}
pore_vol = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
//.........................................................
// don't perform computations at the eight corners

52
tests/lbpm_nondarcy_simulator.cpp
View File

@ -94,6 +94,8 @@ int main(int argc, char **argv)
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
double REYNOLDS_NUMBER = 100.f;
if (argc > 1){
@ -156,28 +158,28 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
//.................................................
comm.bcast(&tau,1,0);
//comm.bcast(&pBC,1,0);
//comm.bcast(&Restart,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
//MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
// MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -306,8 +308,8 @@ int main(int argc, char **argv)
}
}
}
por_vol = comm.sumReduce( sum_local );
//porosity = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
// MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = pore_vol*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
//.........................................................
@ -431,7 +433,7 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
//.........................................
double D32,vawx,vawy,vawz,Fo,Re,velocity,err1D,mag_force,vel_prev;
@ -552,7 +554,7 @@ int main(int argc, char **argv)
fclose(NONDARCY);
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;

26
tests/lbpm_nonnewtonian_simulator.cpp
View File

@ -124,6 +124,8 @@ int main(int argc, char **argv)
// int rank_xz,rank_XZ,rank_xZ,rank_Xz;
// int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -426,8 +428,8 @@ int main(int argc, char **argv)
}
}
pore_vol = comm.sumReduce( sum_local ); /* 6 */
//porosity = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); /* 6 */
//MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = pore_vol*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -572,7 +574,7 @@ int main(int argc, char **argv)
timestep=5;
}
}
comm.bcast(&timestep,1,0);
MPI_Bcast(&timestep,1,MPI_INT,0,comm);
// Read in the restart file to CPU buffers
double *cDen = new double[2*N];
@ -660,7 +662,7 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
starttime = MPI_Wtime();
/*
* Create the thread pool
@ -808,7 +810,7 @@ int main(int argc, char **argv)
//************************************************************************/
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = Utilities::MPI::time();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
@ -833,6 +835,20 @@ int main(int argc, char **argv)
// Scrap
// if (rank==0){

24
tests/lbpm_plates_pp.cpp
View File

@ -31,6 +31,8 @@ int main(int argc, char **argv)
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
double TubeRadius =15.0;
double WIDTH;
@ -75,16 +77,16 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -174,7 +176,7 @@ int main(int argc, char **argv)
}
}
}
pore_vol = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
//.........................................................
// don't perform computations at the eight corners

25
tests/lbpm_porenetwork_pp.cpp
View File

@ -24,6 +24,9 @@ int main(int argc, char **argv)
int iproc,jproc,kproc;
int sendtag,recvtag;
//*****************************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//**********************************
int nsph,ncyl, BC;
nsph = atoi(argv[1]);
@ -64,16 +67,16 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -266,7 +269,7 @@ int main(int argc, char **argv)
}
}
}
pore_vol = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
if (rank==0) printf("Pore volume = %f \n",pore_vol/double(Nx*Ny*Nz));
//.........................................................
// don't perform computations at the eight corners

92
tests/lbpm_random_pp.cpp
View File

@ -98,16 +98,16 @@ int main(int argc, char **argv)
}
comm.barrier();
// Computational domain
comm.bcast(&nx,1,0);
comm.bcast(&ny,1,0);
comm.bcast(&nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&nx,1,MPI_INT,0,comm);
MPI_Bcast(&ny,1,MPI_INT,0,comm);
MPI_Bcast(&nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -166,7 +166,7 @@ int main(int argc, char **argv)
}
}
// total Global is the number of nodes in the pore-space
totalGlobal = sumReduce( count );
MPI_Allreduce(&count,&totalGlobal,1,MPI_INT,MPI_SUM,comm);
float porosity=float(totalGlobal)/(nprocx*nprocy*nprocz*(nx-2)*(ny-2)*(nz-2));
if (rank==0) printf("Media Porosity: %f \n",porosity);
@ -216,12 +216,12 @@ int main(int argc, char **argv)
sizeY = SizeY[bin];
sizeZ = SizeZ[bin];
}
comm.bcast(&x,1,0);
comm.bcast(&y,1,0);
comm.bcast(&z,1,0);
comm.bcast(&sizeX,1,0);
comm.bcast(&sizeY,1,0);
comm.bcast(&sizeZ,1,0);
MPI_Bcast(&x,1,MPI_INT,0,comm);
MPI_Bcast(&y,1,MPI_INT,0,comm);
MPI_Bcast(&z,1,MPI_INT,0,comm);
MPI_Bcast(&sizeX,1,MPI_INT,0,comm);
MPI_Bcast(&sizeY,1,MPI_INT,0,comm);
MPI_Bcast(&sizeZ,1,MPI_INT,0,comm);
//if (rank==0) printf("Broadcast block at %i,%i,%i \n",x,y,z);
@ -269,7 +269,7 @@ int main(int argc, char **argv)
}
}
}
countGlobal = sumReduce( count );
MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm);
sat = float(countGlobal)/totalGlobal;
//if (rank==0) printf("New count=%i\n",countGlobal);
//if (rank==0) printf("New saturation=%f\n",sat);
@ -345,24 +345,42 @@ int main(int argc, char **argv)
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag);
comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag);
comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag);
comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag);
comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag);
comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag);
comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag);
comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag);
comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag);
comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag);
comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag);
comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag);
comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag);
comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag);
comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag);
comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag);
comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag);
comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag);
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);
@ -394,7 +412,7 @@ int main(int argc, char **argv)
}
}
}
countGlobal = comm.sumReduce( count );
MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm);
sat = float(countGlobal)/totalGlobal;
if (rank==0) printf("Final saturation=%f\n",sat);

48
tests/lbpm_segmented_decomp.cpp
View File

@ -85,23 +85,23 @@ int main(int argc, char **argv)
comm.barrier();
// Computational domain
//.................................................
comm.bcast(&nx,1,0);
comm.bcast(&ny,1,0);
comm.bcast(&nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&nx,1,MPI_INT,0,comm);
MPI_Bcast(&ny,1,MPI_INT,0,comm);
MPI_Bcast(&nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&xStart,1,0);
comm.bcast(&yStart,1,0);
comm.bcast(&zStart,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&xStart,1,MPI_INT,0,comm);
MPI_Bcast(&yStart,1,MPI_INT,0,comm);
MPI_Bcast(&zStart,1,MPI_INT,0,comm);
//.................................................
comm.barrier();
@ -191,7 +191,7 @@ int main(int argc, char **argv)
}
else{
printf("Sending data to process %i \n", rnk);
comm.send(tmp,N,rnk,15);
MPI_Send(tmp,N,MPI_CHAR,rnk,15,comm);
}
}
}
@ -200,7 +200,7 @@ int main(int argc, char **argv)
else{
// Recieve the subdomain from rank = 0
printf("Ready to recieve data %i at process %i \n", N,rank);
comm.recv(Dm.id,N,0,15);
MPI_Recv(Dm.id,N,MPI_CHAR,0,15,comm,MPI_STATUS_IGNORE);
}
comm.barrier();
@ -243,8 +243,8 @@ int main(int argc, char **argv)
printf("Original label=%i, New label=%i \n",oldlabel,newlabel);
}
}
comm.barrier();
comm.bcast(LabelList,2*NLABELS,0);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Bcast(LabelList,2*NLABELS,MPI_INT,0,MPI_COMM_WORLD);
char *newIDs;
newIDs= new char [nx*ny*nz];
@ -278,8 +278,8 @@ int main(int argc, char **argv)
}
}
}
countGlobal = comm.sumReduce( count );
totalGlobal = comm.sumReduce( total );
MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm);
MPI_Allreduce(&total,&totalGlobal,1,MPI_INT,MPI_SUM,comm);
float porosity = float(totalGlobal-countGlobal)/totalGlobal;
@ -321,8 +321,8 @@ int main(int argc, char **argv)
}
}
}
countGlobal = comm.sumReduce( count );
totalGlobal = comm.sumReduce( total );
MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm);
MPI_Allreduce(&total,&totalGlobal,1,MPI_INT,MPI_SUM,comm);
float saturation = float(countGlobal)/totalGlobal;
if (rank==0) printf("wetting phase saturation=%f\n",saturation);

2
tests/lbpm_segmented_pp.cpp
View File

@ -180,7 +180,7 @@ int main(int argc, char **argv)
fflush(stdout);
porosity = ReadFromBlock(Dm->id,Dm->iproc(),Dm->jproc(),Dm->kproc(),nx,ny,nz);
comm.barrier();
MPI_Barrier(MPI_COMM_WORLD);
if (rank==0) printf("Writing local ID files (poros=%f) \n",porosity);
fflush(stdout);
FILE *ID = fopen(LocalRankFilename,"wb");

16
tests/lbpm_sphere_pp.cpp
View File

@ -38,6 +38,8 @@ int main(int argc, char **argv)
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -123,10 +125,10 @@ int main(int argc, char **argv)
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
comm.barrier();
// Broadcast the sphere packing to all processes
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
//...........................................................................
comm.barrier();
if (rank == 0) cout << "Domain set." << endl;
@ -142,7 +144,7 @@ int main(int argc, char **argv)
D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx;
printf("Sauter Mean Diameter (computed from sphere packing) = %f \n",D);
}
comm.bcast(&D,1,0);
MPI_Bcast(&D,1,MPI_DOUBLE,0,comm);
//.......................................................................
SignedDistance(SignDist.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz,
@ -175,7 +177,7 @@ int main(int argc, char **argv)
}
}
sum_local = 1.0*sum;
porosity = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = porosity*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
@ -191,7 +193,7 @@ int main(int argc, char **argv)
}
}
}
pore_vol = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
//.........................................................
// don't perform computations at the eight corners

25
tests/lbpm_squaretube_pp.cpp
View File

@ -30,6 +30,9 @@ int main(int argc, char **argv)
int rank_xy,rank_XY,rank_xY,rank_Xy;
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
int ORIENTATION=2; //default: the tube is aligned with Z axis
//ORIENTATION = 0: tube is aligned with X axis
@ -80,16 +83,16 @@ int main(int argc, char **argv)
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
// Computational domain
comm.bcast(&Nx,1,0);
comm.bcast(&Ny,1,0);
comm.bcast(&Nz,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
@ -232,7 +235,7 @@ int main(int argc, char **argv)
}
}
}
pore_vol = comm.sumReduce( sum_local );
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
//.........................................................
// don't perform computations at the eight corners