OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

files match direct from FOM

Browse Source
This commit is contained in:
James McClure
2021-01-04 21:49:54 -05:00
parent fbb2dd67b2
commit 81953c2fce
4 changed files with 123 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

226
common/Communication.h
View File

@@ -1,7 +1,7 @@
#ifndef COMMUNICATION_H_INC
#define COMMUNICATION_H_INC
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "common/Array.h"
@@ -38,7 +38,7 @@ struct RankInfoStruct {
//! Redistribute domain data (dst may be smaller than the src)
template<class TYPE>
Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm );
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, const Utilities::MPI& comm );
/*!
@@ -59,7 +59,7 @@ public:
* @param[in] fill Fill {faces,edges,corners}
* @param[in] periodic Periodic dimensions
*/
fillHalo( MPI_Comm comm, const RankInfoStruct& info,
fillHalo( const Utilities::MPI& comm, const RankInfoStruct& info,
std::array<int,3> n, std::array<int,3> ng, int tag, int depth,
std::array<bool,3> fill = {true,true,true},
std::array<bool,3> periodic = {true,true,true} );
@@ -83,7 +83,7 @@ public:
private:
MPI_Comm comm;
Utilities::MPI comm;
RankInfoStruct info;
std::array<int,3> n, ng;
int depth;
@@ -93,8 +93,6 @@ private:
TYPE *mem;
TYPE *send[3][3][3], *recv[3][3][3];
MPI_Request send_req[3][3][3], recv_req[3][3][3];
size_t N_type;
MPI_Datatype datatype;
fillHalo(); // Private empty constructor
fillHalo(const fillHalo&); // Private copy constructor
fillHalo& operator=(const fillHalo&); // Private assignment operator
@@ -136,7 +134,7 @@ void InitializeRanks( const int rank, const int nprocx, const int nprocy, const
//***************************************************************************************
inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int recvtag,
inline void CommunicateSendRecvCounts( const Utilities::MPI& comm, int sendtag, int recvtag,
int rank_x, int rank_y, int rank_z,
int rank_X, int rank_Y, int rank_Z,
int rank_xy, int rank_XY, int rank_xY, int rank_Xy,
@@ -154,54 +152,53 @@ inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int r
int& recvCount_yz, int& recvCount_YZ, int& recvCount_yZ, int& recvCount_Yz )
{
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]);
req1[0] = comm.Isend(&sendCount_x,1,rank_x,sendtag+0);
req2[0] = comm.Irecv(&recvCount_X,1,rank_X,recvtag+0);
req1[1] = comm.Isend(&sendCount_X,1,rank_X,sendtag+1);
req2[1] = comm.Irecv(&recvCount_x,1,rank_x,recvtag+1);
req1[2] = comm.Isend(&sendCount_y,1,rank_y,sendtag+2);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_Y,recvtag+2);
req1[3] = comm.Isend(&sendCount_Y,1,rank_Y,sendtag+3);
req2[3] = comm.Irecv(&recvCount_y,1,rank_y,recvtag+3);
req1[4] = comm.Isend(&sendCount_z,1,rank_z,sendtag+4);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_Z,recvtag+4);
req1[5] = comm.Isend(&sendCount_Z,1,rank_Z,sendtag+5);
req2[5] = comm.Irecv(&recvCount_z,1,rank_z,recvtag+5);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_xy,sendtag+6);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_XY,recvtag+6);
req1[7] = comm.Isend(&sendCount_XY,1,rank_XY,sendtag+7);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_xy,recvtag+7);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_Xy,sendtag+8);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_xY,recvtag+8);
req1[9] = comm.Isend(&sendCount_xY,1,rank_xY,sendtag+9);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_Xy,recvtag+9);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_xz,sendtag+10);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_XZ,recvtag+10);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_XZ,sendtag+11);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_xz,recvtag+11);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_Xz,sendtag+12);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_xZ,recvtag+12);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_xZ,sendtag+13);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_Xz,recvtag+13);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(Communicator);
req1[14] = comm.Isend(&sendCount_yz,1,rank_yz,sendtag+14);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_YZ,recvtag+14);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_YZ,sendtag+15);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_yz,recvtag+15);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_Yz,sendtag+16);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_yZ,recvtag+16);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_yZ,sendtag+17);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_Yz,recvtag+17);
comm.waitAll( 18, req1 );
comm.waitAll( 18, req2 );
comm.barrier();
}
//***************************************************************************************
inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvtag,
inline void CommunicateRecvLists( const Utilities::MPI& comm, int sendtag, int recvtag,
int *sendList_x, int *sendList_y, int *sendList_z, int *sendList_X, int *sendList_Y, int *sendList_Z,
int *sendList_xy, int *sendList_XY, int *sendList_xY, int *sendList_Xy,
int *sendList_xz, int *sendList_XZ, int *sendList_xZ, int *sendList_Xz,
@@ -222,53 +219,52 @@ inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvta
int rank_Xy, int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz, int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz)
{
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz,recvtag);
comm.waitAll( 18, req1 );
comm.waitAll( 18, req2 );
}
//***************************************************************************************
inline void CommunicateMeshHalo(DoubleArray &Mesh, MPI_Comm Communicator,
inline void CommunicateMeshHalo(DoubleArray &Mesh, const Utilities::MPI& comm,
double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy,
double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz,
@@ -318,42 +314,24 @@ inline void CommunicateMeshHalo(DoubleArray &Mesh, MPI_Comm Communicator,
PackMeshData(sendList_yZ, sendCount_yZ ,sendbuf_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData);
//......................................................................................
MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_DOUBLE,rank_x,sendtag,
recvbuf_X,recvCount_X,MPI_DOUBLE,rank_X,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_DOUBLE,rank_X,sendtag,
recvbuf_x,recvCount_x,MPI_DOUBLE,rank_x,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_DOUBLE,rank_y,sendtag,
recvbuf_Y,recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,
recvbuf_y,recvCount_y,MPI_DOUBLE,rank_y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_DOUBLE,rank_z,sendtag,
recvbuf_Z,recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,
recvbuf_z,recvCount_z,MPI_DOUBLE,rank_z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,
recvbuf_XY,recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,
recvbuf_xy,recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,
recvbuf_xY,recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,
recvbuf_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,
recvbuf_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,
recvbuf_xz,recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,
recvbuf_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,
recvbuf_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,
recvbuf_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,
recvbuf_yz,recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,
recvbuf_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,
recvbuf_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,Communicator,MPI_STATUS_IGNORE);
comm.sendrecv(sendbuf_x,sendCount_x,rank_x,sendtag,recvbuf_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendbuf_X,sendCount_X,rank_X,sendtag,recvbuf_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendbuf_y,sendCount_y,rank_y,sendtag,recvbuf_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendbuf_Y,sendCount_Y,rank_Y,sendtag,recvbuf_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendbuf_z,sendCount_z,rank_z,sendtag,recvbuf_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendbuf_Z,sendCount_Z,rank_Z,sendtag,recvbuf_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendbuf_xy,sendCount_xy,rank_xy,sendtag,recvbuf_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendbuf_XY,sendCount_XY,rank_XY,sendtag,recvbuf_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag,recvbuf_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendbuf_xY,sendCount_xY,rank_xY,sendtag,recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendbuf_xz,sendCount_xz,rank_xz,sendtag,recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag,recvbuf_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag,recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag,recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendbuf_yz,sendCount_yz,rank_yz,sendtag,recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag,recvbuf_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag,recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag,recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData);

53
common/Communication.hpp
View File

@@ -2,9 +2,8 @@
#define COMMUNICATION_HPP_INC
#include "common/Communication.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
//#include "ProfilerApp.h"
/********************************************************
@@ -12,17 +11,19 @@
********************************************************/
template<class TYPE>
Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm )
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, const Utilities::MPI& comm )
{
#ifdef USE_MPI
if ( comm.getSize() == 1 ) {
return src_data.subset( { 0, (size_t) dst_size[0]-1, 0, (size_t) dst_size[1]-1, 0, (size_t) dst_size[2]-1 } );
}
// Get the src size
std::array<int,3> src_size;
int size0[3] = { (int) src_data.size(0), (int) src_data.size(1), (int) src_data.size(2) };
MPI_Allreduce( size0, src_size.data(), 3, MPI_INT, MPI_MAX, comm );
comm.maxReduce( size0, src_size.data(), 3 );
if ( !src_data.empty() )
ASSERT( src_size[0] == size0[0] && src_size[1] == size0[1] && src_size[2] == size0[2] );
// Check that dst_size matches on all ranks
MPI_Allreduce( dst_size.data(), size0, 3, MPI_INT, MPI_MAX, comm );
comm.maxReduce( dst_size.data(), size0, 3 );
ASSERT( dst_size[0] == size0[0] && dst_size[1] == size0[1] && dst_size[2] == size0[2] );
// Function to get overlap range
auto calcOverlap = []( int i1[3], int i2[3], int j1[3], int j2[3] ) {
@@ -60,7 +61,7 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
}
std::vector<MPI_Request> send_request( send_rank.size() );
for (size_t i=0; i<send_rank.size(); i++)
MPI_Isend( send_data[i].data(), sizeof(TYPE)*send_data[i].length(), MPI_BYTE, send_rank[i], 5462, comm, &send_request[i]);
send_request[i] = comm.Isend( send_data[i].data(), send_data[i].length(), send_rank[i], 5462 );
// Unpack data from the appropriate ranks (including myself)
Array<TYPE> dst_data( dst_size[0], dst_size[1], dst_size[2] );
int i1[3] = { dst_size[0] * dst_rank.ix, dst_size[1] * dst_rank.jy, dst_size[2] * dst_rank.kz };
@@ -75,17 +76,14 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
continue;
int rank = src_rank.getRankForBlock(i,j,k);
Array<TYPE> data( index[1] - index[0] + 1, index[3] - index[2] + 1, index[5] - index[4] + 1 );
MPI_Recv( data.data(), sizeof(TYPE)*data.length(), MPI_BYTE, rank, 5462, comm, MPI_STATUS_IGNORE );
comm.recv( data.data(), data.length(), rank, 5462 );
dst_data.copySubset( index, data );
}
}
}
// Free data
MPI_Waitall( send_request.size(), send_request.data(), MPI_STATUSES_IGNORE );
comm.waitAll( send_request.size(), send_request.data() );
return dst_data;
#else
return src_data.subset( { 0, dst_size[0]-1, 0, dst_size[1]-1, 0, dst_size[2]-1 );
#endif
}
@@ -94,27 +92,11 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
* Structure to fill halo cells *
********************************************************/
template<class TYPE>
fillHalo<TYPE>::fillHalo( MPI_Comm comm_, const RankInfoStruct& info_,
fillHalo<TYPE>::fillHalo( const Utilities::MPI& comm_, const RankInfoStruct& info_,
std::array<int,3> n_, std::array<int,3> ng_, int tag0, int depth_,
std::array<bool,3> fill, std::array<bool,3> periodic ):
comm(comm_), info(info_), n(n_), ng(ng_), depth(depth_)
{
if ( std::is_same<TYPE,double>() ) {
N_type = 1;
datatype = MPI_DOUBLE;
} else if ( std::is_same<TYPE,float>() ) {
N_type = 1;
datatype = MPI_FLOAT;
} else if ( sizeof(TYPE)%sizeof(double)==0 ) {
N_type = sizeof(TYPE) / sizeof(double);
datatype = MPI_DOUBLE;
} else if ( sizeof(TYPE)%sizeof(float)==0 ) {
N_type = sizeof(TYPE) / sizeof(float);
datatype = MPI_FLOAT;
} else {
N_type = sizeof(TYPE);
datatype = MPI_BYTE;
}
// Set the fill pattern
memset(fill_pattern,0,sizeof(fill_pattern));
if ( fill[0] ) {
@@ -251,8 +233,8 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
for (int k=0; k<3; k++) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Irecv( recv[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype,
info.rank[i][j][k], tag[2-i][2-j][2-k], comm, &recv_req[i][j][k] );
recv_req[i][j][k] = comm.Irecv( recv[i][j][k], depth2*N_send_recv[i][j][k],
info.rank[i][j][k], tag[2-i][2-j][2-k] );
}
}
}
@@ -263,19 +245,18 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
if ( !fill_pattern[i][j][k] )
continue;
pack( data, i-1, j-1, k-1, send[i][j][k] );
MPI_Isend( send[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype,
info.rank[i][j][k], tag[i][j][k], comm, &send_req[i][j][k] );
send_req[i][j][k] = comm.Isend( send[i][j][k], depth2*N_send_recv[i][j][k],
info.rank[i][j][k], tag[i][j][k] );
}
}
}
// Recv the dst data and unpack (we recive in reverse order to match the sends)
MPI_Status status;
for (int i=2; i>=0; i--) {
for (int j=2; j>=0; j--) {
for (int k=2; k>=0; k--) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Wait(&recv_req[i][j][k],&status);
comm.wait( recv_req[i][j][k] );
unpack( data, i-1, j-1, k-1, recv[i][j][k] );
}
}
@@ -286,7 +267,7 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
for (int k=0; k<3; k++) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Wait(&send_req[i][j][k],&status);
comm.wait( send_req[i][j][k] );
}
}
}

4
common/ReadMicroCT.cpp
View File

@@ -64,11 +64,11 @@ Array<uint8_t> readMicroCT( const std::string& filename )
// Read the compressed micro CT data and distribute
Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm )
Array<uint8_t> readMicroCT( const Database& domain, const Utilities::MPI& comm )
{
// Get the local problem info
auto n = domain.getVector<int>( "n" );
int rank = comm_rank(MPI_COMM_WORLD);
int rank = comm.getRank();
auto nproc = domain.getVector<int>( "nproc" );
RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] );

3
common/ReadMicroCT.h
View File

@@ -5,11 +5,12 @@
#include "common/Array.h"
#include "common/Communication.h"
#include "common/Database.h"
#include "common/MPI.h"
Array<uint8_t> readMicroCT( const std::string& filename );
Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm );
Array<uint8_t> readMicroCT( const Database& domain, const Utilities::MPI& comm );
#endif