OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Trying to fix bug with MPI communication

Browse Source
This commit is contained in:
Mark Berrill 2020-03-12 10:26:17 -04:00
parent 57156d16fc
commit 94156e066e
21 changed files with 777 additions and 795 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

382
analysis/morphology.cpp
View File

@ -1,7 +1,7 @@
#include <analysis/morphology.h>
// Implementation of morphological opening routine
inline void PackID(int *list, int count, signed char *sendbuf, signed char *ID){
inline void PackID(const int *list, int count, signed char *sendbuf, signed char *ID){
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
int idx,n;
@ -13,7 +13,7 @@ inline void PackID(int *list, int count, signed char *sendbuf, signed char *ID){
}
//***************************************************************************************
inline void UnpackID(int *list, int count, signed char *recvbuf, signed char *ID){
inline void UnpackID(const int *list, int count, signed char *recvbuf, signed char *ID){
// Fill in the phase ID values from neighboring processors
// This unpacks the values once they have been recieved from neighbors
int idx,n;
@ -77,44 +77,44 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
// send buffers
sendID_x = new signed char [Dm->sendCount_x];
sendID_y = new signed char [Dm->sendCount_y];
sendID_z = new signed char [Dm->sendCount_z];
sendID_X = new signed char [Dm->sendCount_X];
sendID_Y = new signed char [Dm->sendCount_Y];
sendID_Z = new signed char [Dm->sendCount_Z];
sendID_xy = new signed char [Dm->sendCount_xy];
sendID_yz = new signed char [Dm->sendCount_yz];
sendID_xz = new signed char [Dm->sendCount_xz];
sendID_Xy = new signed char [Dm->sendCount_Xy];
sendID_Yz = new signed char [Dm->sendCount_Yz];
sendID_xZ = new signed char [Dm->sendCount_xZ];
sendID_xY = new signed char [Dm->sendCount_xY];
sendID_yZ = new signed char [Dm->sendCount_yZ];
sendID_Xz = new signed char [Dm->sendCount_Xz];
sendID_XY = new signed char [Dm->sendCount_XY];
sendID_YZ = new signed char [Dm->sendCount_YZ];
sendID_XZ = new signed char [Dm->sendCount_XZ];
sendID_x = new signed char [Dm->sendCount("x")];
sendID_y = new signed char [Dm->sendCount("y")];
sendID_z = new signed char [Dm->sendCount("z")];
sendID_X = new signed char [Dm->sendCount("X")];
sendID_Y = new signed char [Dm->sendCount("Y")];
sendID_Z = new signed char [Dm->sendCount("Z")];
sendID_xy = new signed char [Dm->sendCount("xy")];
sendID_yz = new signed char [Dm->sendCount("yz")];
sendID_xz = new signed char [Dm->sendCount("xz")];
sendID_Xy = new signed char [Dm->sendCount("Xy")];
sendID_Yz = new signed char [Dm->sendCount("Yz")];
sendID_xZ = new signed char [Dm->sendCount("xZ")];
sendID_xY = new signed char [Dm->sendCount("xY")];
sendID_yZ = new signed char [Dm->sendCount("yZ")];
sendID_Xz = new signed char [Dm->sendCount("Xz")];
sendID_XY = new signed char [Dm->sendCount("XY")];
sendID_YZ = new signed char [Dm->sendCount("YZ")];
sendID_XZ = new signed char [Dm->sendCount("XZ")];
//......................................................................................
// recv buffers
recvID_x = new signed char [Dm->recvCount_x];
recvID_y = new signed char [Dm->recvCount_y];
recvID_z = new signed char [Dm->recvCount_z];
recvID_X = new signed char [Dm->recvCount_X];
recvID_Y = new signed char [Dm->recvCount_Y];
recvID_Z = new signed char [Dm->recvCount_Z];
recvID_xy = new signed char [Dm->recvCount_xy];
recvID_yz = new signed char [Dm->recvCount_yz];
recvID_xz = new signed char [Dm->recvCount_xz];
recvID_Xy = new signed char [Dm->recvCount_Xy];
recvID_xZ = new signed char [Dm->recvCount_xZ];
recvID_xY = new signed char [Dm->recvCount_xY];
recvID_yZ = new signed char [Dm->recvCount_yZ];
recvID_Yz = new signed char [Dm->recvCount_Yz];
recvID_Xz = new signed char [Dm->recvCount_Xz];
recvID_XY = new signed char [Dm->recvCount_XY];
recvID_YZ = new signed char [Dm->recvCount_YZ];
recvID_XZ = new signed char [Dm->recvCount_XZ];
recvID_x = new signed char [Dm->recvCount("x")];
recvID_y = new signed char [Dm->recvCount("y")];
recvID_z = new signed char [Dm->recvCount("z")];
recvID_X = new signed char [Dm->recvCount("X")];
recvID_Y = new signed char [Dm->recvCount("Y")];
recvID_Z = new signed char [Dm->recvCount("Z")];
recvID_xy = new signed char [Dm->recvCount("xy")];
recvID_yz = new signed char [Dm->recvCount("yz")];
recvID_xz = new signed char [Dm->recvCount("xz")];
recvID_Xy = new signed char [Dm->recvCount("Xy")];
recvID_xZ = new signed char [Dm->recvCount("xZ")];
recvID_xY = new signed char [Dm->recvCount("xY")];
recvID_yZ = new signed char [Dm->recvCount("yZ")];
recvID_Yz = new signed char [Dm->recvCount("Yz")];
recvID_Xz = new signed char [Dm->recvCount("Xz")];
recvID_XY = new signed char [Dm->recvCount("XY")];
recvID_YZ = new signed char [Dm->recvCount("YZ")];
recvID_XZ = new signed char [Dm->recvCount("XZ")];
//......................................................................................
int sendtag,recvtag;
sendtag = recvtag = 7;
@ -182,62 +182,62 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
}
}
// Pack and send the updated ID values
PackID(Dm->sendList_x, Dm->sendCount_x ,sendID_x, id);
PackID(Dm->sendList_X, Dm->sendCount_X ,sendID_X, id);
PackID(Dm->sendList_y, Dm->sendCount_y ,sendID_y, id);
PackID(Dm->sendList_Y, Dm->sendCount_Y ,sendID_Y, id);
PackID(Dm->sendList_z, Dm->sendCount_z ,sendID_z, id);
PackID(Dm->sendList_Z, Dm->sendCount_Z ,sendID_Z, id);
PackID(Dm->sendList_xy, Dm->sendCount_xy ,sendID_xy, id);
PackID(Dm->sendList_Xy, Dm->sendCount_Xy ,sendID_Xy, id);
PackID(Dm->sendList_xY, Dm->sendCount_xY ,sendID_xY, id);
PackID(Dm->sendList_XY, Dm->sendCount_XY ,sendID_XY, id);
PackID(Dm->sendList_xz, Dm->sendCount_xz ,sendID_xz, id);
PackID(Dm->sendList_Xz, Dm->sendCount_Xz ,sendID_Xz, id);
PackID(Dm->sendList_xZ, Dm->sendCount_xZ ,sendID_xZ, id);
PackID(Dm->sendList_XZ, Dm->sendCount_XZ ,sendID_XZ, id);
PackID(Dm->sendList_yz, Dm->sendCount_yz ,sendID_yz, id);
PackID(Dm->sendList_Yz, Dm->sendCount_Yz ,sendID_Yz, id);
PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
PackID(Dm->sendList("x"), Dm->sendCount("x") ,sendID_x, id);
PackID(Dm->sendList("X"), Dm->sendCount("X") ,sendID_X, id);
PackID(Dm->sendList("y"), Dm->sendCount("y") ,sendID_y, id);
PackID(Dm->sendList("Y"), Dm->sendCount("Y") ,sendID_Y, id);
PackID(Dm->sendList("z"), Dm->sendCount("z") ,sendID_z, id);
PackID(Dm->sendList("Z"), Dm->sendCount("Z") ,sendID_Z, id);
PackID(Dm->sendList("xy"), Dm->sendCount("xy") ,sendID_xy, id);
PackID(Dm->sendList("Xy"), Dm->sendCount("Xy") ,sendID_Xy, id);
PackID(Dm->sendList("xY"), Dm->sendCount("xY") ,sendID_xY, id);
PackID(Dm->sendList("XY"), Dm->sendCount("XY") ,sendID_XY, id);
PackID(Dm->sendList("xz"), Dm->sendCount("xz") ,sendID_xz, id);
PackID(Dm->sendList("Xz"), Dm->sendCount("Xz") ,sendID_Xz, id);
PackID(Dm->sendList("xZ"), Dm->sendCount("xZ") ,sendID_xZ, id);
PackID(Dm->sendList("XZ"), Dm->sendCount("XZ") ,sendID_XZ, id);
PackID(Dm->sendList("yz"), Dm->sendCount("yz") ,sendID_yz, id);
PackID(Dm->sendList("Yz"), Dm->sendCount("Yz") ,sendID_Yz, id);
PackID(Dm->sendList("yZ"), Dm->sendCount("yZ") ,sendID_yZ, id);
PackID(Dm->sendList("YZ"), Dm->sendCount("YZ") ,sendID_YZ, id);
//......................................................................................
Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag);
Dm->Comm.sendrecv(sendID_x,Dm->sendCount("x"),Dm->rank_x(),sendtag,recvID_X,Dm->recvCount("X"),Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount("X"),Dm->rank_X(),sendtag,recvID_x,Dm->recvCount("x"),Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount("y"),Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount("Y"),Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount("Y"),Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount("y"),Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount("z"),Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount("Z"),Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount("Z"),Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount("z"),Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount("xy"),Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount("XY"),Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount("XY"),Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount("xy"),Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount("Xy"),Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount("xY"),Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount("xY"),Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount("Xy"),Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount("xz"),Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount("XZ"),Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount("XZ"),Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount("xz"),Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount("Xz"),Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount("xZ"),Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount("xZ"),Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount("Xz"),Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount("yz"),Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount("YZ"),Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount("YZ"),Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount("yz"),Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount("Yz"),Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount("yZ"),Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount("yZ"),Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount("Yz"),Dm->rank_Yz(),recvtag);
//......................................................................................
UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
UnpackID(Dm->recvList_y, Dm->recvCount_y ,recvID_y, id);
UnpackID(Dm->recvList_Y, Dm->recvCount_Y ,recvID_Y, id);
UnpackID(Dm->recvList_z, Dm->recvCount_z ,recvID_z, id);
UnpackID(Dm->recvList_Z, Dm->recvCount_Z ,recvID_Z, id);
UnpackID(Dm->recvList_xy, Dm->recvCount_xy ,recvID_xy, id);
UnpackID(Dm->recvList_Xy, Dm->recvCount_Xy ,recvID_Xy, id);
UnpackID(Dm->recvList_xY, Dm->recvCount_xY ,recvID_xY, id);
UnpackID(Dm->recvList_XY, Dm->recvCount_XY ,recvID_XY, id);
UnpackID(Dm->recvList_xz, Dm->recvCount_xz ,recvID_xz, id);
UnpackID(Dm->recvList_Xz, Dm->recvCount_Xz ,recvID_Xz, id);
UnpackID(Dm->recvList_xZ, Dm->recvCount_xZ ,recvID_xZ, id);
UnpackID(Dm->recvList_XZ, Dm->recvCount_XZ ,recvID_XZ, id);
UnpackID(Dm->recvList_yz, Dm->recvCount_yz ,recvID_yz, id);
UnpackID(Dm->recvList_Yz, Dm->recvCount_Yz ,recvID_Yz, id);
UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id);
UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id);
UnpackID(Dm->recvList("x"), Dm->recvCount("x") ,recvID_x, id);
UnpackID(Dm->recvList("X"), Dm->recvCount("X") ,recvID_X, id);
UnpackID(Dm->recvList("y"), Dm->recvCount("y") ,recvID_y, id);
UnpackID(Dm->recvList("Y"), Dm->recvCount("Y") ,recvID_Y, id);
UnpackID(Dm->recvList("z"), Dm->recvCount("z") ,recvID_z, id);
UnpackID(Dm->recvList("Z"), Dm->recvCount("Z") ,recvID_Z, id);
UnpackID(Dm->recvList("xy"), Dm->recvCount("xy") ,recvID_xy, id);
UnpackID(Dm->recvList("Xy"), Dm->recvCount("Xy") ,recvID_Xy, id);
UnpackID(Dm->recvList("xY"), Dm->recvCount("xY") ,recvID_xY, id);
UnpackID(Dm->recvList("XY"), Dm->recvCount("XY") ,recvID_XY, id);
UnpackID(Dm->recvList("xz"), Dm->recvCount("xz") ,recvID_xz, id);
UnpackID(Dm->recvList("Xz"), Dm->recvCount("Xz") ,recvID_Xz, id);
UnpackID(Dm->recvList("xZ"), Dm->recvCount("xZ") ,recvID_xZ, id);
UnpackID(Dm->recvList("XZ"), Dm->recvCount("XZ") ,recvID_XZ, id);
UnpackID(Dm->recvList("yz"), Dm->recvCount("yz") ,recvID_yz, id);
UnpackID(Dm->recvList("Yz"), Dm->recvCount("Yz") ,recvID_Yz, id);
UnpackID(Dm->recvList("yZ"), Dm->recvCount("yZ") ,recvID_yZ, id);
UnpackID(Dm->recvList("YZ"), Dm->recvCount("YZ") ,recvID_YZ, id);
//......................................................................................
//double GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
@ -359,44 +359,44 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
// send buffers
sendID_x = new signed char [Dm->sendCount_x];
sendID_y = new signed char [Dm->sendCount_y];
sendID_z = new signed char [Dm->sendCount_z];
sendID_X = new signed char [Dm->sendCount_X];
sendID_Y = new signed char [Dm->sendCount_Y];
sendID_Z = new signed char [Dm->sendCount_Z];
sendID_xy = new signed char [Dm->sendCount_xy];
sendID_yz = new signed char [Dm->sendCount_yz];
sendID_xz = new signed char [Dm->sendCount_xz];
sendID_Xy = new signed char [Dm->sendCount_Xy];
sendID_Yz = new signed char [Dm->sendCount_Yz];
sendID_xZ = new signed char [Dm->sendCount_xZ];
sendID_xY = new signed char [Dm->sendCount_xY];
sendID_yZ = new signed char [Dm->sendCount_yZ];
sendID_Xz = new signed char [Dm->sendCount_Xz];
sendID_XY = new signed char [Dm->sendCount_XY];
sendID_YZ = new signed char [Dm->sendCount_YZ];
sendID_XZ = new signed char [Dm->sendCount_XZ];
sendID_x = new signed char [Dm->sendCount("x")];
sendID_y = new signed char [Dm->sendCount("y")];
sendID_z = new signed char [Dm->sendCount("z")];
sendID_X = new signed char [Dm->sendCount("X")];
sendID_Y = new signed char [Dm->sendCount("Y")];
sendID_Z = new signed char [Dm->sendCount("Z")];
sendID_xy = new signed char [Dm->sendCount("xy")];
sendID_yz = new signed char [Dm->sendCount("yz")];
sendID_xz = new signed char [Dm->sendCount("xz")];
sendID_Xy = new signed char [Dm->sendCount("Xy")];
sendID_Yz = new signed char [Dm->sendCount("Yz")];
sendID_xZ = new signed char [Dm->sendCount("xZ")];
sendID_xY = new signed char [Dm->sendCount("xY")];
sendID_yZ = new signed char [Dm->sendCount("yZ")];
sendID_Xz = new signed char [Dm->sendCount("Xz")];
sendID_XY = new signed char [Dm->sendCount("XY")];
sendID_YZ = new signed char [Dm->sendCount("YZ")];
sendID_XZ = new signed char [Dm->sendCount("XZ")];
//......................................................................................
// recv buffers
recvID_x = new signed char [Dm->recvCount_x];
recvID_y = new signed char [Dm->recvCount_y];
recvID_z = new signed char [Dm->recvCount_z];
recvID_X = new signed char [Dm->recvCount_X];
recvID_Y = new signed char [Dm->recvCount_Y];
recvID_Z = new signed char [Dm->recvCount_Z];
recvID_xy = new signed char [Dm->recvCount_xy];
recvID_yz = new signed char [Dm->recvCount_yz];
recvID_xz = new signed char [Dm->recvCount_xz];
recvID_Xy = new signed char [Dm->recvCount_Xy];
recvID_xZ = new signed char [Dm->recvCount_xZ];
recvID_xY = new signed char [Dm->recvCount_xY];
recvID_yZ = new signed char [Dm->recvCount_yZ];
recvID_Yz = new signed char [Dm->recvCount_Yz];
recvID_Xz = new signed char [Dm->recvCount_Xz];
recvID_XY = new signed char [Dm->recvCount_XY];
recvID_YZ = new signed char [Dm->recvCount_YZ];
recvID_XZ = new signed char [Dm->recvCount_XZ];
recvID_x = new signed char [Dm->recvCount("x")];
recvID_y = new signed char [Dm->recvCount("y")];
recvID_z = new signed char [Dm->recvCount("z")];
recvID_X = new signed char [Dm->recvCount("X")];
recvID_Y = new signed char [Dm->recvCount("Y")];
recvID_Z = new signed char [Dm->recvCount("Z")];
recvID_xy = new signed char [Dm->recvCount("xy")];
recvID_yz = new signed char [Dm->recvCount("yz")];
recvID_xz = new signed char [Dm->recvCount("xz")];
recvID_Xy = new signed char [Dm->recvCount("Xy")];
recvID_xZ = new signed char [Dm->recvCount("xZ")];
recvID_xY = new signed char [Dm->recvCount("xY")];
recvID_yZ = new signed char [Dm->recvCount("yZ")];
recvID_Yz = new signed char [Dm->recvCount("Yz")];
recvID_Xz = new signed char [Dm->recvCount("Xz")];
recvID_XY = new signed char [Dm->recvCount("XY")];
recvID_YZ = new signed char [Dm->recvCount("YZ")];
recvID_XZ = new signed char [Dm->recvCount("XZ")];
//......................................................................................
int sendtag,recvtag;
sendtag = recvtag = 7;
@ -469,80 +469,62 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
}
// Pack and send the updated ID values
PackID(Dm->sendList_x, Dm->sendCount_x ,sendID_x, id);
PackID(Dm->sendList_X, Dm->sendCount_X ,sendID_X, id);
PackID(Dm->sendList_y, Dm->sendCount_y ,sendID_y, id);
PackID(Dm->sendList_Y, Dm->sendCount_Y ,sendID_Y, id);
PackID(Dm->sendList_z, Dm->sendCount_z ,sendID_z, id);
PackID(Dm->sendList_Z, Dm->sendCount_Z ,sendID_Z, id);
PackID(Dm->sendList_xy, Dm->sendCount_xy ,sendID_xy, id);
PackID(Dm->sendList_Xy, Dm->sendCount_Xy ,sendID_Xy, id);
PackID(Dm->sendList_xY, Dm->sendCount_xY ,sendID_xY, id);
PackID(Dm->sendList_XY, Dm->sendCount_XY ,sendID_XY, id);
PackID(Dm->sendList_xz, Dm->sendCount_xz ,sendID_xz, id);
PackID(Dm->sendList_Xz, Dm->sendCount_Xz ,sendID_Xz, id);
PackID(Dm->sendList_xZ, Dm->sendCount_xZ ,sendID_xZ, id);
PackID(Dm->sendList_XZ, Dm->sendCount_XZ ,sendID_XZ, id);
PackID(Dm->sendList_yz, Dm->sendCount_yz ,sendID_yz, id);
PackID(Dm->sendList_Yz, Dm->sendCount_Yz ,sendID_Yz, id);
PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
PackID(Dm->sendList("x"), Dm->sendCount("x") ,sendID_x, id);
PackID(Dm->sendList("X"), Dm->sendCount("X") ,sendID_X, id);
PackID(Dm->sendList("y"), Dm->sendCount("y") ,sendID_y, id);
PackID(Dm->sendList("Y"), Dm->sendCount("Y") ,sendID_Y, id);
PackID(Dm->sendList("z"), Dm->sendCount("z") ,sendID_z, id);
PackID(Dm->sendList("Z"), Dm->sendCount("Z") ,sendID_Z, id);
PackID(Dm->sendList("xy"), Dm->sendCount("xy") ,sendID_xy, id);
PackID(Dm->sendList("Xy"), Dm->sendCount("Xy") ,sendID_Xy, id);
PackID(Dm->sendList("xY"), Dm->sendCount("xY") ,sendID_xY, id);
PackID(Dm->sendList("XY"), Dm->sendCount("XY") ,sendID_XY, id);
PackID(Dm->sendList("xz"), Dm->sendCount("xz") ,sendID_xz, id);
PackID(Dm->sendList("Xz"), Dm->sendCount("Xz") ,sendID_Xz, id);
PackID(Dm->sendList("xZ"), Dm->sendCount("xZ") ,sendID_xZ, id);
PackID(Dm->sendList("XZ"), Dm->sendCount("XZ") ,sendID_XZ, id);
PackID(Dm->sendList("yz"), Dm->sendCount("yz") ,sendID_yz, id);
PackID(Dm->sendList("Yz"), Dm->sendCount("Yz") ,sendID_Yz, id);
PackID(Dm->sendList("yZ"), Dm->sendCount("yZ") ,sendID_yZ, id);
PackID(Dm->sendList("YZ"), Dm->sendCount("YZ") ,sendID_YZ, id);
//......................................................................................
Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag);
Dm->Comm.sendrecv(sendID_x,Dm->sendCount("x"),Dm->rank_x(),sendtag,recvID_X,Dm->recvCount("X"),Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount("X"),Dm->rank_X(),sendtag,recvID_x,Dm->recvCount("x"),Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount("y"),Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount("Y"),Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount("Y"),Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount("y"),Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount("z"),Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount("Z"),Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount("Z"),Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount("z"),Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount("xy"),Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount("XY"),Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount("XY"),Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount("xy"),Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount("Xy"),Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount("xY"),Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount("xY"),Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount("Xy"),Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount("xz"),Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount("XZ"),Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount("XZ"),Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount("xz"),Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount("Xz"),Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount("xZ"),Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount("xZ"),Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount("Xz"),Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount("yz"),Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount("YZ"),Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount("YZ"),Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount("yz"),Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount("Yz"),Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount("yZ"),Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount("yZ"),Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount("Yz"),Dm->rank_Yz(),recvtag);
//......................................................................................
UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
UnpackID(Dm->recvList_y, Dm->recvCount_y ,recvID_y, id);
UnpackID(Dm->recvList_Y, Dm->recvCount_Y ,recvID_Y, id);
UnpackID(Dm->recvList_z, Dm->recvCount_z ,recvID_z, id);
UnpackID(Dm->recvList_Z, Dm->recvCount_Z ,recvID_Z, id);
UnpackID(Dm->recvList_xy, Dm->recvCount_xy ,recvID_xy, id);
UnpackID(Dm->recvList_Xy, Dm->recvCount_Xy ,recvID_Xy, id);
UnpackID(Dm->recvList_xY, Dm->recvCount_xY ,recvID_xY, id);
UnpackID(Dm->recvList_XY, Dm->recvCount_XY ,recvID_XY, id);
UnpackID(Dm->recvList_xz, Dm->recvCount_xz ,recvID_xz, id);
UnpackID(Dm->recvList_Xz, Dm->recvCount_Xz ,recvID_Xz, id);
UnpackID(Dm->recvList_xZ, Dm->recvCount_xZ ,recvID_xZ, id);
UnpackID(Dm->recvList_XZ, Dm->recvCount_XZ ,recvID_XZ, id);
UnpackID(Dm->recvList_yz, Dm->recvCount_yz ,recvID_yz, id);
UnpackID(Dm->recvList_Yz, Dm->recvCount_Yz ,recvID_Yz, id);
UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id);
UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id);
UnpackID(Dm->recvList("x"), Dm->recvCount("x") ,recvID_x, id);
UnpackID(Dm->recvList("X"), Dm->recvCount("X") ,recvID_X, id);
UnpackID(Dm->recvList("y"), Dm->recvCount("y") ,recvID_y, id);
UnpackID(Dm->recvList("Y"), Dm->recvCount("Y") ,recvID_Y, id);
UnpackID(Dm->recvList("z"), Dm->recvCount("z") ,recvID_z, id);
UnpackID(Dm->recvList("Z"), Dm->recvCount("Z") ,recvID_Z, id);
UnpackID(Dm->recvList("xy"), Dm->recvCount("xy") ,recvID_xy, id);
UnpackID(Dm->recvList("Xy"), Dm->recvCount("Xy") ,recvID_Xy, id);
UnpackID(Dm->recvList("xY"), Dm->recvCount("xY") ,recvID_xY, id);
UnpackID(Dm->recvList("XY"), Dm->recvCount("XY") ,recvID_XY, id);
UnpackID(Dm->recvList("xz"), Dm->recvCount("xz") ,recvID_xz, id);
UnpackID(Dm->recvList("Xz"), Dm->recvCount("Xz") ,recvID_Xz, id);
UnpackID(Dm->recvList("xZ"), Dm->recvCount("xZ") ,recvID_xZ, id);
UnpackID(Dm->recvList("XZ"), Dm->recvCount("XZ") ,recvID_XZ, id);
UnpackID(Dm->recvList("yz"), Dm->recvCount("yz") ,recvID_yz, id);
UnpackID(Dm->recvList("Yz"), Dm->recvCount("Yz") ,recvID_Yz, id);
UnpackID(Dm->recvList("yZ"), Dm->recvCount("yZ") ,recvID_yZ, id);
UnpackID(Dm->recvList("YZ"), Dm->recvCount("YZ") ,recvID_YZ, id);
//......................................................................................
// double GlobalNumber = Dm->Comm.sumReduce( LocalNumber );

41
analysis/runAnalysis.cpp
View File

@ -510,7 +510,7 @@ runAnalysis::commWrapper runAnalysis::getComm( )
runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
const RankInfoStruct& rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm,
std::shared_ptr <Domain> Dm,
std::shared_ptr<Domain> Dm,
int Np,
bool Regular,
IntArray Map ):
@ -518,8 +518,7 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
d_regular ( Regular),
d_rank_info( rank_info ),
d_Map( Map ),
d_fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1),
d_comm( Utilities::MPI( MPI_COMM_WORLD ).dup() ),
d_comm( Dm->Comm.dup() ),
d_ScaLBL_Comm( ScaLBL_Comm)
{
@ -535,6 +534,9 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
char rankString[20];
sprintf(rankString,"%05d",Dm->rank());
d_n[0] = Dm->Nx-2;
d_n[1] = Dm->Ny-2;
d_n[2] = Dm->Nz-2;
d_N[0] = Dm->Nx;
d_N[1] = Dm->Ny;
d_N[2] = Dm->Nz;
@ -566,7 +568,7 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
d_meshData.resize(1);
d_meshData[0].meshName = "domain";
d_meshData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
d_meshData[0].mesh = std::make_shared<IO::DomainMesh>( d_rank_info,d_n[0],d_n[1],d_n[2],Dm->Lx,Dm->Ly,Dm->Lz );
auto PhaseVar = std::make_shared<IO::Variable>();
auto PressVar = std::make_shared<IO::Variable>();
auto VxVar = std::make_shared<IO::Variable>();
@ -579,7 +581,7 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
PhaseVar->name = "phase";
PhaseVar->type = IO::VariableType::VolumeVariable;
PhaseVar->dim = 1;
PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
PhaseVar->data.resize(d_n[0],d_n[1],d_n[2]);
d_meshData[0].vars.push_back(PhaseVar);
}
@ -587,7 +589,7 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
PressVar->name = "Pressure";
PressVar->type = IO::VariableType::VolumeVariable;
PressVar->dim = 1;
PressVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
PressVar->data.resize(d_n[0],d_n[1],d_n[2]);
d_meshData[0].vars.push_back(PressVar);
}
@ -595,17 +597,17 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
VxVar->name = "Velocity_x";
VxVar->type = IO::VariableType::VolumeVariable;
VxVar->dim = 1;
VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
VxVar->data.resize(d_n[0],d_n[1],d_n[2]);
d_meshData[0].vars.push_back(VxVar);
VyVar->name = "Velocity_y";
VyVar->type = IO::VariableType::VolumeVariable;
VyVar->dim = 1;
VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
VyVar->data.resize(d_n[0],d_n[1],d_n[2]);
d_meshData[0].vars.push_back(VyVar);
VzVar->name = "Velocity_z";
VzVar->type = IO::VariableType::VolumeVariable;
VzVar->dim = 1;
VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
VzVar->data.resize(d_n[0],d_n[1],d_n[2]);
d_meshData[0].vars.push_back(VzVar);
}
@ -613,7 +615,7 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
SignDistVar->name = "SignDist";
SignDistVar->type = IO::VariableType::VolumeVariable;
SignDistVar->dim = 1;
SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
SignDistVar->data.resize(d_n[0],d_n[1],d_n[2]);
d_meshData[0].vars.push_back(SignDistVar);
}
@ -621,7 +623,7 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
BlobIDVar->name = "BlobID";
BlobIDVar->type = IO::VariableType::VolumeVariable;
BlobIDVar->dim = 1;
BlobIDVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
BlobIDVar->data.resize(d_n[0],d_n[1],d_n[2]);
d_meshData[0].vars.push_back(BlobIDVar);
}
@ -682,7 +684,10 @@ void runAnalysis::createThreads( const std::string& method, int N_threads )
}
// Create the threads
const auto cores = d_tpool.getProcessAffinity();
if ( cores.empty() ) {
if ( N_threads == 0 ) {
// Special case to serials the analysis for debugging
d_tpool.setNumThreads( 0 );
} else if ( cores.empty() ) {
// We were not able to get the cores for the process
d_tpool.setNumThreads( N_threads );
} else if ( method == "default" ) {
@ -921,7 +926,9 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
// if ( matches(type,AnalysisType::CreateRestart) ) {
if (timestep%d_restart_interval==0){
// Write the vis files
auto work = new WriteVisWorkItem( timestep, d_meshData, Averages, d_fillData, getComm() );
commWrapper comm = getComm();
fillHalo<double> fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 );
auto work = new WriteVisWorkItem( timestep, d_meshData, Averages, fillData, std::move( comm ) );
work->add_dependency(d_wait_blobID);
work->add_dependency(d_wait_analysis);
work->add_dependency(d_wait_vis);
@ -1025,7 +1032,9 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
if (timestep%d_visualization_interval==0){
// Write the vis files
auto work = new IOWorkItem( timestep, input_db, d_meshData, Averages, d_fillData, getComm() );
commWrapper comm = getComm();
fillHalo<double> fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 );
auto work = new IOWorkItem( timestep, input_db, d_meshData, Averages, fillData, std::move( comm ) );
work->add_dependency(d_wait_analysis);
work->add_dependency(d_wait_subphase);
work->add_dependency(d_wait_vis);
@ -1058,7 +1067,9 @@ void runAnalysis::WriteVisData(int timestep, std::shared_ptr<Database> input_db,
PROFILE_START("write vis",1);
// if (Averages.WriteVis == true){
auto work2 = new IOWorkItem(timestep, input_db, d_meshData, Averages, d_fillData, getComm() );
commWrapper comm = getComm();
fillHalo<double> fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 );
auto work2 = new IOWorkItem(timestep, input_db, d_meshData, Averages, fillData, std::move( comm ) );
work2->add_dependency(d_wait_vis);
d_wait_vis = d_tpool.add_work(work2);

4
analysis/runAnalysis.h
View File

@ -84,7 +84,8 @@ public:
private:
int d_N[3];
std::array<int,3> d_n; // Number of local cells
std::array<int,3> d_N; // NNumber of local cells with ghosts
int d_Np;
int d_rank;
int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval;
@ -98,7 +99,6 @@ private:
BlobIDstruct d_last_index;
BlobIDList d_last_id_map;
std::vector<IO::MeshDataStruct> d_meshData;
fillHalo<double> d_fillData;
std::string d_restartFile;
Utilities::MPI d_comm;
Utilities::MPI d_comms[1024];

4
common/Communication.h
View File

@ -102,7 +102,7 @@ private:
//***************************************************************************************
inline void PackMeshData(int *list, int count, double *sendbuf, double *data){
inline void PackMeshData(const int *list, int count, double *sendbuf, double *data){
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
int idx,n;
@ -111,7 +111,7 @@ inline void PackMeshData(int *list, int count, double *sendbuf, double *data){
sendbuf[idx] = data[n];
}
}
inline void UnpackMeshData(int *list, int count, double *recvbuf, double *data){
inline void UnpackMeshData(const int *list, int count, double *recvbuf, double *data){
// Fill in the phase ID values from neighboring processors
// This unpacks the values once they have been recieved from neighbors
int idx,n;

627
common/Domain.cpp
View File

@ -23,40 +23,15 @@ static inline void fgetl( char * str, int num, FILE * stream )
}
/********************************************************
* Constructors/Destructor *
* Constructors *
********************************************************/
Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
double lx, double ly, double lz, int BC):
database(NULL), Nx(0), Ny(0), Nz(0),
database(nullptr), Nx(0), Ny(0), Nz(0),
Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), voxel_length(1),
Comm(MPI_COMM_WORLD),
Comm( Utilities::MPI( MPI_COMM_WORLD).dup() ),
inlet_layers_x(0), inlet_layers_y(0), inlet_layers_z(0),
inlet_layers_phase(1),outlet_layers_phase(2),
sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0),
sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0),
sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0),
sendList_x(NULL), sendList_y(NULL), sendList_z(NULL), sendList_X(NULL), sendList_Y(NULL), sendList_Z(NULL),
sendList_xy(NULL), sendList_yz(NULL), sendList_xz(NULL), sendList_Xy(NULL), sendList_Yz(NULL), sendList_xZ(NULL),
sendList_xY(NULL), sendList_yZ(NULL), sendList_Xz(NULL), sendList_XY(NULL), sendList_YZ(NULL), sendList_XZ(NULL),
sendBuf_x(NULL), sendBuf_y(NULL), sendBuf_z(NULL), sendBuf_X(NULL), sendBuf_Y(NULL), sendBuf_Z(NULL),
sendBuf_xy(NULL), sendBuf_yz(NULL), sendBuf_xz(NULL), sendBuf_Xy(NULL), sendBuf_Yz(NULL), sendBuf_xZ(NULL),
sendBuf_xY(NULL), sendBuf_yZ(NULL), sendBuf_Xz(NULL), sendBuf_XY(NULL), sendBuf_YZ(NULL), sendBuf_XZ(NULL),
recvCount_x(0), recvCount_y(0), recvCount_z(0), recvCount_X(0), recvCount_Y(0), recvCount_Z(0),
recvCount_xy(0), recvCount_yz(0), recvCount_xz(0), recvCount_Xy(0), recvCount_Yz(0), recvCount_xZ(0),
recvCount_xY(0), recvCount_yZ(0), recvCount_Xz(0), recvCount_XY(0), recvCount_YZ(0), recvCount_XZ(0),
recvList_x(NULL), recvList_y(NULL), recvList_z(NULL), recvList_X(NULL), recvList_Y(NULL), recvList_Z(NULL),
recvList_xy(NULL), recvList_yz(NULL), recvList_xz(NULL), recvList_Xy(NULL), recvList_Yz(NULL), recvList_xZ(NULL),
recvList_xY(NULL), recvList_yZ(NULL), recvList_Xz(NULL), recvList_XY(NULL), recvList_YZ(NULL), recvList_XZ(NULL),
recvBuf_x(NULL), recvBuf_y(NULL), recvBuf_z(NULL), recvBuf_X(NULL), recvBuf_Y(NULL), recvBuf_Z(NULL),
recvBuf_xy(NULL), recvBuf_yz(NULL), recvBuf_xz(NULL), recvBuf_Xy(NULL), recvBuf_Yz(NULL), recvBuf_xZ(NULL),
recvBuf_xY(NULL), recvBuf_yZ(NULL), recvBuf_Xz(NULL), recvBuf_XY(NULL), recvBuf_YZ(NULL), recvBuf_XZ(NULL),
sendData_x(NULL), sendData_y(NULL), sendData_z(NULL), sendData_X(NULL), sendData_Y(NULL), sendData_Z(NULL),
sendData_xy(NULL), sendData_yz(NULL), sendData_xz(NULL), sendData_Xy(NULL), sendData_Yz(NULL), sendData_xZ(NULL),
sendData_xY(NULL), sendData_yZ(NULL), sendData_Xz(NULL), sendData_XY(NULL), sendData_YZ(NULL), sendData_XZ(NULL),
recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL),
recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL),
recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL),
id(NULL)
inlet_layers_phase(1),outlet_layers_phase(2)
{
NULL_USE( rnk );
NULL_USE( npy );
@ -80,32 +55,7 @@ Domain::Domain( std::shared_ptr<Database> db, const Utilities::MPI& Communicator
Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0),
inlet_layers_x(0), inlet_layers_y(0), inlet_layers_z(0),
outlet_layers_x(0), outlet_layers_y(0), outlet_layers_z(0),
inlet_layers_phase(1),outlet_layers_phase(2),
sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0),
sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0),
sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0),
sendList_x(NULL), sendList_y(NULL), sendList_z(NULL), sendList_X(NULL), sendList_Y(NULL), sendList_Z(NULL),
sendList_xy(NULL), sendList_yz(NULL), sendList_xz(NULL), sendList_Xy(NULL), sendList_Yz(NULL), sendList_xZ(NULL),
sendList_xY(NULL), sendList_yZ(NULL), sendList_Xz(NULL), sendList_XY(NULL), sendList_YZ(NULL), sendList_XZ(NULL),
sendBuf_x(NULL), sendBuf_y(NULL), sendBuf_z(NULL), sendBuf_X(NULL), sendBuf_Y(NULL), sendBuf_Z(NULL),
sendBuf_xy(NULL), sendBuf_yz(NULL), sendBuf_xz(NULL), sendBuf_Xy(NULL), sendBuf_Yz(NULL), sendBuf_xZ(NULL),
sendBuf_xY(NULL), sendBuf_yZ(NULL), sendBuf_Xz(NULL), sendBuf_XY(NULL), sendBuf_YZ(NULL), sendBuf_XZ(NULL),
recvCount_x(0), recvCount_y(0), recvCount_z(0), recvCount_X(0), recvCount_Y(0), recvCount_Z(0),
recvCount_xy(0), recvCount_yz(0), recvCount_xz(0), recvCount_Xy(0), recvCount_Yz(0), recvCount_xZ(0),
recvCount_xY(0), recvCount_yZ(0), recvCount_Xz(0), recvCount_XY(0), recvCount_YZ(0), recvCount_XZ(0),
recvList_x(NULL), recvList_y(NULL), recvList_z(NULL), recvList_X(NULL), recvList_Y(NULL), recvList_Z(NULL),
recvList_xy(NULL), recvList_yz(NULL), recvList_xz(NULL), recvList_Xy(NULL), recvList_Yz(NULL), recvList_xZ(NULL),
recvList_xY(NULL), recvList_yZ(NULL), recvList_Xz(NULL), recvList_XY(NULL), recvList_YZ(NULL), recvList_XZ(NULL),
recvBuf_x(NULL), recvBuf_y(NULL), recvBuf_z(NULL), recvBuf_X(NULL), recvBuf_Y(NULL), recvBuf_Z(NULL),
recvBuf_xy(NULL), recvBuf_yz(NULL), recvBuf_xz(NULL), recvBuf_Xy(NULL), recvBuf_Yz(NULL), recvBuf_xZ(NULL),
recvBuf_xY(NULL), recvBuf_yZ(NULL), recvBuf_Xz(NULL), recvBuf_XY(NULL), recvBuf_YZ(NULL), recvBuf_XZ(NULL),
sendData_x(NULL), sendData_y(NULL), sendData_z(NULL), sendData_X(NULL), sendData_Y(NULL), sendData_Z(NULL),
sendData_xy(NULL), sendData_yz(NULL), sendData_xz(NULL), sendData_Xy(NULL), sendData_Yz(NULL), sendData_xZ(NULL),
sendData_xY(NULL), sendData_yZ(NULL), sendData_Xz(NULL), sendData_XY(NULL), sendData_YZ(NULL), sendData_XZ(NULL),
recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL),
recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL),
recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL),
id(NULL)
inlet_layers_phase(1),outlet_layers_phase(2)
{
Comm = Communicator.dup();
@ -116,54 +66,18 @@ Domain::Domain( std::shared_ptr<Database> db, const Utilities::MPI& Communicator
Comm.barrier();
}
/********************************************************
* Destructor *
********************************************************/
Domain::~Domain()
{
// Free sendList
delete [] sendList_x; delete [] sendList_y; delete [] sendList_z;
delete [] sendList_X; delete [] sendList_Y; delete [] sendList_Z;
delete [] sendList_xy; delete [] sendList_yz; delete [] sendList_xz;
delete [] sendList_Xy; delete [] sendList_Yz; delete [] sendList_xZ;
delete [] sendList_xY; delete [] sendList_yZ; delete [] sendList_Xz;
delete [] sendList_XY; delete [] sendList_YZ; delete [] sendList_XZ;
// Free sendBuf
delete [] sendBuf_x; delete [] sendBuf_y; delete [] sendBuf_z;
delete [] sendBuf_X; delete [] sendBuf_Y; delete [] sendBuf_Z;
delete [] sendBuf_xy; delete [] sendBuf_yz; delete [] sendBuf_xz;
delete [] sendBuf_Xy; delete [] sendBuf_Yz; delete [] sendBuf_xZ;
delete [] sendBuf_xY; delete [] sendBuf_yZ; delete [] sendBuf_Xz;
delete [] sendBuf_XY; delete [] sendBuf_YZ; delete [] sendBuf_XZ;
// Free recvList
delete [] recvList_x; delete [] recvList_y; delete [] recvList_z;
delete [] recvList_X; delete [] recvList_Y; delete [] recvList_Z;
delete [] recvList_xy; delete [] recvList_yz; delete [] recvList_xz;
delete [] recvList_Xy; delete [] recvList_Yz; delete [] recvList_xZ;
delete [] recvList_xY; delete [] recvList_yZ; delete [] recvList_Xz;
delete [] recvList_XY; delete [] recvList_YZ; delete [] recvList_XZ;
// Free recvBuf
delete [] recvBuf_x; delete [] recvBuf_y; delete [] recvBuf_z;
delete [] recvBuf_X; delete [] recvBuf_Y; delete [] recvBuf_Z;
delete [] recvBuf_xy; delete [] recvBuf_yz; delete [] recvBuf_xz;
delete [] recvBuf_Xy; delete [] recvBuf_Yz; delete [] recvBuf_xZ;
delete [] recvBuf_xY; delete [] recvBuf_yZ; delete [] recvBuf_Xz;
delete [] recvBuf_XY; delete [] recvBuf_YZ; delete [] recvBuf_XZ;
// Free sendData
delete [] sendData_x; delete [] sendData_y; delete [] sendData_z;
delete [] sendData_X; delete [] sendData_Y; delete [] sendData_Z;
delete [] sendData_xy; delete [] sendData_xY; delete [] sendData_Xy;
delete [] sendData_XY; delete [] sendData_xz; delete [] sendData_xZ;
delete [] sendData_Xz; delete [] sendData_XZ; delete [] sendData_yz;
delete [] sendData_yZ; delete [] sendData_Yz; delete [] sendData_YZ;
// Free recvData
delete [] recvData_x; delete [] recvData_y; delete [] recvData_z;
delete [] recvData_X; delete [] recvData_Y; delete [] recvData_Z;
delete [] recvData_xy; delete [] recvData_xY; delete [] recvData_Xy;
delete [] recvData_XY; delete [] recvData_xz; delete [] recvData_xZ;
delete [] recvData_Xz; delete [] recvData_XZ; delete [] recvData_yz;
delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ;
// Free id
delete [] id;
}
/********************************************************
* Initialization *
********************************************************/
void Domain::initialize( std::shared_ptr<Database> db )
{
d_db = db;
@ -228,13 +142,115 @@ void Domain::initialize( std::shared_ptr<Database> db )
if (myrank==0) printf("voxel length = %f micron \n", voxel_length);
id = new signed char[N];
memset(id,0,N);
id = std::vector<signed char>( N, 0 );
BoundaryCondition = d_db->getScalar<int>("BC");
int nprocs = Comm.getSize();
INSIST(nprocs == nproc[0]*nproc[1]*nproc[2],"Fatal error in processor count!");
}
/********************************************************
* Get send/recv lists *
********************************************************/
const std::vector<int>& Domain::getRecvList( const char* dir ) const
{
if ( dir[0] == 'x' ) {
if ( dir[1] == 0 )
return recvList_x;
else if ( dir[1] == 'y' )
return recvList_xy;
else if ( dir[1] == 'Y' )
return recvList_xY;
else if ( dir[1] == 'z' )
return recvList_xz;
else if ( dir[1] == 'Z' )
return recvList_xZ;
} else if ( dir[0] == 'y' ) {
if ( dir[1] == 0 )
return recvList_y;
else if ( dir[1] == 'z' )
return recvList_yz;
else if ( dir[1] == 'Z' )
return recvList_yZ;
} else if ( dir[0] == 'z' ) {
if ( dir[1] == 0 )
return recvList_z;
} else if ( dir[0] == 'X' ) {
if ( dir[1] == 0 )
return recvList_X;
else if ( dir[1] == 'y' )
return recvList_Xy;
else if ( dir[1] == 'Y' )
return recvList_XY;
else if ( dir[1] == 'z' )
return recvList_Xz;
else if ( dir[1] == 'Z' )
return recvList_XZ;
} else if ( dir[0] == 'Y' ) {
if ( dir[1] == 0 )
return recvList_Y;
else if ( dir[1] == 'z' )
return recvList_Yz;
else if ( dir[1] == 'Z' )
return recvList_YZ;
} else if ( dir[0] == 'Z' ) {
if ( dir[1] == 0 )
return recvList_Z;
}
throw std::logic_error("Internal error");
}
const std::vector<int>& Domain::getSendList( const char* dir ) const
{
if ( dir[0] == 'x' ) {
if ( dir[1] == 0 )
return sendList_x;
else if ( dir[1] == 'y' )
return sendList_xy;
else if ( dir[1] == 'Y' )
return sendList_xY;
else if ( dir[1] == 'z' )
return sendList_xz;
else if ( dir[1] == 'Z' )
return sendList_xZ;
} else if ( dir[0] == 'y' ) {
if ( dir[1] == 0 )
return sendList_y;
else if ( dir[1] == 'z' )
return sendList_yz;
else if ( dir[1] == 'Z' )
return sendList_yZ;
} else if ( dir[0] == 'z' ) {
if ( dir[1] == 0 )
return sendList_z;
} else if ( dir[0] == 'X' ) {
if ( dir[1] == 0 )
return sendList_X;
else if ( dir[1] == 'y' )
return sendList_Xy;
else if ( dir[1] == 'Y' )
return sendList_XY;
else if ( dir[1] == 'z' )
return sendList_Xz;
else if ( dir[1] == 'Z' )
return sendList_XZ;
} else if ( dir[0] == 'Y' ) {
if ( dir[1] == 0 )
return sendList_Y;
else if ( dir[1] == 'z' )
return sendList_Yz;
else if ( dir[1] == 'Z' )
return sendList_YZ;
} else if ( dir[0] == 'Z' ) {
if ( dir[1] == 0 )
return sendList_Z;
}
throw std::logic_error("Internal error");
}
/********************************************************
* Decomp *
********************************************************/
void Domain::Decomp( const std::string& Filename )
{
//.......................................................................
@ -319,7 +335,7 @@ void Domain::Decomp( const std::string& Filename )
global_Ny = SIZE[1];
global_Nz = SIZE[2];
nprocs=nprocx*nprocy*nprocz;
char *SegData = NULL;
char *SegData = nullptr;
if (RANK==0){
printf("Input media: %s\n",Filename.c_str());
@ -337,7 +353,7 @@ void Domain::Decomp( const std::string& Filename )
if (ReadType == "8bit"){
printf("Reading 8-bit input data \n");
FILE *SEGDAT = fopen(Filename.c_str(),"rb");
if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading segmented data");
if (!SEGDAT) ERROR("Domain.cpp: Error reading segmented data");
size_t ReadSeg;
ReadSeg=fread(SegData,1,SIZE,SEGDAT);
if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading segmented data \n");
@ -348,7 +364,7 @@ void Domain::Decomp( const std::string& Filename )
short int *InputData;
InputData = new short int[SIZE];
FILE *SEGDAT = fopen(Filename.c_str(),"rb");
if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading segmented data");
if (!SEGDAT) ERROR("Domain.cpp: Error reading segmented data");
size_t ReadSeg;
ReadSeg=fread(InputData,2,SIZE,SEGDAT);
if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading segmented data \n");
@ -575,7 +591,7 @@ void Domain::Decomp( const std::string& Filename )
else{
// Recieve the subdomain from rank = 0
//printf("Ready to recieve data %i at process %i \n", N,rank);
Comm.recv(id,N,0,15);
Comm.recv(id.data(),N,0,15);
}
Comm.barrier();
}
@ -680,6 +696,9 @@ void Domain::CommInit()
int sendtag = 21;
int recvtag = 21;
//......................................................................................
int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0;
sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0;
sendCount_xY = sendCount_yZ = sendCount_Xz = sendCount_XY = sendCount_YZ = sendCount_XZ = 0;
@ -717,24 +736,24 @@ void Domain::CommInit()
}
// allocate send lists
sendList_x = new int [sendCount_x];
sendList_y = new int [sendCount_y];
sendList_z = new int [sendCount_z];
sendList_X = new int [sendCount_X];
sendList_Y = new int [sendCount_Y];
sendList_Z = new int [sendCount_Z];
sendList_xy = new int [sendCount_xy];
sendList_yz = new int [sendCount_yz];
sendList_xz = new int [sendCount_xz];
sendList_Xy = new int [sendCount_Xy];
sendList_Yz = new int [sendCount_Yz];
sendList_xZ = new int [sendCount_xZ];
sendList_xY = new int [sendCount_xY];
sendList_yZ = new int [sendCount_yZ];
sendList_Xz = new int [sendCount_Xz];
sendList_XY = new int [sendCount_XY];
sendList_YZ = new int [sendCount_YZ];
sendList_XZ = new int [sendCount_XZ];
sendList_x.resize( sendCount_x, 0 );
sendList_y.resize( sendCount_y, 0 );
sendList_z.resize( sendCount_z, 0 );
sendList_X.resize( sendCount_X, 0 );
sendList_Y.resize( sendCount_Y, 0 );
sendList_Z.resize( sendCount_Z, 0 );
sendList_xy.resize( sendCount_xy, 0 );
sendList_yz.resize( sendCount_yz, 0 );
sendList_xz.resize( sendCount_xz, 0 );
sendList_Xy.resize( sendCount_Xy, 0 );
sendList_Yz.resize( sendCount_Yz, 0 );
sendList_xZ.resize( sendCount_xZ, 0 );
sendList_xY.resize( sendCount_xY, 0 );
sendList_yZ.resize( sendCount_yZ, 0 );
sendList_Xz.resize( sendCount_Xz, 0 );
sendList_XY.resize( sendCount_XY, 0 );
sendList_YZ.resize( sendCount_YZ, 0 );
sendList_XZ.resize( sendCount_XZ, 0 );
// Populate the send list
sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0;
sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0;
@ -772,26 +791,10 @@ void Domain::CommInit()
}
}
// allocate send buffers
sendBuf_x = new int [sendCount_x];
sendBuf_y = new int [sendCount_y];
sendBuf_z = new int [sendCount_z];
sendBuf_X = new int [sendCount_X];
sendBuf_Y = new int [sendCount_Y];
sendBuf_Z = new int [sendCount_Z];
sendBuf_xy = new int [sendCount_xy];
sendBuf_yz = new int [sendCount_yz];
sendBuf_xz = new int [sendCount_xz];
sendBuf_Xy = new int [sendCount_Xy];
sendBuf_Yz = new int [sendCount_Yz];
sendBuf_xZ = new int [sendCount_xZ];
sendBuf_xY = new int [sendCount_xY];
sendBuf_yZ = new int [sendCount_yZ];
sendBuf_Xz = new int [sendCount_Xz];
sendBuf_XY = new int [sendCount_XY];
sendBuf_YZ = new int [sendCount_YZ];
sendBuf_XZ = new int [sendCount_XZ];
//......................................................................................
int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
req1[0] = Comm.Isend(&sendCount_x,1,rank_x(),sendtag+0);
req2[0] = Comm.Irecv(&recvCount_X,1,rank_X(),recvtag+0);
req1[1] = Comm.Isend(&sendCount_X,1,rank_X(),sendtag+1);
@ -831,63 +834,62 @@ void Domain::CommInit()
Comm.waitAll(18,req1);
Comm.waitAll(18,req2);
Comm.barrier();
// allocate recv lists
recvList_x.resize( recvCount_x, 0 );
recvList_y.resize( recvCount_y, 0 );
recvList_z.resize( recvCount_z, 0 );
recvList_X.resize( recvCount_X, 0 );
recvList_Y.resize( recvCount_Y, 0 );
recvList_Z.resize( recvCount_Z, 0 );
recvList_xy.resize( recvCount_xy, 0 );
recvList_yz.resize( recvCount_yz, 0 );
recvList_xz.resize( recvCount_xz, 0 );
recvList_Xy.resize( recvCount_Xy, 0 );
recvList_Yz.resize( recvCount_Yz, 0 );
recvList_xZ.resize( recvCount_xZ, 0 );
recvList_xY.resize( recvCount_xY, 0 );
recvList_yZ.resize( recvCount_yZ, 0 );
recvList_Xz.resize( recvCount_Xz, 0 );
recvList_XY.resize( recvCount_XY, 0 );
recvList_YZ.resize( recvCount_YZ, 0 );
recvList_XZ.resize( recvCount_XZ, 0 );
//......................................................................................
// recv buffers
recvList_x = new int [recvCount_x];
recvList_y = new int [recvCount_y];
recvList_z = new int [recvCount_z];
recvList_X = new int [recvCount_X];
recvList_Y = new int [recvCount_Y];
recvList_Z = new int [recvCount_Z];
recvList_xy = new int [recvCount_xy];
recvList_yz = new int [recvCount_yz];
recvList_xz = new int [recvCount_xz];
recvList_Xy = new int [recvCount_Xy];
recvList_Yz = new int [recvCount_Yz];
recvList_xZ = new int [recvCount_xZ];
recvList_xY = new int [recvCount_xY];
recvList_yZ = new int [recvCount_yZ];
recvList_Xz = new int [recvCount_Xz];
recvList_XY = new int [recvCount_XY];
recvList_YZ = new int [recvCount_YZ];
recvList_XZ = new int [recvCount_XZ];
//......................................................................................
req1[0] = Comm.Isend(sendList_x,sendCount_x,rank_x(),sendtag);
req2[0] = Comm.Irecv(recvList_X,recvCount_X,rank_X(),recvtag);
req1[1] = Comm.Isend(sendList_X,sendCount_X,rank_X(),sendtag);
req2[1] = Comm.Irecv(recvList_x,recvCount_x,rank_x(),recvtag);
req1[2] = Comm.Isend(sendList_y,sendCount_y,rank_y(),sendtag);
req2[2] = Comm.Irecv(recvList_Y,recvCount_Y,rank_Y(),recvtag);
req1[3] = Comm.Isend(sendList_Y,sendCount_Y,rank_Y(),sendtag);
req2[3] = Comm.Irecv(recvList_y,recvCount_y,rank_y(),recvtag);
req1[4] = Comm.Isend(sendList_z,sendCount_z,rank_z(),sendtag);
req2[4] = Comm.Irecv(recvList_Z,recvCount_Z,rank_Z(),recvtag);
req1[5] = Comm.Isend(sendList_Z,sendCount_Z,rank_Z(),sendtag);
req2[5] = Comm.Irecv(recvList_z,recvCount_z,rank_z(),recvtag);
req1[6] = Comm.Isend(sendList_xy,sendCount_xy,rank_xy(),sendtag);
req2[6] = Comm.Irecv(recvList_XY,recvCount_XY,rank_XY(),recvtag);
req1[7] = Comm.Isend(sendList_XY,sendCount_XY,rank_XY(),sendtag);
req2[7] = Comm.Irecv(recvList_xy,recvCount_xy,rank_xy(),recvtag);
req1[8] = Comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy(),sendtag);
req2[8] = Comm.Irecv(recvList_xY,recvCount_xY,rank_xY(),recvtag);
req1[9] = Comm.Isend(sendList_xY,sendCount_xY,rank_xY(),sendtag);
req2[9] = Comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy(),recvtag);
req1[10] = Comm.Isend(sendList_xz,sendCount_xz,rank_xz(),sendtag);
req2[10] = Comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ(),recvtag);
req1[11] = Comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ(),sendtag);
req2[11] = Comm.Irecv(recvList_xz,recvCount_xz,rank_xz(),recvtag);
req1[12] = Comm.Isend(sendList_Xz,sendCount_Xz,rank_Xz(),sendtag);
req2[12] = Comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ(),recvtag);
req1[13] = Comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ(),sendtag);
req2[13] = Comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz(),recvtag);
req1[14] = Comm.Isend(sendList_yz,sendCount_yz,rank_yz(),sendtag);
req2[14] = Comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ(),recvtag);
req1[15] = Comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ(),sendtag);
req2[15] = Comm.Irecv(recvList_yz,recvCount_yz,rank_yz(),recvtag);
req1[16] = Comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz(),sendtag);
req2[16] = Comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ(),recvtag);
req1[17] = Comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ(),sendtag);
req2[17] = Comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz(),recvtag);
req1[0] = Comm.Isend(sendList_x.data(),sendCount_x,rank_x(),sendtag);
req2[0] = Comm.Irecv(recvList_X.data(),recvCount_X,rank_X(),recvtag);
req1[1] = Comm.Isend(sendList_X.data(),sendCount_X,rank_X(),sendtag);
req2[1] = Comm.Irecv(recvList_x.data(),recvCount_x,rank_x(),recvtag);
req1[2] = Comm.Isend(sendList_y.data(),sendCount_y,rank_y(),sendtag);
req2[2] = Comm.Irecv(recvList_Y.data(),recvCount_Y,rank_Y(),recvtag);
req1[3] = Comm.Isend(sendList_Y.data(),sendCount_Y,rank_Y(),sendtag);
req2[3] = Comm.Irecv(recvList_y.data(),recvCount_y,rank_y(),recvtag);
req1[4] = Comm.Isend(sendList_z.data(),sendCount_z,rank_z(),sendtag);
req2[4] = Comm.Irecv(recvList_Z.data(),recvCount_Z,rank_Z(),recvtag);
req1[5] = Comm.Isend(sendList_Z.data(),sendCount_Z,rank_Z(),sendtag);
req2[5] = Comm.Irecv(recvList_z.data(),recvCount_z,rank_z(),recvtag);
req1[6] = Comm.Isend(sendList_xy.data(),sendCount_xy,rank_xy(),sendtag);
req2[6] = Comm.Irecv(recvList_XY.data(),recvCount_XY,rank_XY(),recvtag);
req1[7] = Comm.Isend(sendList_XY.data(),sendCount_XY,rank_XY(),sendtag);
req2[7] = Comm.Irecv(recvList_xy.data(),recvCount_xy,rank_xy(),recvtag);
req1[8] = Comm.Isend(sendList_Xy.data(),sendCount_Xy,rank_Xy(),sendtag);
req2[8] = Comm.Irecv(recvList_xY.data(),recvCount_xY,rank_xY(),recvtag);
req1[9] = Comm.Isend(sendList_xY.data(),sendCount_xY,rank_xY(),sendtag);
req2[9] = Comm.Irecv(recvList_Xy.data(),recvCount_Xy,rank_Xy(),recvtag);
req1[10] = Comm.Isend(sendList_xz.data(),sendCount_xz,rank_xz(),sendtag);
req2[10] = Comm.Irecv(recvList_XZ.data(),recvCount_XZ,rank_XZ(),recvtag);
req1[11] = Comm.Isend(sendList_XZ.data(),sendCount_XZ,rank_XZ(),sendtag);
req2[11] = Comm.Irecv(recvList_xz.data(),recvCount_xz,rank_xz(),recvtag);
req1[12] = Comm.Isend(sendList_Xz.data(),sendCount_Xz,rank_Xz(),sendtag);
req2[12] = Comm.Irecv(recvList_xZ.data(),recvCount_xZ,rank_xZ(),recvtag);
req1[13] = Comm.Isend(sendList_xZ.data(),sendCount_xZ,rank_xZ(),sendtag);
req2[13] = Comm.Irecv(recvList_Xz.data(),recvCount_Xz,rank_Xz(),recvtag);
req1[14] = Comm.Isend(sendList_yz.data(),sendCount_yz,rank_yz(),sendtag);
req2[14] = Comm.Irecv(recvList_YZ.data(),recvCount_YZ,rank_YZ(),recvtag);
req1[15] = Comm.Isend(sendList_YZ.data(),sendCount_YZ,rank_YZ(),sendtag);
req2[15] = Comm.Irecv(recvList_yz.data(),recvCount_yz,rank_yz(),recvtag);
req1[16] = Comm.Isend(sendList_Yz.data(),sendCount_Yz,rank_Yz(),sendtag);
req2[16] = Comm.Irecv(recvList_yZ.data(),recvCount_yZ,rank_yZ(),recvtag);
req1[17] = Comm.Isend(sendList_yZ.data(),sendCount_yZ,rank_yZ(),sendtag);
req2[17] = Comm.Irecv(recvList_Yz.data(),recvCount_Yz,rank_Yz(),recvtag);
Comm.waitAll(18,req1);
Comm.waitAll(18,req2);
//......................................................................................
@ -910,65 +912,7 @@ void Domain::CommInit()
for (int idx=0; idx<recvCount_yZ; idx++) recvList_yZ[idx] -= (Ny-2)*Nx - (Nz-2)*Nx*Ny;
for (int idx=0; idx<recvCount_Yz; idx++) recvList_Yz[idx] += (Ny-2)*Nx - (Nz-2)*Nx*Ny;
//......................................................................................
// allocate recv buffers
recvBuf_x = new int [recvCount_x];
recvBuf_y = new int [recvCount_y];
recvBuf_z = new int [recvCount_z];
recvBuf_X = new int [recvCount_X];
recvBuf_Y = new int [recvCount_Y];
recvBuf_Z = new int [recvCount_Z];
recvBuf_xy = new int [recvCount_xy];
recvBuf_yz = new int [recvCount_yz];
recvBuf_xz = new int [recvCount_xz];
recvBuf_Xy = new int [recvCount_Xy];
recvBuf_Yz = new int [recvCount_Yz];
recvBuf_xZ = new int [recvCount_xZ];
recvBuf_xY = new int [recvCount_xY];
recvBuf_yZ = new int [recvCount_yZ];
recvBuf_Xz = new int [recvCount_Xz];
recvBuf_XY = new int [recvCount_XY];
recvBuf_YZ = new int [recvCount_YZ];
recvBuf_XZ = new int [recvCount_XZ];
//......................................................................................
// send buffers
sendData_x = new double [sendCount_x];
sendData_y = new double [sendCount_y];
sendData_z = new double [sendCount_z];
sendData_X = new double [sendCount_X];
sendData_Y = new double [sendCount_Y];
sendData_Z = new double [sendCount_Z];
sendData_xy = new double [sendCount_xy];
sendData_yz = new double [sendCount_yz];
sendData_xz = new double [sendCount_xz];
sendData_Xy = new double [sendCount_Xy];
sendData_Yz = new double [sendCount_Yz];
sendData_xZ = new double [sendCount_xZ];
sendData_xY = new double [sendCount_xY];
sendData_yZ = new double [sendCount_yZ];
sendData_Xz = new double [sendCount_Xz];
sendData_XY = new double [sendCount_XY];
sendData_YZ = new double [sendCount_YZ];
sendData_XZ = new double [sendCount_XZ];
//......................................................................................
// recv buffers
recvData_x = new double [recvCount_x];
recvData_y = new double [recvCount_y];
recvData_z = new double [recvCount_z];
recvData_X = new double [recvCount_X];
recvData_Y = new double [recvCount_Y];
recvData_Z = new double [recvCount_Z];
recvData_xy = new double [recvCount_xy];
recvData_yz = new double [recvCount_yz];
recvData_xz = new double [recvCount_xz];
recvData_Xy = new double [recvCount_Xy];
recvData_xZ = new double [recvCount_xZ];
recvData_xY = new double [recvCount_xY];
recvData_yZ = new double [recvCount_yZ];
recvData_Yz = new double [recvCount_Yz];
recvData_Xz = new double [recvCount_Xz];
recvData_XY = new double [recvCount_XY];
recvData_YZ = new double [recvCount_YZ];
recvData_XZ = new double [recvCount_XZ];
//......................................................................................
}
@ -988,8 +932,8 @@ void Domain::ReadIDs(){
if (rank()==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n");
sprintf(LocalRankFilename,"ID.%05i",rank());
FILE *IDFILE = fopen(LocalRankFilename,"rb");
if (IDFILE==NULL) ERROR("Domain::ReadIDs -- Error opening file: ID.xxxxx");
readID=fread(id,1,N,IDFILE);
if (!IDFILE) ERROR("Domain::ReadIDs -- Error opening file: ID.xxxxx");
readID=fread(id.data(),1,N,IDFILE);
if (readID != size_t(N)) printf("Domain::ReadIDs -- Error reading ID (rank=%i) \n",rank());
fclose(IDFILE);
@ -1060,62 +1004,115 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh)
int sendtag, recvtag;
sendtag = recvtag = 7;
double *MeshData = Mesh.data();
PackMeshData(sendList_x, sendCount_x ,sendData_x, MeshData);
PackMeshData(sendList_X, sendCount_X ,sendData_X, MeshData);
PackMeshData(sendList_y, sendCount_y ,sendData_y, MeshData);
PackMeshData(sendList_Y, sendCount_Y ,sendData_Y, MeshData);
PackMeshData(sendList_z, sendCount_z ,sendData_z, MeshData);
PackMeshData(sendList_Z, sendCount_Z ,sendData_Z, MeshData);
PackMeshData(sendList_xy, sendCount_xy ,sendData_xy, MeshData);
PackMeshData(sendList_Xy, sendCount_Xy ,sendData_Xy, MeshData);
PackMeshData(sendList_xY, sendCount_xY ,sendData_xY, MeshData);
PackMeshData(sendList_XY, sendCount_XY ,sendData_XY, MeshData);
PackMeshData(sendList_xz, sendCount_xz ,sendData_xz, MeshData);
PackMeshData(sendList_Xz, sendCount_Xz ,sendData_Xz, MeshData);
PackMeshData(sendList_xZ, sendCount_xZ ,sendData_xZ, MeshData);
PackMeshData(sendList_XZ, sendCount_XZ ,sendData_XZ, MeshData);
PackMeshData(sendList_yz, sendCount_yz ,sendData_yz, MeshData);
PackMeshData(sendList_Yz, sendCount_Yz ,sendData_Yz, MeshData);
PackMeshData(sendList_yZ, sendCount_yZ ,sendData_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendData_YZ, MeshData);
//......................................................................................
Comm.sendrecv(sendData_x,sendCount_x,rank_x(),sendtag,recvData_X,recvCount_X,rank_X(),recvtag);
Comm.sendrecv(sendData_X,sendCount_X,rank_X(),sendtag,recvData_x,recvCount_x,rank_x(),recvtag);
Comm.sendrecv(sendData_y,sendCount_y,rank_y(),sendtag,recvData_Y,recvCount_Y,rank_Y(),recvtag);
Comm.sendrecv(sendData_Y,sendCount_Y,rank_Y(),sendtag,recvData_y,recvCount_y,rank_y(),recvtag);
Comm.sendrecv(sendData_z,sendCount_z,rank_z(),sendtag,recvData_Z,recvCount_Z,rank_Z(),recvtag);
Comm.sendrecv(sendData_Z,sendCount_Z,rank_Z(),sendtag,recvData_z,recvCount_z,rank_z(),recvtag);
Comm.sendrecv(sendData_xy,sendCount_xy,rank_xy(),sendtag,recvData_XY,recvCount_XY,rank_XY(),recvtag);
Comm.sendrecv(sendData_XY,sendCount_XY,rank_XY(),sendtag,recvData_xy,recvCount_xy,rank_xy(),recvtag);
Comm.sendrecv(sendData_Xy,sendCount_Xy,rank_Xy(),sendtag,recvData_xY,recvCount_xY,rank_xY(),recvtag);
Comm.sendrecv(sendData_xY,sendCount_xY,rank_xY(),sendtag,recvData_Xy,recvCount_Xy,rank_Xy(),recvtag);
Comm.sendrecv(sendData_xz,sendCount_xz,rank_xz(),sendtag,recvData_XZ,recvCount_XZ,rank_XZ(),recvtag);
Comm.sendrecv(sendData_XZ,sendCount_XZ,rank_XZ(),sendtag,recvData_xz,recvCount_xz,rank_xz(),recvtag);
Comm.sendrecv(sendData_Xz,sendCount_Xz,rank_Xz(),sendtag,recvData_xZ,recvCount_xZ,rank_xZ(),recvtag);
Comm.sendrecv(sendData_xZ,sendCount_xZ,rank_xZ(),sendtag,recvData_Xz,recvCount_Xz,rank_Xz(),recvtag);
Comm.sendrecv(sendData_yz,sendCount_yz,rank_yz(),sendtag,recvData_YZ,recvCount_YZ,rank_YZ(),recvtag);
Comm.sendrecv(sendData_YZ,sendCount_YZ,rank_YZ(),sendtag,recvData_yz,recvCount_yz,rank_yz(),recvtag);
Comm.sendrecv(sendData_Yz,sendCount_Yz,rank_Yz(),sendtag,recvData_yZ,recvCount_yZ,rank_yZ(),recvtag);
Comm.sendrecv(sendData_yZ,sendCount_yZ,rank_yZ(),sendtag,recvData_Yz,recvCount_Yz,rank_Yz(),recvtag);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvData_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvData_X, MeshData);
UnpackMeshData(recvList_y, recvCount_y ,recvData_y, MeshData);
UnpackMeshData(recvList_Y, recvCount_Y ,recvData_Y, MeshData);
UnpackMeshData(recvList_z, recvCount_z ,recvData_z, MeshData);
UnpackMeshData(recvList_Z, recvCount_Z ,recvData_Z, MeshData);
UnpackMeshData(recvList_xy, recvCount_xy ,recvData_xy, MeshData);
UnpackMeshData(recvList_Xy, recvCount_Xy ,recvData_Xy, MeshData);
UnpackMeshData(recvList_xY, recvCount_xY ,recvData_xY, MeshData);
UnpackMeshData(recvList_XY, recvCount_XY ,recvData_XY, MeshData);
UnpackMeshData(recvList_xz, recvCount_xz ,recvData_xz, MeshData);
UnpackMeshData(recvList_Xz, recvCount_Xz ,recvData_Xz, MeshData);
UnpackMeshData(recvList_xZ, recvCount_xZ ,recvData_xZ, MeshData);
UnpackMeshData(recvList_XZ, recvCount_XZ ,recvData_XZ, MeshData);
UnpackMeshData(recvList_yz, recvCount_yz ,recvData_yz, MeshData);
UnpackMeshData(recvList_Yz, recvCount_Yz ,recvData_Yz, MeshData);
UnpackMeshData(recvList_yZ, recvCount_yZ ,recvData_yZ, MeshData);
UnpackMeshData(recvList_YZ, recvCount_YZ ,recvData_YZ, MeshData);
// send buffers
auto sendData_x = new double [sendCount("x")];
auto sendData_y = new double [sendCount("y")];
auto sendData_z = new double [sendCount("z")];
auto sendData_X = new double [sendCount("X")];
auto sendData_Y = new double [sendCount("Y")];
auto sendData_Z = new double [sendCount("Z")];
auto sendData_xy = new double [sendCount("xy")];
auto sendData_yz = new double [sendCount("yz")];
auto sendData_xz = new double [sendCount("xz")];
auto sendData_Xy = new double [sendCount("Xy")];
auto sendData_Yz = new double [sendCount("Yz")];
auto sendData_xZ = new double [sendCount("xZ")];
auto sendData_xY = new double [sendCount("xY")];
auto sendData_yZ = new double [sendCount("yZ")];
auto sendData_Xz = new double [sendCount("Xz")];
auto sendData_XY = new double [sendCount("XY")];
auto sendData_YZ = new double [sendCount("YZ")];
auto sendData_XZ = new double [sendCount("XZ")];
// recv buffers
auto recvData_x = new double [recvCount("x")];
auto recvData_y = new double [recvCount("y")];
auto recvData_z = new double [recvCount("z")];
auto recvData_X = new double [recvCount("X")];
auto recvData_Y = new double [recvCount("Y")];
auto recvData_Z = new double [recvCount("Z")];
auto recvData_xy = new double [recvCount("xy")];
auto recvData_yz = new double [recvCount("yz")];
auto recvData_xz = new double [recvCount("xz")];
auto recvData_Xy = new double [recvCount("Xy")];
auto recvData_xZ = new double [recvCount("xZ")];
auto recvData_xY = new double [recvCount("xY")];
auto recvData_yZ = new double [recvCount("yZ")];
auto recvData_Yz = new double [recvCount("Yz")];
auto recvData_Xz = new double [recvCount("Xz")];
auto recvData_XY = new double [recvCount("XY")];
auto recvData_YZ = new double [recvCount("YZ")];
auto recvData_XZ = new double [recvCount("XZ")];
// Pack data
PackMeshData(sendList("x"), sendCount("x"), sendData_x, MeshData);
PackMeshData(sendList("X"), sendCount("X"), sendData_X, MeshData);
PackMeshData(sendList("y"), sendCount("y"), sendData_y, MeshData);
PackMeshData(sendList("Y"), sendCount("Y"), sendData_Y, MeshData);
PackMeshData(sendList("z"), sendCount("z"), sendData_z, MeshData);
PackMeshData(sendList("Z"), sendCount("Z"), sendData_Z, MeshData);
PackMeshData(sendList("xy"), sendCount("xy"), sendData_xy, MeshData);
PackMeshData(sendList("Xy"), sendCount("Xy"), sendData_Xy, MeshData);
PackMeshData(sendList("xY"), sendCount("xY"), sendData_xY, MeshData);
PackMeshData(sendList("XY"), sendCount("XY"), sendData_XY, MeshData);
PackMeshData(sendList("xz"), sendCount("xz"), sendData_xz, MeshData);
PackMeshData(sendList("Xz"), sendCount("Xz"), sendData_Xz, MeshData);
PackMeshData(sendList("xZ"), sendCount("xZ"), sendData_xZ, MeshData);
PackMeshData(sendList("XZ"), sendCount("XZ"), sendData_XZ, MeshData);
PackMeshData(sendList("yz"), sendCount("yz"), sendData_yz, MeshData);
PackMeshData(sendList("Yz"), sendCount("Yz"), sendData_Yz, MeshData);
PackMeshData(sendList("yZ"), sendCount("yZ"), sendData_yZ, MeshData);
PackMeshData(sendList("YZ"), sendCount("YZ"), sendData_YZ, MeshData);
// send/recv
Comm.sendrecv(sendData_x,sendCount("x"),rank_x(),sendtag,recvData_X,recvCount("X"),rank_X(),recvtag);
Comm.sendrecv(sendData_X,sendCount("X"),rank_X(),sendtag,recvData_x,recvCount("x"),rank_x(),recvtag);
Comm.sendrecv(sendData_y,sendCount("y"),rank_y(),sendtag,recvData_Y,recvCount("Y"),rank_Y(),recvtag);
Comm.sendrecv(sendData_Y,sendCount("Y"),rank_Y(),sendtag,recvData_y,recvCount("y"),rank_y(),recvtag);
Comm.sendrecv(sendData_z,sendCount("z"),rank_z(),sendtag,recvData_Z,recvCount("Z"),rank_Z(),recvtag);
Comm.sendrecv(sendData_Z,sendCount("Z"),rank_Z(),sendtag,recvData_z,recvCount("z"),rank_z(),recvtag);
Comm.sendrecv(sendData_xy,sendCount("xy"),rank_xy(),sendtag,recvData_XY,recvCount("XY"),rank_XY(),recvtag);
Comm.sendrecv(sendData_XY,sendCount("XY"),rank_XY(),sendtag,recvData_xy,recvCount("xy"),rank_xy(),recvtag);
Comm.sendrecv(sendData_Xy,sendCount("Xy"),rank_Xy(),sendtag,recvData_xY,recvCount("xY"),rank_xY(),recvtag);
Comm.sendrecv(sendData_xY,sendCount("xY"),rank_xY(),sendtag,recvData_Xy,recvCount("Xy"),rank_Xy(),recvtag);
Comm.sendrecv(sendData_xz,sendCount("xz"),rank_xz(),sendtag,recvData_XZ,recvCount("XZ"),rank_XZ(),recvtag);
Comm.sendrecv(sendData_XZ,sendCount("XZ"),rank_XZ(),sendtag,recvData_xz,recvCount("xz"),rank_xz(),recvtag);
Comm.sendrecv(sendData_Xz,sendCount("Xz"),rank_Xz(),sendtag,recvData_xZ,recvCount("xZ"),rank_xZ(),recvtag);
Comm.sendrecv(sendData_xZ,sendCount("xZ"),rank_xZ(),sendtag,recvData_Xz,recvCount("Xz"),rank_Xz(),recvtag);
Comm.sendrecv(sendData_yz,sendCount("yz"),rank_yz(),sendtag,recvData_YZ,recvCount("YZ"),rank_YZ(),recvtag);
Comm.sendrecv(sendData_YZ,sendCount("YZ"),rank_YZ(),sendtag,recvData_yz,recvCount("yz"),rank_yz(),recvtag);
Comm.sendrecv(sendData_Yz,sendCount("Yz"),rank_Yz(),sendtag,recvData_yZ,recvCount("yZ"),rank_yZ(),recvtag);
Comm.sendrecv(sendData_yZ,sendCount("yZ"),rank_yZ(),sendtag,recvData_Yz,recvCount("Yz"),rank_Yz(),recvtag);
// unpack data
UnpackMeshData(recvList("x"), recvCount("x") ,recvData_x, MeshData);
UnpackMeshData(recvList("X"), recvCount("X") ,recvData_X, MeshData);
UnpackMeshData(recvList("y"), recvCount("y") ,recvData_y, MeshData);
UnpackMeshData(recvList("Y"), recvCount("Y") ,recvData_Y, MeshData);
UnpackMeshData(recvList("z"), recvCount("z") ,recvData_z, MeshData);
UnpackMeshData(recvList("Z"), recvCount("Z") ,recvData_Z, MeshData);
UnpackMeshData(recvList("xy"), recvCount("xy") ,recvData_xy, MeshData);
UnpackMeshData(recvList("Xy"), recvCount("Xy") ,recvData_Xy, MeshData);
UnpackMeshData(recvList("xY"), recvCount("xY") ,recvData_xY, MeshData);
UnpackMeshData(recvList("XY"), recvCount("XY") ,recvData_XY, MeshData);
UnpackMeshData(recvList("xz"), recvCount("xz") ,recvData_xz, MeshData);
UnpackMeshData(recvList("Xz"), recvCount("Xz") ,recvData_Xz, MeshData);
UnpackMeshData(recvList("xZ"), recvCount("xZ") ,recvData_xZ, MeshData);
UnpackMeshData(recvList("XZ"), recvCount("XZ") ,recvData_XZ, MeshData);
UnpackMeshData(recvList("yz"), recvCount("yz") ,recvData_yz, MeshData);
UnpackMeshData(recvList("Yz"), recvCount("Yz") ,recvData_Yz, MeshData);
UnpackMeshData(recvList("yZ"), recvCount("yZ") ,recvData_yZ, MeshData);
UnpackMeshData(recvList("YZ"), recvCount("YZ") ,recvData_YZ, MeshData);
// Free sendData
delete [] sendData_x; delete [] sendData_y; delete [] sendData_z;
delete [] sendData_X; delete [] sendData_Y; delete [] sendData_Z;
delete [] sendData_xy; delete [] sendData_xY; delete [] sendData_Xy;
delete [] sendData_XY; delete [] sendData_xz; delete [] sendData_xZ;
delete [] sendData_Xz; delete [] sendData_XZ; delete [] sendData_yz;
delete [] sendData_yZ; delete [] sendData_Yz; delete [] sendData_YZ;
// Free recvData
delete [] recvData_x; delete [] recvData_y; delete [] recvData_z;
delete [] recvData_X; delete [] recvData_Y; delete [] recvData_Z;
delete [] recvData_xy; delete [] recvData_xY; delete [] recvData_Xy;
delete [] recvData_XY; delete [] recvData_xz; delete [] recvData_xZ;
delete [] recvData_Xz; delete [] recvData_XZ; delete [] recvData_yz;
delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ;
}
// Ideally stuff below here should be moved somewhere else -- doesn't really belong here

48
common/Domain.h
View File

@ -5,6 +5,7 @@
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <vector>
#include <math.h>
#include <time.h>
#include <exception>
@ -155,24 +156,14 @@ public: // Public variables (need to create accessors instead)
// Get the actual D3Q19 communication counts (based on location of solid phase)
// Discrete velocity set symmetry implies the sendcount = recvcount
//......................................................................................
int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
//......................................................................................
int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z;
int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ;
int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ;
//......................................................................................
int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ;
inline int recvCount( const char* dir ) const { return getRecvList( dir ).size(); }
inline int sendCount( const char* dir ) const { return getSendList( dir ).size(); }
inline const int* recvList( const char* dir ) const { return getRecvList( dir ).data(); }
inline const int* sendList( const char* dir ) const { return getSendList( dir ).data(); }
//......................................................................................
// Solid indicator function
signed char *id;
std::vector<signed char> id;
void ReadIDs();
void Decomp( const std::string& filename );
@ -189,21 +180,18 @@ private:
//......................................................................................
MPI_Request req1[18], req2[18];
//......................................................................................
std::vector<int> sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z;
std::vector<int> sendList_xy, sendList_yz, sendList_xz, sendList_Xy, sendList_Yz, sendList_xZ;
std::vector<int> sendList_xY, sendList_yZ, sendList_Xz, sendList_XY, sendList_YZ, sendList_XZ;
//......................................................................................
std::vector<int> recvList_x, recvList_y, recvList_z, recvList_X, recvList_Y, recvList_Z;
std::vector<int> recvList_xy, recvList_yz, recvList_xz, recvList_Xy, recvList_Yz, recvList_xZ;
std::vector<int> recvList_xY, recvList_yZ, recvList_Xz, recvList_XY, recvList_YZ, recvList_XZ;
//......................................................................................
const std::vector<int>& getRecvList( const char* dir ) const;
const std::vector<int>& getSendList( const char* dir ) const;
int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z;
int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ;
int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ;
//......................................................................................
int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z;
int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ;
int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ;
//......................................................................................
double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z;
double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ;
double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ;
double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z;
double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ;
double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ;
};

20
common/MPI.cpp
View File

@ -3491,7 +3491,8 @@ void MPI_CLASS::wait( MPI_Request request )
{
PROFILE_START( "wait", profile_level );
MPI_Status status;
int flag = 0;
MPI_Wait( &request, &status );
/*int flag = 0;
int err = MPI_Test( &request, &flag, &status );
MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid
while ( !flag ) {
@ -3499,7 +3500,7 @@ void MPI_CLASS::wait( MPI_Request request )
sched_yield();
// Check if the request has finished
MPI_Test( &request, &flag, &status );
}
}*/
PROFILE_STOP( "wait", profile_level );
}
int MPI_CLASS::waitAny( int count, MPI_Request *request )
@ -3508,8 +3509,9 @@ int MPI_CLASS::waitAny( int count, MPI_Request *request )
return -1;
PROFILE_START( "waitAny", profile_level );
int index = -1;
int flag = 0;
auto status = new MPI_Status[count];
MPI_Waitany( count, request, &index, status );
/*int flag = 0;
int err = MPI_Testany( count, request, &index, &flag, status );
MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid
while ( !flag ) {
@ -3518,7 +3520,7 @@ int MPI_CLASS::waitAny( int count, MPI_Request *request )
// Check if the request has finished
MPI_Testany( count, request, &index, &flag, status );
}
MPI_ASSERT( index >= 0 ); // Check that the index is valid
MPI_ASSERT( index >= 0 ); // Check that the index is valid*/
delete[] status;
PROFILE_STOP( "waitAny", profile_level );
return index;
@ -3528,8 +3530,9 @@ void MPI_CLASS::waitAll( int count, MPI_Request *request )
if ( count == 0 )
return;
PROFILE_START( "waitAll", profile_level );
int flag = 0;
auto status = new MPI_Status[count];
MPI_Waitall( count, request, status );
/*int flag = 0;
int err = MPI_Testall( count, request, &flag, status );
MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid
while ( !flag ) {
@ -3537,7 +3540,7 @@ void MPI_CLASS::waitAll( int count, MPI_Request *request )
sched_yield();
// Check if the request has finished
MPI_Testall( count, request, &flag, status );
}
}*/
PROFILE_STOP( "waitAll", profile_level );
delete[] status;
}
@ -3549,7 +3552,8 @@ std::vector<int> MPI_CLASS::waitSome( int count, MPI_Request *request )
std::vector<int> indicies( count, -1 );
auto *status = new MPI_Status[count];
int outcount = 0;
int err = MPI_Testsome( count, request, &outcount, &indicies[0], status );
MPI_Waitsome( count, request, &outcount, indicies.data(), status );
/*int err = MPI_Testsome( count, request, &outcount, &indicies[0], status );
MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid
MPI_ASSERT( outcount != MPI_UNDEFINED ); // Check that the first call is valid
while ( outcount == 0 ) {
@ -3557,7 +3561,7 @@ std::vector<int> MPI_CLASS::waitSome( int count, MPI_Request *request )
sched_yield();
// Check if the request has finished
MPI_Testsome( count, request, &outcount, &indicies[0], status );
}
}*/
indicies.resize( outcount );
delete[] status;
PROFILE_STOP( "waitSome", profile_level );

230
common/ScaLBL.cpp
View File

@ -32,42 +32,42 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
rank_YZ=Dm->rank_YZ();
rank_yZ=Dm->rank_yZ();
rank_Yz=Dm->rank_Yz();
sendCount_x=Dm->sendCount_x;
sendCount_y=Dm->sendCount_y;
sendCount_z=Dm->sendCount_z;
sendCount_X=Dm->sendCount_X;
sendCount_Y=Dm->sendCount_Y;
sendCount_Z=Dm->sendCount_Z;
sendCount_xy=Dm->sendCount_xy;
sendCount_yz=Dm->sendCount_yz;
sendCount_xz=Dm->sendCount_xz;
sendCount_Xy=Dm->sendCount_Xy;
sendCount_Yz=Dm->sendCount_Yz;
sendCount_xZ=Dm->sendCount_xZ;
sendCount_xY=Dm->sendCount_xY;
sendCount_yZ=Dm->sendCount_yZ;
sendCount_Xz=Dm->sendCount_Xz;
sendCount_XY=Dm->sendCount_XY;
sendCount_YZ=Dm->sendCount_YZ;
sendCount_XZ=Dm->sendCount_XZ;
recvCount_x=Dm->recvCount_x;
recvCount_y=Dm->recvCount_y;
recvCount_z=Dm->recvCount_z;
recvCount_X=Dm->recvCount_X;
recvCount_Y=Dm->recvCount_Y;
recvCount_Z=Dm->recvCount_Z;
recvCount_xy=Dm->recvCount_xy;
recvCount_yz=Dm->recvCount_yz;
recvCount_xz=Dm->recvCount_xz;
recvCount_Xy=Dm->recvCount_Xy;
recvCount_Yz=Dm->recvCount_Yz;
recvCount_xZ=Dm->recvCount_xZ;
recvCount_xY=Dm->recvCount_xY;
recvCount_yZ=Dm->recvCount_yZ;
recvCount_Xz=Dm->recvCount_Xz;
recvCount_XY=Dm->recvCount_XY;
recvCount_YZ=Dm->recvCount_YZ;
recvCount_XZ=Dm->recvCount_XZ;
sendCount_x=Dm->sendCount("x");
sendCount_y=Dm->sendCount("y");
sendCount_z=Dm->sendCount("z");
sendCount_X=Dm->sendCount("X");
sendCount_Y=Dm->sendCount("Y");
sendCount_Z=Dm->sendCount("Z");
sendCount_xy=Dm->sendCount("xy");
sendCount_yz=Dm->sendCount("yz");
sendCount_xz=Dm->sendCount("xz");
sendCount_Xy=Dm->sendCount("Xy");
sendCount_Yz=Dm->sendCount("Yz");
sendCount_xZ=Dm->sendCount("xZ");
sendCount_xY=Dm->sendCount("xY");
sendCount_yZ=Dm->sendCount("yZ");
sendCount_Xz=Dm->sendCount("Xz");
sendCount_XY=Dm->sendCount("XY");
sendCount_YZ=Dm->sendCount("YZ");
sendCount_XZ=Dm->sendCount("XZ");
recvCount_x=Dm->recvCount("x");
recvCount_y=Dm->recvCount("y");
recvCount_z=Dm->recvCount("z");
recvCount_X=Dm->recvCount("X");
recvCount_Y=Dm->recvCount("Y");
recvCount_Z=Dm->recvCount("Z");
recvCount_xy=Dm->recvCount("xy");
recvCount_yz=Dm->recvCount("yz");
recvCount_xz=Dm->recvCount("xz");
recvCount_Xy=Dm->recvCount("Xy");
recvCount_Yz=Dm->recvCount("Yz");
recvCount_xZ=Dm->recvCount("xZ");
recvCount_xY=Dm->recvCount("xY");
recvCount_yZ=Dm->recvCount("yZ");
recvCount_Xz=Dm->recvCount("Xz");
recvCount_XY=Dm->recvCount("XY");
recvCount_YZ=Dm->recvCount("YZ");
recvCount_XZ=Dm->recvCount("XZ");
iproc = Dm->iproc();
jproc = Dm->jproc();
@ -174,43 +174,43 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
ScaLBL_AllocateZeroCopy((void **) &dvcRecvDist_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
//......................................................................................
ScaLBL_CopyToZeroCopy(dvcSendList_x,Dm->sendList_x,sendCount_x*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_X,Dm->sendList_X,sendCount_X*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_y,Dm->sendList_y,sendCount_y*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Y,Dm->sendList_Y,sendCount_Y*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_z,Dm->sendList_z,sendCount_z*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Z,Dm->sendList_Z,sendCount_Z*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_xy,Dm->sendList_xy,sendCount_xy*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_XY,Dm->sendList_XY,sendCount_XY*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_xY,Dm->sendList_xY,sendCount_xY*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Xy,Dm->sendList_Xy,sendCount_Xy*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_xz,Dm->sendList_xz,sendCount_xz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_XZ,Dm->sendList_XZ,sendCount_XZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_xZ,Dm->sendList_xZ,sendCount_xZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Xz,Dm->sendList_Xz,sendCount_Xz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_yz,Dm->sendList_yz,sendCount_yz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_YZ,Dm->sendList_YZ,sendCount_YZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_yZ,Dm->sendList_yZ,sendCount_yZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Yz,Dm->sendList_Yz,sendCount_Yz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_x,Dm->sendList("x"),sendCount_x*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_X,Dm->sendList("X"),sendCount_X*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_y,Dm->sendList("y"),sendCount_y*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Y,Dm->sendList("Y"),sendCount_Y*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_z,Dm->sendList("z"),sendCount_z*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Z,Dm->sendList("Z"),sendCount_Z*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_xy,Dm->sendList("xy"),sendCount_xy*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_XY,Dm->sendList("XY"),sendCount_XY*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_xY,Dm->sendList("xY"),sendCount_xY*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Xy,Dm->sendList("Xy"),sendCount_Xy*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_xz,Dm->sendList("xz"),sendCount_xz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_XZ,Dm->sendList("XZ"),sendCount_XZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_xZ,Dm->sendList("xZ"),sendCount_xZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Xz,Dm->sendList("Xz"),sendCount_Xz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_yz,Dm->sendList("yz"),sendCount_yz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_YZ,Dm->sendList("YZ"),sendCount_YZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_yZ,Dm->sendList("yZ"),sendCount_yZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcSendList_Yz,Dm->sendList("Yz"),sendCount_Yz*sizeof(int));
//......................................................................................
ScaLBL_CopyToZeroCopy(dvcRecvList_x,Dm->recvList_x,recvCount_x*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_X,Dm->recvList_X,recvCount_X*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_y,Dm->recvList_y,recvCount_y*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Y,Dm->recvList_Y,recvCount_Y*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_z,Dm->recvList_z,recvCount_z*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Z,Dm->recvList_Z,recvCount_Z*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_xy,Dm->recvList_xy,recvCount_xy*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_XY,Dm->recvList_XY,recvCount_XY*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_xY,Dm->recvList_xY,recvCount_xY*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Xy,Dm->recvList_Xy,recvCount_Xy*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_xz,Dm->recvList_xz,recvCount_xz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_XZ,Dm->recvList_XZ,recvCount_XZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_xZ,Dm->recvList_xZ,recvCount_xZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Xz,Dm->recvList_Xz,recvCount_Xz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_yz,Dm->recvList_yz,recvCount_yz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_YZ,Dm->recvList_YZ,recvCount_YZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_yZ,Dm->recvList_yZ,recvCount_yZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm->recvList_Yz,recvCount_Yz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_x,Dm->recvList("x"),recvCount_x*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_X,Dm->recvList("X"),recvCount_X*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_y,Dm->recvList("y"),recvCount_y*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Y,Dm->recvList("Y"),recvCount_Y*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_z,Dm->recvList("z"),recvCount_z*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Z,Dm->recvList("Z"),recvCount_Z*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_xy,Dm->recvList("xy"),recvCount_xy*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_XY,Dm->recvList("XY"),recvCount_XY*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_xY,Dm->recvList("xY"),recvCount_xY*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Xy,Dm->recvList("Xy"),recvCount_Xy*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_xz,Dm->recvList("xz"),recvCount_xz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_XZ,Dm->recvList("XZ"),recvCount_XZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_xZ,Dm->recvList("xZ"),recvCount_xZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Xz,Dm->recvList("Xz"),recvCount_Xz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_yz,Dm->recvList("yz"),recvCount_yz*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_YZ,Dm->recvList("YZ"),recvCount_YZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_yZ,Dm->recvList("yZ"),recvCount_yZ*sizeof(int));
ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm->recvList("Yz"),recvCount_Yz*sizeof(int));
//......................................................................................
MPI_COMM_SCALBL.barrier();
@ -219,70 +219,70 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
// Set up the recieve distribution lists
//...................................................................................
//...Map recieve list for the X face: q=2,8,10,12,14 .................................
D3Q19_MapRecv(-1,0,0,Dm->recvList_X,0,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,-1,0,Dm->recvList_X,recvCount_X,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,1,0,Dm->recvList_X,2*recvCount_X,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,0,-1,Dm->recvList_X,3*recvCount_X,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,0,1,Dm->recvList_X,4*recvCount_X,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,0,0, Dm->recvList("X"),0,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,-1,0,Dm->recvList("X"),recvCount_X,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,1,0, Dm->recvList("X"),2*recvCount_X,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,0,-1,Dm->recvList("X"),3*recvCount_X,recvCount_X,dvcRecvDist_X);
D3Q19_MapRecv(-1,0,1, Dm->recvList("X"),4*recvCount_X,recvCount_X,dvcRecvDist_X);
//...................................................................................
//...Map recieve list for the x face: q=1,7,9,11,13..................................
D3Q19_MapRecv(1,0,0,Dm->recvList_x,0,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,1,0,Dm->recvList_x,recvCount_x,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,-1,0,Dm->recvList_x,2*recvCount_x,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,0,1,Dm->recvList_x,3*recvCount_x,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,0,-1,Dm->recvList_x,4*recvCount_x,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,0,0, Dm->recvList("x"),0,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,1,0, Dm->recvList("x"),recvCount_x,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,-1,0,Dm->recvList("x"),2*recvCount_x,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,0,1, Dm->recvList("x"),3*recvCount_x,recvCount_x,dvcRecvDist_x);
D3Q19_MapRecv(1,0,-1,Dm->recvList("x"),4*recvCount_x,recvCount_x,dvcRecvDist_x);
//...................................................................................
//...Map recieve list for the y face: q=4,8,9,16,18 ...................................
D3Q19_MapRecv(0,-1,0,Dm->recvList_Y,0,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(-1,-1,0,Dm->recvList_Y,recvCount_Y,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(1,-1,0,Dm->recvList_Y,2*recvCount_Y,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(0,-1,-1,Dm->recvList_Y,3*recvCount_Y,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(0,-1,1,Dm->recvList_Y,4*recvCount_Y,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(0,-1,0, Dm->recvList("Y"),0,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(-1,-1,0,Dm->recvList("Y"),recvCount_Y,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(1,-1,0, Dm->recvList("Y"),2*recvCount_Y,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(0,-1,-1,Dm->recvList("Y"),3*recvCount_Y,recvCount_Y,dvcRecvDist_Y);
D3Q19_MapRecv(0,-1,1, Dm->recvList("Y"),4*recvCount_Y,recvCount_Y,dvcRecvDist_Y);
//...................................................................................
//...Map recieve list for the Y face: q=3,7,10,15,17 ..................................
D3Q19_MapRecv(0,1,0,Dm->recvList_y,0,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(1,1,0,Dm->recvList_y,recvCount_y,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(-1,1,0,Dm->recvList_y,2*recvCount_y,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(0,1,1,Dm->recvList_y,3*recvCount_y,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(0,1,-1,Dm->recvList_y,4*recvCount_y,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(0,1,0, Dm->recvList("y"),0,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(1,1,0, Dm->recvList("y"),recvCount_y,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(-1,1,0,Dm->recvList("y"),2*recvCount_y,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(0,1,1, Dm->recvList("y"),3*recvCount_y,recvCount_y,dvcRecvDist_y);
D3Q19_MapRecv(0,1,-1,Dm->recvList("y"),4*recvCount_y,recvCount_y,dvcRecvDist_y);
//...................................................................................
//...Map recieve list for the z face<<<6,12,13,16,17)..............................................
D3Q19_MapRecv(0,0,-1,Dm->recvList_Z,0,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(-1,0,-1,Dm->recvList_Z,recvCount_Z,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(1,0,-1,Dm->recvList_Z,2*recvCount_Z,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(0,-1,-1,Dm->recvList_Z,3*recvCount_Z,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(0,1,-1,Dm->recvList_Z,4*recvCount_Z,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(0,0,-1, Dm->recvList("Z"),0,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(-1,0,-1,Dm->recvList("Z"),recvCount_Z,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(1,0,-1, Dm->recvList("Z"),2*recvCount_Z,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(0,-1,-1,Dm->recvList("Z"),3*recvCount_Z,recvCount_Z,dvcRecvDist_Z);
D3Q19_MapRecv(0,1,-1, Dm->recvList("Z"),4*recvCount_Z,recvCount_Z,dvcRecvDist_Z);
//...Map recieve list for the Z face<<<5,11,14,15,18)..............................................
D3Q19_MapRecv(0,0,1,Dm->recvList_z,0,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(1,0,1,Dm->recvList_z,recvCount_z,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(-1,0,1,Dm->recvList_z,2*recvCount_z,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(0,1,1,Dm->recvList_z,3*recvCount_z,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(0,-1,1,Dm->recvList_z,4*recvCount_z,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(0,0,1, Dm->recvList("z"),0,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(1,0,1, Dm->recvList("z"),recvCount_z,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(-1,0,1,Dm->recvList("z"),2*recvCount_z,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(0,1,1, Dm->recvList("z"),3*recvCount_z,recvCount_z,dvcRecvDist_z);
D3Q19_MapRecv(0,-1,1,Dm->recvList("z"),4*recvCount_z,recvCount_z,dvcRecvDist_z);
//..................................................................................
//...Map recieve list for the xy edge <<<8)................................
D3Q19_MapRecv(-1,-1,0,Dm->recvList_XY,0,recvCount_XY,dvcRecvDist_XY);
D3Q19_MapRecv(-1,-1,0,Dm->recvList("XY"),0,recvCount_XY,dvcRecvDist_XY);
//...Map recieve list for the Xy edge <<<9)................................
D3Q19_MapRecv(1,-1,0,Dm->recvList_xY,0,recvCount_xY,dvcRecvDist_xY);
D3Q19_MapRecv(1,-1,0,Dm->recvList("xY"),0,recvCount_xY,dvcRecvDist_xY);
//...Map recieve list for the xY edge <<<10)................................
D3Q19_MapRecv(-1,1,0,Dm->recvList_Xy,0,recvCount_Xy,dvcRecvDist_Xy);
D3Q19_MapRecv(-1,1,0,Dm->recvList("Xy"),0,recvCount_Xy,dvcRecvDist_Xy);
//...Map recieve list for the XY edge <<<7)................................
D3Q19_MapRecv(1,1,0,Dm->recvList_xy,0,recvCount_xy,dvcRecvDist_xy);
D3Q19_MapRecv(1,1,0,Dm->recvList("xy"),0,recvCount_xy,dvcRecvDist_xy);
//...Map recieve list for the xz edge <<<12)................................
D3Q19_MapRecv(-1,0,-1,Dm->recvList_XZ,0,recvCount_XZ,dvcRecvDist_XZ);
D3Q19_MapRecv(-1,0,-1,Dm->recvList("XZ"),0,recvCount_XZ,dvcRecvDist_XZ);
//...Map recieve list for the xZ edge <<<14)................................
D3Q19_MapRecv(-1,0,1,Dm->recvList_Xz,0,recvCount_Xz,dvcRecvDist_Xz);
D3Q19_MapRecv(-1,0,1,Dm->recvList("Xz"),0,recvCount_Xz,dvcRecvDist_Xz);
//...Map recieve list for the Xz edge <<<13)................................
D3Q19_MapRecv(1,0,-1,Dm->recvList_xZ,0,recvCount_xZ,dvcRecvDist_xZ);
D3Q19_MapRecv(1,0,-1,Dm->recvList("xZ"),0,recvCount_xZ,dvcRecvDist_xZ);
//...Map recieve list for the XZ edge <<<11)................................
D3Q19_MapRecv(1,0,1,Dm->recvList_xz,0,recvCount_xz,dvcRecvDist_xz);
D3Q19_MapRecv(1,0,1,Dm->recvList("xz"),0,recvCount_xz,dvcRecvDist_xz);
//...Map recieve list for the yz edge <<<16)................................
D3Q19_MapRecv(0,-1,-1,Dm->recvList_YZ,0,recvCount_YZ,dvcRecvDist_YZ);
D3Q19_MapRecv(0,-1,-1,Dm->recvList("YZ"),0,recvCount_YZ,dvcRecvDist_YZ);
//...Map recieve list for the yZ edge <<<18)................................
D3Q19_MapRecv(0,-1,1,Dm->recvList_Yz,0,recvCount_Yz,dvcRecvDist_Yz);
D3Q19_MapRecv(0,-1,1,Dm->recvList("Yz"),0,recvCount_Yz,dvcRecvDist_Yz);
//...Map recieve list for the Yz edge <<<17)................................
D3Q19_MapRecv(0,1,-1,Dm->recvList_yZ,0,recvCount_yZ,dvcRecvDist_yZ);
D3Q19_MapRecv(0,1,-1,Dm->recvList("yZ"),0,recvCount_yZ,dvcRecvDist_yZ);
//...Map recieve list for the YZ edge <<<15)................................
D3Q19_MapRecv(0,1,1,Dm->recvList_yz,0,recvCount_yz,dvcRecvDist_yz);
D3Q19_MapRecv(0,1,1,Dm->recvList("yz"),0,recvCount_yz,dvcRecvDist_yz);
//...................................................................................
//......................................................................................
@ -319,7 +319,7 @@ int ScaLBL_Communicator::LastInterior(){
return last_interior;
}
void ScaLBL_Communicator::D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list, int start, int count,
void ScaLBL_Communicator::D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, const int *list, int start, int count,
int *d3q19_recvlist){
int i,j,k,n,nn,idx;
int * ReturnDist;

2
common/ScaLBL.h
View File

@ -195,7 +195,7 @@ public:
private:
//void D3Q19_MapRecv_OLD(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, int *d3q19_recvlist);
void D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list, int start, int count, int *d3q19_recvlist);
void D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, const int *list, int start, int count, int *d3q19_recvlist);
bool Lock; // use Lock to make sure only one call at a time to protect data in transit
// only one set of Send requests can be active at any time (per instance)

4
models/ColorModel.cpp
View File

@ -199,7 +199,7 @@ void ScaLBL_ColorModel::ReadInput(){
ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 );
fillHalo<signed char> fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 );
Array<signed char> id_view;
id_view.viewRaw( size1, Mask->id );
id_view.viewRaw( size1, Mask->id.data() );
fill.copy( input_id, id_view );
fill.fill( id_view );
}
@ -332,7 +332,7 @@ void ScaLBL_ColorModel::Create(){
if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N);
Map.resize(Nx,Ny,Nz); Map.fill(-2);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np);
comm.barrier();
//...........................................................................

2
models/DFHModel.cpp
View File

@ -205,7 +205,7 @@ void ScaLBL_DFHModel::Create(){
if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N);
Map.resize(Nx,Ny,Nz); Map.fill(-2);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np);
comm.barrier();
//...........................................................................

2
models/MRTModel.cpp
View File

@ -151,7 +151,7 @@ void ScaLBL_MRTModel::Create(){
if (rank==0) printf ("Set up memory efficient layout \n");
Map.resize(Nx,Ny,Nz); Map.fill(-2);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np);
comm.barrier();
//...........................................................................
// MAIN VARIABLES ALLOCATED HERE

184
tests/GenerateSphereTest.cpp
View File

@ -20,7 +20,7 @@
using namespace std;
inline void PackID(int *list, int count, char *sendbuf, char *ID){
inline void PackID(const int *list, int count, char *sendbuf, char *ID){
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
int idx,n;
@ -32,7 +32,7 @@ inline void PackID(int *list, int count, char *sendbuf, char *ID){
}
//***************************************************************************************
inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
inline void UnpackID(const int *list, int count, char *recvbuf, char *ID){
// Fill in the phase ID values from neighboring processors
// This unpacks the values once they have been recieved from neighbors
int idx,n;
@ -91,44 +91,44 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
// send buffers
sendID_x = new char [Dm.sendCount_x];
sendID_y = new char [Dm.sendCount_y];
sendID_z = new char [Dm.sendCount_z];
sendID_X = new char [Dm.sendCount_X];
sendID_Y = new char [Dm.sendCount_Y];
sendID_Z = new char [Dm.sendCount_Z];
sendID_xy = new char [Dm.sendCount_xy];
sendID_yz = new char [Dm.sendCount_yz];
sendID_xz = new char [Dm.sendCount_xz];
sendID_Xy = new char [Dm.sendCount_Xy];
sendID_Yz = new char [Dm.sendCount_Yz];
sendID_xZ = new char [Dm.sendCount_xZ];
sendID_xY = new char [Dm.sendCount_xY];
sendID_yZ = new char [Dm.sendCount_yZ];
sendID_Xz = new char [Dm.sendCount_Xz];
sendID_XY = new char [Dm.sendCount_XY];
sendID_YZ = new char [Dm.sendCount_YZ];
sendID_XZ = new char [Dm.sendCount_XZ];
sendID_x = new char [Dm.sendCount("x")];
sendID_y = new char [Dm.sendCount("y")];
sendID_z = new char [Dm.sendCount("z")];
sendID_X = new char [Dm.sendCount("X")];
sendID_Y = new char [Dm.sendCount("Y")];
sendID_Z = new char [Dm.sendCount("Z")];
sendID_xy = new char [Dm.sendCount("xy")];
sendID_yz = new char [Dm.sendCount("yz")];
sendID_xz = new char [Dm.sendCount("xz")];
sendID_Xy = new char [Dm.sendCount("Xy")];
sendID_Yz = new char [Dm.sendCount("Yz")];
sendID_xZ = new char [Dm.sendCount("xZ")];
sendID_xY = new char [Dm.sendCount("xY")];
sendID_yZ = new char [Dm.sendCount("yZ")];
sendID_Xz = new char [Dm.sendCount("Xz")];
sendID_XY = new char [Dm.sendCount("XY")];
sendID_YZ = new char [Dm.sendCount("YZ")];
sendID_XZ = new char [Dm.sendCount("XZ")];
//......................................................................................
// recv buffers
recvID_x = new char [Dm.recvCount_x];
recvID_y = new char [Dm.recvCount_y];
recvID_z = new char [Dm.recvCount_z];
recvID_X = new char [Dm.recvCount_X];
recvID_Y = new char [Dm.recvCount_Y];
recvID_Z = new char [Dm.recvCount_Z];
recvID_xy = new char [Dm.recvCount_xy];
recvID_yz = new char [Dm.recvCount_yz];
recvID_xz = new char [Dm.recvCount_xz];
recvID_Xy = new char [Dm.recvCount_Xy];
recvID_xZ = new char [Dm.recvCount_xZ];
recvID_xY = new char [Dm.recvCount_xY];
recvID_yZ = new char [Dm.recvCount_yZ];
recvID_Yz = new char [Dm.recvCount_Yz];
recvID_Xz = new char [Dm.recvCount_Xz];
recvID_XY = new char [Dm.recvCount_XY];
recvID_YZ = new char [Dm.recvCount_YZ];
recvID_XZ = new char [Dm.recvCount_XZ];
recvID_x = new char [Dm.recvCount("x")];
recvID_y = new char [Dm.recvCount("y")];
recvID_z = new char [Dm.recvCount("z")];
recvID_X = new char [Dm.recvCount("X")];
recvID_Y = new char [Dm.recvCount("Y")];
recvID_Z = new char [Dm.recvCount("Z")];
recvID_xy = new char [Dm.recvCount("xy")];
recvID_yz = new char [Dm.recvCount("yz")];
recvID_xz = new char [Dm.recvCount("xz")];
recvID_Xy = new char [Dm.recvCount("Xy")];
recvID_xZ = new char [Dm.recvCount("xZ")];
recvID_xY = new char [Dm.recvCount("xY")];
recvID_yZ = new char [Dm.recvCount("yZ")];
recvID_Yz = new char [Dm.recvCount("Yz")];
recvID_Xz = new char [Dm.recvCount("Xz")];
recvID_XY = new char [Dm.recvCount("XY")];
recvID_YZ = new char [Dm.recvCount("YZ")];
recvID_XZ = new char [Dm.recvCount("XZ")];
//......................................................................................
int sendtag,recvtag;
sendtag = recvtag = 7;
@ -194,62 +194,62 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
// Pack and send the updated ID values
PackID(Dm.sendList_x, Dm.sendCount_x ,sendID_x, id);
PackID(Dm.sendList_X, Dm.sendCount_X ,sendID_X, id);
PackID(Dm.sendList_y, Dm.sendCount_y ,sendID_y, id);
PackID(Dm.sendList_Y, Dm.sendCount_Y ,sendID_Y, id);
PackID(Dm.sendList_z, Dm.sendCount_z ,sendID_z, id);
PackID(Dm.sendList_Z, Dm.sendCount_Z ,sendID_Z, id);
PackID(Dm.sendList_xy, Dm.sendCount_xy ,sendID_xy, id);
PackID(Dm.sendList_Xy, Dm.sendCount_Xy ,sendID_Xy, id);
PackID(Dm.sendList_xY, Dm.sendCount_xY ,sendID_xY, id);
PackID(Dm.sendList_XY, Dm.sendCount_XY ,sendID_XY, id);
PackID(Dm.sendList_xz, Dm.sendCount_xz ,sendID_xz, id);
PackID(Dm.sendList_Xz, Dm.sendCount_Xz ,sendID_Xz, id);
PackID(Dm.sendList_xZ, Dm.sendCount_xZ ,sendID_xZ, id);
PackID(Dm.sendList_XZ, Dm.sendCount_XZ ,sendID_XZ, id);
PackID(Dm.sendList_yz, Dm.sendCount_yz ,sendID_yz, id);
PackID(Dm.sendList_Yz, Dm.sendCount_Yz ,sendID_Yz, id);
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
PackID(Dm.sendList("x"), Dm.sendCount("x") ,sendID_x, id);
PackID(Dm.sendList("X"), Dm.sendCount("X") ,sendID_X, id);
PackID(Dm.sendList("y"), Dm.sendCount("y") ,sendID_y, id);
PackID(Dm.sendList("Y"), Dm.sendCount("Y") ,sendID_Y, id);
PackID(Dm.sendList("z"), Dm.sendCount("z") ,sendID_z, id);
PackID(Dm.sendList("Z"), Dm.sendCount("Z") ,sendID_Z, id);
PackID(Dm.sendList("xy"), Dm.sendCount("xy") ,sendID_xy, id);
PackID(Dm.sendList("Xy"), Dm.sendCount("Xy") ,sendID_Xy, id);
PackID(Dm.sendList("xY"), Dm.sendCount("xY") ,sendID_xY, id);
PackID(Dm.sendList("XY"), Dm.sendCount("XY") ,sendID_XY, id);
PackID(Dm.sendList("xz"), Dm.sendCount("xz") ,sendID_xz, id);
PackID(Dm.sendList("Xz"), Dm.sendCount("Xz") ,sendID_Xz, id);
PackID(Dm.sendList("xZ"), Dm.sendCount("xZ") ,sendID_xZ, id);
PackID(Dm.sendList("XZ"), Dm.sendCount("XZ") ,sendID_XZ, id);
PackID(Dm.sendList("yz"), Dm.sendCount("yz") ,sendID_yz, id);
PackID(Dm.sendList("Yz"), Dm.sendCount("Yz") ,sendID_Yz, id);
PackID(Dm.sendList("yZ"), Dm.sendCount("yZ") ,sendID_yZ, id);
PackID(Dm.sendList("YZ"), Dm.sendCount("YZ") ,sendID_YZ, id);
//......................................................................................
Dm.Comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag);
Dm.Comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag);
Dm.Comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag);
Dm.Comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag);
Dm.Comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag);
Dm.Comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag);
Dm.Comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag);
Dm.Comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag);
Dm.Comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag);
Dm.Comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag);
Dm.Comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag);
Dm.Comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag);
Dm.Comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag);
Dm.Comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag);
Dm.Comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag);
Dm.Comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag);
Dm.Comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag);
Dm.Comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag);
Dm.Comm.sendrecv(sendID_x,Dm.sendCount("x"),Dm.rank_x(),sendtag,recvID_X,Dm.recvCount("X"),Dm.rank_X(),recvtag);
Dm.Comm.sendrecv(sendID_X,Dm.sendCount("X"),Dm.rank_X(),sendtag,recvID_x,Dm.recvCount("x"),Dm.rank_x(),recvtag);
Dm.Comm.sendrecv(sendID_y,Dm.sendCount("y"),Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount("Y"),Dm.rank_Y(),recvtag);
Dm.Comm.sendrecv(sendID_Y,Dm.sendCount("Y"),Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount("y"),Dm.rank_y(),recvtag);
Dm.Comm.sendrecv(sendID_z,Dm.sendCount("z"),Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount("Z"),Dm.rank_Z(),recvtag);
Dm.Comm.sendrecv(sendID_Z,Dm.sendCount("Z"),Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount("z"),Dm.rank_z(),recvtag);
Dm.Comm.sendrecv(sendID_xy,Dm.sendCount("xy"),Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount("XY"),Dm.rank_XY(),recvtag);
Dm.Comm.sendrecv(sendID_XY,Dm.sendCount("XY"),Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount("xy"),Dm.rank_xy(),recvtag);
Dm.Comm.sendrecv(sendID_Xy,Dm.sendCount("Xy"),Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount("xY"),Dm.rank_xY(),recvtag);
Dm.Comm.sendrecv(sendID_xY,Dm.sendCount("xY"),Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount("Xy"),Dm.rank_Xy(),recvtag);
Dm.Comm.sendrecv(sendID_xz,Dm.sendCount("xz"),Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount("XZ"),Dm.rank_XZ(),recvtag);
Dm.Comm.sendrecv(sendID_XZ,Dm.sendCount("XZ"),Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount("xz"),Dm.rank_xz(),recvtag);
Dm.Comm.sendrecv(sendID_Xz,Dm.sendCount("Xz"),Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount("xZ"),Dm.rank_xZ(),recvtag);
Dm.Comm.sendrecv(sendID_xZ,Dm.sendCount("xZ"),Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount("Xz"),Dm.rank_Xz(),recvtag);
Dm.Comm.sendrecv(sendID_yz,Dm.sendCount("yz"),Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount("YZ"),Dm.rank_YZ(),recvtag);
Dm.Comm.sendrecv(sendID_YZ,Dm.sendCount("YZ"),Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount("yz"),Dm.rank_yz(),recvtag);
Dm.Comm.sendrecv(sendID_Yz,Dm.sendCount("Yz"),Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount("yZ"),Dm.rank_yZ(),recvtag);
Dm.Comm.sendrecv(sendID_yZ,Dm.sendCount("yZ"),Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount("Yz"),Dm.rank_Yz(),recvtag);
//......................................................................................
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);
UnpackID(Dm.recvList_y, Dm.recvCount_y ,recvID_y, id);
UnpackID(Dm.recvList_Y, Dm.recvCount_Y ,recvID_Y, id);
UnpackID(Dm.recvList_z, Dm.recvCount_z ,recvID_z, id);
UnpackID(Dm.recvList_Z, Dm.recvCount_Z ,recvID_Z, id);
UnpackID(Dm.recvList_xy, Dm.recvCount_xy ,recvID_xy, id);
UnpackID(Dm.recvList_Xy, Dm.recvCount_Xy ,recvID_Xy, id);
UnpackID(Dm.recvList_xY, Dm.recvCount_xY ,recvID_xY, id);
UnpackID(Dm.recvList_XY, Dm.recvCount_XY ,recvID_XY, id);
UnpackID(Dm.recvList_xz, Dm.recvCount_xz ,recvID_xz, id);
UnpackID(Dm.recvList_Xz, Dm.recvCount_Xz ,recvID_Xz, id);
UnpackID(Dm.recvList_xZ, Dm.recvCount_xZ ,recvID_xZ, id);
UnpackID(Dm.recvList_XZ, Dm.recvCount_XZ ,recvID_XZ, id);
UnpackID(Dm.recvList_yz, Dm.recvCount_yz ,recvID_yz, id);
UnpackID(Dm.recvList_Yz, Dm.recvCount_Yz ,recvID_Yz, id);
UnpackID(Dm.recvList_yZ, Dm.recvCount_yZ ,recvID_yZ, id);
UnpackID(Dm.recvList_YZ, Dm.recvCount_YZ ,recvID_YZ, id);
UnpackID(Dm.recvList("x"), Dm.recvCount("x") ,recvID_x, id);
UnpackID(Dm.recvList("X"), Dm.recvCount("X") ,recvID_X, id);
UnpackID(Dm.recvList("y"), Dm.recvCount("y") ,recvID_y, id);
UnpackID(Dm.recvList("Y"), Dm.recvCount("Y") ,recvID_Y, id);
UnpackID(Dm.recvList("z"), Dm.recvCount("z") ,recvID_z, id);
UnpackID(Dm.recvList("Z"), Dm.recvCount("Z") ,recvID_Z, id);
UnpackID(Dm.recvList("xy"), Dm.recvCount("xy") ,recvID_xy, id);
UnpackID(Dm.recvList("Xy"), Dm.recvCount("Xy") ,recvID_Xy, id);
UnpackID(Dm.recvList("xY"), Dm.recvCount("xY") ,recvID_xY, id);
UnpackID(Dm.recvList("XY"), Dm.recvCount("XY") ,recvID_XY, id);
UnpackID(Dm.recvList("xz"), Dm.recvCount("xz") ,recvID_xz, id);
UnpackID(Dm.recvList("Xz"), Dm.recvCount("Xz") ,recvID_Xz, id);
UnpackID(Dm.recvList("xZ"), Dm.recvCount("xZ") ,recvID_xZ, id);
UnpackID(Dm.recvList("XZ"), Dm.recvCount("XZ") ,recvID_XZ, id);
UnpackID(Dm.recvList("yz"), Dm.recvCount("yz") ,recvID_yz, id);
UnpackID(Dm.recvList("Yz"), Dm.recvCount("Yz") ,recvID_Yz, id);
UnpackID(Dm.recvList("yZ"), Dm.recvCount("yZ") ,recvID_yZ, id);
UnpackID(Dm.recvList("YZ"), Dm.recvCount("YZ") ,recvID_YZ, id);
//......................................................................................
//double GlobalNumber = Dm.Comm.sumReduce( LocalNumber );

2
tests/TestBubbleDFH.cpp
View File

@ -248,7 +248,7 @@ int main(int argc, char **argv)
if (rank==0) printf ("Set up memory efficient layout Npad=%i \n",Npad);
IntArray Map(Nx,Ny,Nz);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np);
comm.barrier();
//...........................................................................

2
tests/TestColorGradDFH.cpp
View File

@ -100,7 +100,7 @@ int main(int argc, char **argv)
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np);
comm.barrier();
//......................device distributions.................................

2
tests/TestCommD3Q19.cpp
View File

@ -283,7 +283,7 @@ int main(int argc, char **argv)
auto neighborList= new int[18*Npad];
IntArray Map(Nx,Ny,Nz);
Map.fill(-2);
Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np);
comm.barrier();
int neighborSize=18*Np*sizeof(int);
//......................device distributions.................................

2
tests/TestFluxBC.cpp
View File

@ -88,7 +88,7 @@ int main (int argc, char **argv)
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np);
comm.barrier();
//......................device distributions.................................

4
tests/TestForceMoments.cpp
View File

@ -162,7 +162,7 @@ int main(int argc, char **argv)
char *ID;
ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Copy to the device
ScaLBL_CopyToDevice(ID, Dm->id, N);
ScaLBL_CopyToDevice(ID, Dm->id.data(), N);
//...........................................................................
if (rank==0){
@ -179,7 +179,7 @@ int main(int argc, char **argv)
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Np];
ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np);
if (rank == 0) PrintNeighborList(neighborList,Np, rank);

2
tests/TestMap.cpp
View File

@ -88,7 +88,7 @@ int main(int argc, char **argv)
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np);
comm.barrier();
// Check the neighborlist

6
tests/TestPressVel.cpp
View File

@ -106,13 +106,13 @@ int main(int argc, char **argv)
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device
auto ScaLBL_Comm = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Dm));
auto ScaLBL_Comm = std::make_shared<ScaLBL_Communicator>( Dm );
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
char *ID;
ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Copy to the device
ScaLBL_CopyToDevice(ID, Dm->id, N);
ScaLBL_CopyToDevice(ID, Dm->id.data(), N);
//...........................................................................
if (rank==0){
@ -127,7 +127,7 @@ int main(int argc, char **argv)
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np);
comm.barrier();
//......................device distributions.................................

2
tests/lbpm_minkowski_scalar.cpp
View File

@ -159,7 +159,7 @@ int main(int argc, char **argv)
else{
// Recieve the subdomain from rank = 0
printf("Ready to recieve data %i at process %i \n", N,rank);
comm.recv(Dm->id,N,0,15);
comm.recv(Dm->id.data(),N,0,15);
}
comm.barrier();