From 94156e066e4ff02ba2a8f38e15bee0617982bacc Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Thu, 12 Mar 2020 10:26:17 -0400 Subject: [PATCH] Trying to fix bug with MPI communication --- analysis/morphology.cpp | 382 +++++++++---------- analysis/runAnalysis.cpp | 41 ++- analysis/runAnalysis.h | 4 +- common/Communication.h | 4 +- common/Domain.cpp | 627 ++++++++++++++++---------------- common/Domain.h | 48 +-- common/MPI.cpp | 20 +- common/ScaLBL.cpp | 230 ++++++------ common/ScaLBL.h | 2 +- models/ColorModel.cpp | 4 +- models/DFHModel.cpp | 2 +- models/MRTModel.cpp | 2 +- tests/GenerateSphereTest.cpp | 184 +++++----- tests/TestBubbleDFH.cpp | 2 +- tests/TestColorGradDFH.cpp | 2 +- tests/TestCommD3Q19.cpp | 2 +- tests/TestFluxBC.cpp | 2 +- tests/TestForceMoments.cpp | 4 +- tests/TestMap.cpp | 2 +- tests/TestPressVel.cpp | 6 +- tests/lbpm_minkowski_scalar.cpp | 2 +- 21 files changed, 777 insertions(+), 795 deletions(-) diff --git a/analysis/morphology.cpp b/analysis/morphology.cpp index 84ed3652..0980a4f0 100644 --- a/analysis/morphology.cpp +++ b/analysis/morphology.cpp @@ -1,7 +1,7 @@ #include // Implementation of morphological opening routine -inline void PackID(int *list, int count, signed char *sendbuf, signed char *ID){ +inline void PackID(const int *list, int count, signed char *sendbuf, signed char *ID){ // Fill in the phase ID values from neighboring processors // This packs up the values that need to be sent from one processor to another int idx,n; @@ -13,7 +13,7 @@ inline void PackID(int *list, int count, signed char *sendbuf, signed char *ID){ } //*************************************************************************************** -inline void UnpackID(int *list, int count, signed char *recvbuf, signed char *ID){ +inline void UnpackID(const int *list, int count, signed char *recvbuf, signed char *ID){ // Fill in the phase ID values from neighboring processors // This unpacks the values once they have been recieved from neighbors int idx,n; @@ -77,44 +77,44 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ; signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ; // send buffers - sendID_x = new signed char [Dm->sendCount_x]; - sendID_y = new signed char [Dm->sendCount_y]; - sendID_z = new signed char [Dm->sendCount_z]; - sendID_X = new signed char [Dm->sendCount_X]; - sendID_Y = new signed char [Dm->sendCount_Y]; - sendID_Z = new signed char [Dm->sendCount_Z]; - sendID_xy = new signed char [Dm->sendCount_xy]; - sendID_yz = new signed char [Dm->sendCount_yz]; - sendID_xz = new signed char [Dm->sendCount_xz]; - sendID_Xy = new signed char [Dm->sendCount_Xy]; - sendID_Yz = new signed char [Dm->sendCount_Yz]; - sendID_xZ = new signed char [Dm->sendCount_xZ]; - sendID_xY = new signed char [Dm->sendCount_xY]; - sendID_yZ = new signed char [Dm->sendCount_yZ]; - sendID_Xz = new signed char [Dm->sendCount_Xz]; - sendID_XY = new signed char [Dm->sendCount_XY]; - sendID_YZ = new signed char [Dm->sendCount_YZ]; - sendID_XZ = new signed char [Dm->sendCount_XZ]; + sendID_x = new signed char [Dm->sendCount("x")]; + sendID_y = new signed char [Dm->sendCount("y")]; + sendID_z = new signed char [Dm->sendCount("z")]; + sendID_X = new signed char [Dm->sendCount("X")]; + sendID_Y = new signed char [Dm->sendCount("Y")]; + sendID_Z = new signed char [Dm->sendCount("Z")]; + sendID_xy = new signed char [Dm->sendCount("xy")]; + sendID_yz = new signed char [Dm->sendCount("yz")]; + sendID_xz = new signed char [Dm->sendCount("xz")]; + sendID_Xy = new signed char [Dm->sendCount("Xy")]; + sendID_Yz = new signed char [Dm->sendCount("Yz")]; + sendID_xZ = new signed char [Dm->sendCount("xZ")]; + sendID_xY = new signed char [Dm->sendCount("xY")]; + sendID_yZ = new signed char [Dm->sendCount("yZ")]; + sendID_Xz = new signed char [Dm->sendCount("Xz")]; + sendID_XY = new signed char [Dm->sendCount("XY")]; + sendID_YZ = new signed char [Dm->sendCount("YZ")]; + sendID_XZ = new signed char [Dm->sendCount("XZ")]; //...................................................................................... // recv buffers - recvID_x = new signed char [Dm->recvCount_x]; - recvID_y = new signed char [Dm->recvCount_y]; - recvID_z = new signed char [Dm->recvCount_z]; - recvID_X = new signed char [Dm->recvCount_X]; - recvID_Y = new signed char [Dm->recvCount_Y]; - recvID_Z = new signed char [Dm->recvCount_Z]; - recvID_xy = new signed char [Dm->recvCount_xy]; - recvID_yz = new signed char [Dm->recvCount_yz]; - recvID_xz = new signed char [Dm->recvCount_xz]; - recvID_Xy = new signed char [Dm->recvCount_Xy]; - recvID_xZ = new signed char [Dm->recvCount_xZ]; - recvID_xY = new signed char [Dm->recvCount_xY]; - recvID_yZ = new signed char [Dm->recvCount_yZ]; - recvID_Yz = new signed char [Dm->recvCount_Yz]; - recvID_Xz = new signed char [Dm->recvCount_Xz]; - recvID_XY = new signed char [Dm->recvCount_XY]; - recvID_YZ = new signed char [Dm->recvCount_YZ]; - recvID_XZ = new signed char [Dm->recvCount_XZ]; + recvID_x = new signed char [Dm->recvCount("x")]; + recvID_y = new signed char [Dm->recvCount("y")]; + recvID_z = new signed char [Dm->recvCount("z")]; + recvID_X = new signed char [Dm->recvCount("X")]; + recvID_Y = new signed char [Dm->recvCount("Y")]; + recvID_Z = new signed char [Dm->recvCount("Z")]; + recvID_xy = new signed char [Dm->recvCount("xy")]; + recvID_yz = new signed char [Dm->recvCount("yz")]; + recvID_xz = new signed char [Dm->recvCount("xz")]; + recvID_Xy = new signed char [Dm->recvCount("Xy")]; + recvID_xZ = new signed char [Dm->recvCount("xZ")]; + recvID_xY = new signed char [Dm->recvCount("xY")]; + recvID_yZ = new signed char [Dm->recvCount("yZ")]; + recvID_Yz = new signed char [Dm->recvCount("Yz")]; + recvID_Xz = new signed char [Dm->recvCount("Xz")]; + recvID_XY = new signed char [Dm->recvCount("XY")]; + recvID_YZ = new signed char [Dm->recvCount("YZ")]; + recvID_XZ = new signed char [Dm->recvCount("XZ")]; //...................................................................................... int sendtag,recvtag; sendtag = recvtag = 7; @@ -182,62 +182,62 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr } } // Pack and send the updated ID values - PackID(Dm->sendList_x, Dm->sendCount_x ,sendID_x, id); - PackID(Dm->sendList_X, Dm->sendCount_X ,sendID_X, id); - PackID(Dm->sendList_y, Dm->sendCount_y ,sendID_y, id); - PackID(Dm->sendList_Y, Dm->sendCount_Y ,sendID_Y, id); - PackID(Dm->sendList_z, Dm->sendCount_z ,sendID_z, id); - PackID(Dm->sendList_Z, Dm->sendCount_Z ,sendID_Z, id); - PackID(Dm->sendList_xy, Dm->sendCount_xy ,sendID_xy, id); - PackID(Dm->sendList_Xy, Dm->sendCount_Xy ,sendID_Xy, id); - PackID(Dm->sendList_xY, Dm->sendCount_xY ,sendID_xY, id); - PackID(Dm->sendList_XY, Dm->sendCount_XY ,sendID_XY, id); - PackID(Dm->sendList_xz, Dm->sendCount_xz ,sendID_xz, id); - PackID(Dm->sendList_Xz, Dm->sendCount_Xz ,sendID_Xz, id); - PackID(Dm->sendList_xZ, Dm->sendCount_xZ ,sendID_xZ, id); - PackID(Dm->sendList_XZ, Dm->sendCount_XZ ,sendID_XZ, id); - PackID(Dm->sendList_yz, Dm->sendCount_yz ,sendID_yz, id); - PackID(Dm->sendList_Yz, Dm->sendCount_Yz ,sendID_Yz, id); - PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id); - PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id); + PackID(Dm->sendList("x"), Dm->sendCount("x") ,sendID_x, id); + PackID(Dm->sendList("X"), Dm->sendCount("X") ,sendID_X, id); + PackID(Dm->sendList("y"), Dm->sendCount("y") ,sendID_y, id); + PackID(Dm->sendList("Y"), Dm->sendCount("Y") ,sendID_Y, id); + PackID(Dm->sendList("z"), Dm->sendCount("z") ,sendID_z, id); + PackID(Dm->sendList("Z"), Dm->sendCount("Z") ,sendID_Z, id); + PackID(Dm->sendList("xy"), Dm->sendCount("xy") ,sendID_xy, id); + PackID(Dm->sendList("Xy"), Dm->sendCount("Xy") ,sendID_Xy, id); + PackID(Dm->sendList("xY"), Dm->sendCount("xY") ,sendID_xY, id); + PackID(Dm->sendList("XY"), Dm->sendCount("XY") ,sendID_XY, id); + PackID(Dm->sendList("xz"), Dm->sendCount("xz") ,sendID_xz, id); + PackID(Dm->sendList("Xz"), Dm->sendCount("Xz") ,sendID_Xz, id); + PackID(Dm->sendList("xZ"), Dm->sendCount("xZ") ,sendID_xZ, id); + PackID(Dm->sendList("XZ"), Dm->sendCount("XZ") ,sendID_XZ, id); + PackID(Dm->sendList("yz"), Dm->sendCount("yz") ,sendID_yz, id); + PackID(Dm->sendList("Yz"), Dm->sendCount("Yz") ,sendID_Yz, id); + PackID(Dm->sendList("yZ"), Dm->sendCount("yZ") ,sendID_yZ, id); + PackID(Dm->sendList("YZ"), Dm->sendCount("YZ") ,sendID_YZ, id); //...................................................................................... - Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag); - Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag); - Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag); - Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag); - Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag); - Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag); - Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag); - Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag); - Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag); - Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag); - Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag); - Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag); - Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag); - Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag); - Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag); - Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag); - Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag); - Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag); + Dm->Comm.sendrecv(sendID_x,Dm->sendCount("x"),Dm->rank_x(),sendtag,recvID_X,Dm->recvCount("X"),Dm->rank_X(),recvtag); + Dm->Comm.sendrecv(sendID_X,Dm->sendCount("X"),Dm->rank_X(),sendtag,recvID_x,Dm->recvCount("x"),Dm->rank_x(),recvtag); + Dm->Comm.sendrecv(sendID_y,Dm->sendCount("y"),Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount("Y"),Dm->rank_Y(),recvtag); + Dm->Comm.sendrecv(sendID_Y,Dm->sendCount("Y"),Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount("y"),Dm->rank_y(),recvtag); + Dm->Comm.sendrecv(sendID_z,Dm->sendCount("z"),Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount("Z"),Dm->rank_Z(),recvtag); + Dm->Comm.sendrecv(sendID_Z,Dm->sendCount("Z"),Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount("z"),Dm->rank_z(),recvtag); + Dm->Comm.sendrecv(sendID_xy,Dm->sendCount("xy"),Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount("XY"),Dm->rank_XY(),recvtag); + Dm->Comm.sendrecv(sendID_XY,Dm->sendCount("XY"),Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount("xy"),Dm->rank_xy(),recvtag); + Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount("Xy"),Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount("xY"),Dm->rank_xY(),recvtag); + Dm->Comm.sendrecv(sendID_xY,Dm->sendCount("xY"),Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount("Xy"),Dm->rank_Xy(),recvtag); + Dm->Comm.sendrecv(sendID_xz,Dm->sendCount("xz"),Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount("XZ"),Dm->rank_XZ(),recvtag); + Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount("XZ"),Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount("xz"),Dm->rank_xz(),recvtag); + Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount("Xz"),Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount("xZ"),Dm->rank_xZ(),recvtag); + Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount("xZ"),Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount("Xz"),Dm->rank_Xz(),recvtag); + Dm->Comm.sendrecv(sendID_yz,Dm->sendCount("yz"),Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount("YZ"),Dm->rank_YZ(),recvtag); + Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount("YZ"),Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount("yz"),Dm->rank_yz(),recvtag); + Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount("Yz"),Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount("yZ"),Dm->rank_yZ(),recvtag); + Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount("yZ"),Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount("Yz"),Dm->rank_Yz(),recvtag); //...................................................................................... - UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id); - UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id); - UnpackID(Dm->recvList_y, Dm->recvCount_y ,recvID_y, id); - UnpackID(Dm->recvList_Y, Dm->recvCount_Y ,recvID_Y, id); - UnpackID(Dm->recvList_z, Dm->recvCount_z ,recvID_z, id); - UnpackID(Dm->recvList_Z, Dm->recvCount_Z ,recvID_Z, id); - UnpackID(Dm->recvList_xy, Dm->recvCount_xy ,recvID_xy, id); - UnpackID(Dm->recvList_Xy, Dm->recvCount_Xy ,recvID_Xy, id); - UnpackID(Dm->recvList_xY, Dm->recvCount_xY ,recvID_xY, id); - UnpackID(Dm->recvList_XY, Dm->recvCount_XY ,recvID_XY, id); - UnpackID(Dm->recvList_xz, Dm->recvCount_xz ,recvID_xz, id); - UnpackID(Dm->recvList_Xz, Dm->recvCount_Xz ,recvID_Xz, id); - UnpackID(Dm->recvList_xZ, Dm->recvCount_xZ ,recvID_xZ, id); - UnpackID(Dm->recvList_XZ, Dm->recvCount_XZ ,recvID_XZ, id); - UnpackID(Dm->recvList_yz, Dm->recvCount_yz ,recvID_yz, id); - UnpackID(Dm->recvList_Yz, Dm->recvCount_Yz ,recvID_Yz, id); - UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id); - UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id); + UnpackID(Dm->recvList("x"), Dm->recvCount("x") ,recvID_x, id); + UnpackID(Dm->recvList("X"), Dm->recvCount("X") ,recvID_X, id); + UnpackID(Dm->recvList("y"), Dm->recvCount("y") ,recvID_y, id); + UnpackID(Dm->recvList("Y"), Dm->recvCount("Y") ,recvID_Y, id); + UnpackID(Dm->recvList("z"), Dm->recvCount("z") ,recvID_z, id); + UnpackID(Dm->recvList("Z"), Dm->recvCount("Z") ,recvID_Z, id); + UnpackID(Dm->recvList("xy"), Dm->recvCount("xy") ,recvID_xy, id); + UnpackID(Dm->recvList("Xy"), Dm->recvCount("Xy") ,recvID_Xy, id); + UnpackID(Dm->recvList("xY"), Dm->recvCount("xY") ,recvID_xY, id); + UnpackID(Dm->recvList("XY"), Dm->recvCount("XY") ,recvID_XY, id); + UnpackID(Dm->recvList("xz"), Dm->recvCount("xz") ,recvID_xz, id); + UnpackID(Dm->recvList("Xz"), Dm->recvCount("Xz") ,recvID_Xz, id); + UnpackID(Dm->recvList("xZ"), Dm->recvCount("xZ") ,recvID_xZ, id); + UnpackID(Dm->recvList("XZ"), Dm->recvCount("XZ") ,recvID_XZ, id); + UnpackID(Dm->recvList("yz"), Dm->recvCount("yz") ,recvID_yz, id); + UnpackID(Dm->recvList("Yz"), Dm->recvCount("Yz") ,recvID_Yz, id); + UnpackID(Dm->recvList("yZ"), Dm->recvCount("yZ") ,recvID_yZ, id); + UnpackID(Dm->recvList("YZ"), Dm->recvCount("YZ") ,recvID_YZ, id); //...................................................................................... //double GlobalNumber = Dm->Comm.sumReduce( LocalNumber ); @@ -359,44 +359,44 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrsendCount_x]; - sendID_y = new signed char [Dm->sendCount_y]; - sendID_z = new signed char [Dm->sendCount_z]; - sendID_X = new signed char [Dm->sendCount_X]; - sendID_Y = new signed char [Dm->sendCount_Y]; - sendID_Z = new signed char [Dm->sendCount_Z]; - sendID_xy = new signed char [Dm->sendCount_xy]; - sendID_yz = new signed char [Dm->sendCount_yz]; - sendID_xz = new signed char [Dm->sendCount_xz]; - sendID_Xy = new signed char [Dm->sendCount_Xy]; - sendID_Yz = new signed char [Dm->sendCount_Yz]; - sendID_xZ = new signed char [Dm->sendCount_xZ]; - sendID_xY = new signed char [Dm->sendCount_xY]; - sendID_yZ = new signed char [Dm->sendCount_yZ]; - sendID_Xz = new signed char [Dm->sendCount_Xz]; - sendID_XY = new signed char [Dm->sendCount_XY]; - sendID_YZ = new signed char [Dm->sendCount_YZ]; - sendID_XZ = new signed char [Dm->sendCount_XZ]; + sendID_x = new signed char [Dm->sendCount("x")]; + sendID_y = new signed char [Dm->sendCount("y")]; + sendID_z = new signed char [Dm->sendCount("z")]; + sendID_X = new signed char [Dm->sendCount("X")]; + sendID_Y = new signed char [Dm->sendCount("Y")]; + sendID_Z = new signed char [Dm->sendCount("Z")]; + sendID_xy = new signed char [Dm->sendCount("xy")]; + sendID_yz = new signed char [Dm->sendCount("yz")]; + sendID_xz = new signed char [Dm->sendCount("xz")]; + sendID_Xy = new signed char [Dm->sendCount("Xy")]; + sendID_Yz = new signed char [Dm->sendCount("Yz")]; + sendID_xZ = new signed char [Dm->sendCount("xZ")]; + sendID_xY = new signed char [Dm->sendCount("xY")]; + sendID_yZ = new signed char [Dm->sendCount("yZ")]; + sendID_Xz = new signed char [Dm->sendCount("Xz")]; + sendID_XY = new signed char [Dm->sendCount("XY")]; + sendID_YZ = new signed char [Dm->sendCount("YZ")]; + sendID_XZ = new signed char [Dm->sendCount("XZ")]; //...................................................................................... // recv buffers - recvID_x = new signed char [Dm->recvCount_x]; - recvID_y = new signed char [Dm->recvCount_y]; - recvID_z = new signed char [Dm->recvCount_z]; - recvID_X = new signed char [Dm->recvCount_X]; - recvID_Y = new signed char [Dm->recvCount_Y]; - recvID_Z = new signed char [Dm->recvCount_Z]; - recvID_xy = new signed char [Dm->recvCount_xy]; - recvID_yz = new signed char [Dm->recvCount_yz]; - recvID_xz = new signed char [Dm->recvCount_xz]; - recvID_Xy = new signed char [Dm->recvCount_Xy]; - recvID_xZ = new signed char [Dm->recvCount_xZ]; - recvID_xY = new signed char [Dm->recvCount_xY]; - recvID_yZ = new signed char [Dm->recvCount_yZ]; - recvID_Yz = new signed char [Dm->recvCount_Yz]; - recvID_Xz = new signed char [Dm->recvCount_Xz]; - recvID_XY = new signed char [Dm->recvCount_XY]; - recvID_YZ = new signed char [Dm->recvCount_YZ]; - recvID_XZ = new signed char [Dm->recvCount_XZ]; + recvID_x = new signed char [Dm->recvCount("x")]; + recvID_y = new signed char [Dm->recvCount("y")]; + recvID_z = new signed char [Dm->recvCount("z")]; + recvID_X = new signed char [Dm->recvCount("X")]; + recvID_Y = new signed char [Dm->recvCount("Y")]; + recvID_Z = new signed char [Dm->recvCount("Z")]; + recvID_xy = new signed char [Dm->recvCount("xy")]; + recvID_yz = new signed char [Dm->recvCount("yz")]; + recvID_xz = new signed char [Dm->recvCount("xz")]; + recvID_Xy = new signed char [Dm->recvCount("Xy")]; + recvID_xZ = new signed char [Dm->recvCount("xZ")]; + recvID_xY = new signed char [Dm->recvCount("xY")]; + recvID_yZ = new signed char [Dm->recvCount("yZ")]; + recvID_Yz = new signed char [Dm->recvCount("Yz")]; + recvID_Xz = new signed char [Dm->recvCount("Xz")]; + recvID_XY = new signed char [Dm->recvCount("XY")]; + recvID_YZ = new signed char [Dm->recvCount("YZ")]; + recvID_XZ = new signed char [Dm->recvCount("XZ")]; //...................................................................................... int sendtag,recvtag; sendtag = recvtag = 7; @@ -469,80 +469,62 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrsendList_x, Dm->sendCount_x ,sendID_x, id); - PackID(Dm->sendList_X, Dm->sendCount_X ,sendID_X, id); - PackID(Dm->sendList_y, Dm->sendCount_y ,sendID_y, id); - PackID(Dm->sendList_Y, Dm->sendCount_Y ,sendID_Y, id); - PackID(Dm->sendList_z, Dm->sendCount_z ,sendID_z, id); - PackID(Dm->sendList_Z, Dm->sendCount_Z ,sendID_Z, id); - PackID(Dm->sendList_xy, Dm->sendCount_xy ,sendID_xy, id); - PackID(Dm->sendList_Xy, Dm->sendCount_Xy ,sendID_Xy, id); - PackID(Dm->sendList_xY, Dm->sendCount_xY ,sendID_xY, id); - PackID(Dm->sendList_XY, Dm->sendCount_XY ,sendID_XY, id); - PackID(Dm->sendList_xz, Dm->sendCount_xz ,sendID_xz, id); - PackID(Dm->sendList_Xz, Dm->sendCount_Xz ,sendID_Xz, id); - PackID(Dm->sendList_xZ, Dm->sendCount_xZ ,sendID_xZ, id); - PackID(Dm->sendList_XZ, Dm->sendCount_XZ ,sendID_XZ, id); - PackID(Dm->sendList_yz, Dm->sendCount_yz ,sendID_yz, id); - PackID(Dm->sendList_Yz, Dm->sendCount_Yz ,sendID_Yz, id); - PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id); - PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id); + PackID(Dm->sendList("x"), Dm->sendCount("x") ,sendID_x, id); + PackID(Dm->sendList("X"), Dm->sendCount("X") ,sendID_X, id); + PackID(Dm->sendList("y"), Dm->sendCount("y") ,sendID_y, id); + PackID(Dm->sendList("Y"), Dm->sendCount("Y") ,sendID_Y, id); + PackID(Dm->sendList("z"), Dm->sendCount("z") ,sendID_z, id); + PackID(Dm->sendList("Z"), Dm->sendCount("Z") ,sendID_Z, id); + PackID(Dm->sendList("xy"), Dm->sendCount("xy") ,sendID_xy, id); + PackID(Dm->sendList("Xy"), Dm->sendCount("Xy") ,sendID_Xy, id); + PackID(Dm->sendList("xY"), Dm->sendCount("xY") ,sendID_xY, id); + PackID(Dm->sendList("XY"), Dm->sendCount("XY") ,sendID_XY, id); + PackID(Dm->sendList("xz"), Dm->sendCount("xz") ,sendID_xz, id); + PackID(Dm->sendList("Xz"), Dm->sendCount("Xz") ,sendID_Xz, id); + PackID(Dm->sendList("xZ"), Dm->sendCount("xZ") ,sendID_xZ, id); + PackID(Dm->sendList("XZ"), Dm->sendCount("XZ") ,sendID_XZ, id); + PackID(Dm->sendList("yz"), Dm->sendCount("yz") ,sendID_yz, id); + PackID(Dm->sendList("Yz"), Dm->sendCount("Yz") ,sendID_Yz, id); + PackID(Dm->sendList("yZ"), Dm->sendCount("yZ") ,sendID_yZ, id); + PackID(Dm->sendList("YZ"), Dm->sendCount("YZ") ,sendID_YZ, id); //...................................................................................... - Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag, - recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag); - Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag, - recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag); - Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag, - recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag); - Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag, - recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag); - Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag, - recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag); - Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag, - recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag); - Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag, - recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag); - Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag, - recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag); - Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag, - recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag); - Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag, - recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag); - Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag, - recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag); - Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag, - recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag); - Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag, - recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag); - Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag, - recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag); - Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag, - recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag); - Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag, - recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag); - Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag, - recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag); - Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag, - recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag); + Dm->Comm.sendrecv(sendID_x,Dm->sendCount("x"),Dm->rank_x(),sendtag,recvID_X,Dm->recvCount("X"),Dm->rank_X(),recvtag); + Dm->Comm.sendrecv(sendID_X,Dm->sendCount("X"),Dm->rank_X(),sendtag,recvID_x,Dm->recvCount("x"),Dm->rank_x(),recvtag); + Dm->Comm.sendrecv(sendID_y,Dm->sendCount("y"),Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount("Y"),Dm->rank_Y(),recvtag); + Dm->Comm.sendrecv(sendID_Y,Dm->sendCount("Y"),Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount("y"),Dm->rank_y(),recvtag); + Dm->Comm.sendrecv(sendID_z,Dm->sendCount("z"),Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount("Z"),Dm->rank_Z(),recvtag); + Dm->Comm.sendrecv(sendID_Z,Dm->sendCount("Z"),Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount("z"),Dm->rank_z(),recvtag); + Dm->Comm.sendrecv(sendID_xy,Dm->sendCount("xy"),Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount("XY"),Dm->rank_XY(),recvtag); + Dm->Comm.sendrecv(sendID_XY,Dm->sendCount("XY"),Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount("xy"),Dm->rank_xy(),recvtag); + Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount("Xy"),Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount("xY"),Dm->rank_xY(),recvtag); + Dm->Comm.sendrecv(sendID_xY,Dm->sendCount("xY"),Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount("Xy"),Dm->rank_Xy(),recvtag); + Dm->Comm.sendrecv(sendID_xz,Dm->sendCount("xz"),Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount("XZ"),Dm->rank_XZ(),recvtag); + Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount("XZ"),Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount("xz"),Dm->rank_xz(),recvtag); + Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount("Xz"),Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount("xZ"),Dm->rank_xZ(),recvtag); + Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount("xZ"),Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount("Xz"),Dm->rank_Xz(),recvtag); + Dm->Comm.sendrecv(sendID_yz,Dm->sendCount("yz"),Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount("YZ"),Dm->rank_YZ(),recvtag); + Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount("YZ"),Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount("yz"),Dm->rank_yz(),recvtag); + Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount("Yz"),Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount("yZ"),Dm->rank_yZ(),recvtag); + Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount("yZ"),Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount("Yz"),Dm->rank_Yz(),recvtag); //...................................................................................... - UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id); - UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id); - UnpackID(Dm->recvList_y, Dm->recvCount_y ,recvID_y, id); - UnpackID(Dm->recvList_Y, Dm->recvCount_Y ,recvID_Y, id); - UnpackID(Dm->recvList_z, Dm->recvCount_z ,recvID_z, id); - UnpackID(Dm->recvList_Z, Dm->recvCount_Z ,recvID_Z, id); - UnpackID(Dm->recvList_xy, Dm->recvCount_xy ,recvID_xy, id); - UnpackID(Dm->recvList_Xy, Dm->recvCount_Xy ,recvID_Xy, id); - UnpackID(Dm->recvList_xY, Dm->recvCount_xY ,recvID_xY, id); - UnpackID(Dm->recvList_XY, Dm->recvCount_XY ,recvID_XY, id); - UnpackID(Dm->recvList_xz, Dm->recvCount_xz ,recvID_xz, id); - UnpackID(Dm->recvList_Xz, Dm->recvCount_Xz ,recvID_Xz, id); - UnpackID(Dm->recvList_xZ, Dm->recvCount_xZ ,recvID_xZ, id); - UnpackID(Dm->recvList_XZ, Dm->recvCount_XZ ,recvID_XZ, id); - UnpackID(Dm->recvList_yz, Dm->recvCount_yz ,recvID_yz, id); - UnpackID(Dm->recvList_Yz, Dm->recvCount_Yz ,recvID_Yz, id); - UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id); - UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id); + UnpackID(Dm->recvList("x"), Dm->recvCount("x") ,recvID_x, id); + UnpackID(Dm->recvList("X"), Dm->recvCount("X") ,recvID_X, id); + UnpackID(Dm->recvList("y"), Dm->recvCount("y") ,recvID_y, id); + UnpackID(Dm->recvList("Y"), Dm->recvCount("Y") ,recvID_Y, id); + UnpackID(Dm->recvList("z"), Dm->recvCount("z") ,recvID_z, id); + UnpackID(Dm->recvList("Z"), Dm->recvCount("Z") ,recvID_Z, id); + UnpackID(Dm->recvList("xy"), Dm->recvCount("xy") ,recvID_xy, id); + UnpackID(Dm->recvList("Xy"), Dm->recvCount("Xy") ,recvID_Xy, id); + UnpackID(Dm->recvList("xY"), Dm->recvCount("xY") ,recvID_xY, id); + UnpackID(Dm->recvList("XY"), Dm->recvCount("XY") ,recvID_XY, id); + UnpackID(Dm->recvList("xz"), Dm->recvCount("xz") ,recvID_xz, id); + UnpackID(Dm->recvList("Xz"), Dm->recvCount("Xz") ,recvID_Xz, id); + UnpackID(Dm->recvList("xZ"), Dm->recvCount("xZ") ,recvID_xZ, id); + UnpackID(Dm->recvList("XZ"), Dm->recvCount("XZ") ,recvID_XZ, id); + UnpackID(Dm->recvList("yz"), Dm->recvCount("yz") ,recvID_yz, id); + UnpackID(Dm->recvList("Yz"), Dm->recvCount("Yz") ,recvID_Yz, id); + UnpackID(Dm->recvList("yZ"), Dm->recvCount("yZ") ,recvID_yZ, id); + UnpackID(Dm->recvList("YZ"), Dm->recvCount("YZ") ,recvID_YZ, id); //...................................................................................... // double GlobalNumber = Dm->Comm.sumReduce( LocalNumber ); diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 89451c7b..83153f6c 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -510,7 +510,7 @@ runAnalysis::commWrapper runAnalysis::getComm( ) runAnalysis::runAnalysis( std::shared_ptr input_db, const RankInfoStruct& rank_info, std::shared_ptr ScaLBL_Comm, - std::shared_ptr Dm, + std::shared_ptr Dm, int Np, bool Regular, IntArray Map ): @@ -518,8 +518,7 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, d_regular ( Regular), d_rank_info( rank_info ), d_Map( Map ), - d_fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1), - d_comm( Utilities::MPI( MPI_COMM_WORLD ).dup() ), + d_comm( Dm->Comm.dup() ), d_ScaLBL_Comm( ScaLBL_Comm) { @@ -535,6 +534,9 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, char rankString[20]; sprintf(rankString,"%05d",Dm->rank()); + d_n[0] = Dm->Nx-2; + d_n[1] = Dm->Ny-2; + d_n[2] = Dm->Nz-2; d_N[0] = Dm->Nx; d_N[1] = Dm->Ny; d_N[2] = Dm->Nz; @@ -566,7 +568,7 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, d_meshData.resize(1); d_meshData[0].meshName = "domain"; - d_meshData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + d_meshData[0].mesh = std::make_shared( d_rank_info,d_n[0],d_n[1],d_n[2],Dm->Lx,Dm->Ly,Dm->Lz ); auto PhaseVar = std::make_shared(); auto PressVar = std::make_shared(); auto VxVar = std::make_shared(); @@ -579,7 +581,7 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, PhaseVar->name = "phase"; PhaseVar->type = IO::VariableType::VolumeVariable; PhaseVar->dim = 1; - PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + PhaseVar->data.resize(d_n[0],d_n[1],d_n[2]); d_meshData[0].vars.push_back(PhaseVar); } @@ -587,7 +589,7 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, PressVar->name = "Pressure"; PressVar->type = IO::VariableType::VolumeVariable; PressVar->dim = 1; - PressVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + PressVar->data.resize(d_n[0],d_n[1],d_n[2]); d_meshData[0].vars.push_back(PressVar); } @@ -595,17 +597,17 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, VxVar->name = "Velocity_x"; VxVar->type = IO::VariableType::VolumeVariable; VxVar->dim = 1; - VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + VxVar->data.resize(d_n[0],d_n[1],d_n[2]); d_meshData[0].vars.push_back(VxVar); VyVar->name = "Velocity_y"; VyVar->type = IO::VariableType::VolumeVariable; VyVar->dim = 1; - VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + VyVar->data.resize(d_n[0],d_n[1],d_n[2]); d_meshData[0].vars.push_back(VyVar); VzVar->name = "Velocity_z"; VzVar->type = IO::VariableType::VolumeVariable; VzVar->dim = 1; - VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + VzVar->data.resize(d_n[0],d_n[1],d_n[2]); d_meshData[0].vars.push_back(VzVar); } @@ -613,7 +615,7 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, SignDistVar->name = "SignDist"; SignDistVar->type = IO::VariableType::VolumeVariable; SignDistVar->dim = 1; - SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + SignDistVar->data.resize(d_n[0],d_n[1],d_n[2]); d_meshData[0].vars.push_back(SignDistVar); } @@ -621,7 +623,7 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, BlobIDVar->name = "BlobID"; BlobIDVar->type = IO::VariableType::VolumeVariable; BlobIDVar->dim = 1; - BlobIDVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + BlobIDVar->data.resize(d_n[0],d_n[1],d_n[2]); d_meshData[0].vars.push_back(BlobIDVar); } @@ -682,7 +684,10 @@ void runAnalysis::createThreads( const std::string& method, int N_threads ) } // Create the threads const auto cores = d_tpool.getProcessAffinity(); - if ( cores.empty() ) { + if ( N_threads == 0 ) { + // Special case to serials the analysis for debugging + d_tpool.setNumThreads( 0 ); + } else if ( cores.empty() ) { // We were not able to get the cores for the process d_tpool.setNumThreads( N_threads ); } else if ( method == "default" ) { @@ -921,7 +926,9 @@ void runAnalysis::run(int timestep, std::shared_ptr input_db, TwoPhase // if ( matches(type,AnalysisType::CreateRestart) ) { if (timestep%d_restart_interval==0){ // Write the vis files - auto work = new WriteVisWorkItem( timestep, d_meshData, Averages, d_fillData, getComm() ); + commWrapper comm = getComm(); + fillHalo fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 ); + auto work = new WriteVisWorkItem( timestep, d_meshData, Averages, fillData, std::move( comm ) ); work->add_dependency(d_wait_blobID); work->add_dependency(d_wait_analysis); work->add_dependency(d_wait_vis); @@ -1025,7 +1032,9 @@ void runAnalysis::basic(int timestep, std::shared_ptr input_db, SubPha if (timestep%d_visualization_interval==0){ // Write the vis files - auto work = new IOWorkItem( timestep, input_db, d_meshData, Averages, d_fillData, getComm() ); + commWrapper comm = getComm(); + fillHalo fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 ); + auto work = new IOWorkItem( timestep, input_db, d_meshData, Averages, fillData, std::move( comm ) ); work->add_dependency(d_wait_analysis); work->add_dependency(d_wait_subphase); work->add_dependency(d_wait_vis); @@ -1058,7 +1067,9 @@ void runAnalysis::WriteVisData(int timestep, std::shared_ptr input_db, PROFILE_START("write vis",1); // if (Averages.WriteVis == true){ - auto work2 = new IOWorkItem(timestep, input_db, d_meshData, Averages, d_fillData, getComm() ); + commWrapper comm = getComm(); + fillHalo fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 ); + auto work2 = new IOWorkItem(timestep, input_db, d_meshData, Averages, fillData, std::move( comm ) ); work2->add_dependency(d_wait_vis); d_wait_vis = d_tpool.add_work(work2); diff --git a/analysis/runAnalysis.h b/analysis/runAnalysis.h index 3c5bc7f0..33adbcb0 100644 --- a/analysis/runAnalysis.h +++ b/analysis/runAnalysis.h @@ -84,7 +84,8 @@ public: private: - int d_N[3]; + std::array d_n; // Number of local cells + std::array d_N; // NNumber of local cells with ghosts int d_Np; int d_rank; int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval; @@ -98,7 +99,6 @@ private: BlobIDstruct d_last_index; BlobIDList d_last_id_map; std::vector d_meshData; - fillHalo d_fillData; std::string d_restartFile; Utilities::MPI d_comm; Utilities::MPI d_comms[1024]; diff --git a/common/Communication.h b/common/Communication.h index cf83ffe3..4cd9ad70 100644 --- a/common/Communication.h +++ b/common/Communication.h @@ -102,7 +102,7 @@ private: //*************************************************************************************** -inline void PackMeshData(int *list, int count, double *sendbuf, double *data){ +inline void PackMeshData(const int *list, int count, double *sendbuf, double *data){ // Fill in the phase ID values from neighboring processors // This packs up the values that need to be sent from one processor to another int idx,n; @@ -111,7 +111,7 @@ inline void PackMeshData(int *list, int count, double *sendbuf, double *data){ sendbuf[idx] = data[n]; } } -inline void UnpackMeshData(int *list, int count, double *recvbuf, double *data){ +inline void UnpackMeshData(const int *list, int count, double *recvbuf, double *data){ // Fill in the phase ID values from neighboring processors // This unpacks the values once they have been recieved from neighbors int idx,n; diff --git a/common/Domain.cpp b/common/Domain.cpp index eadda60d..ab457f33 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -23,40 +23,15 @@ static inline void fgetl( char * str, int num, FILE * stream ) } /******************************************************** - * Constructors/Destructor * + * Constructors * ********************************************************/ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, double lx, double ly, double lz, int BC): - database(NULL), Nx(0), Ny(0), Nz(0), + database(nullptr), Nx(0), Ny(0), Nz(0), Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), voxel_length(1), - Comm(MPI_COMM_WORLD), + Comm( Utilities::MPI( MPI_COMM_WORLD).dup() ), inlet_layers_x(0), inlet_layers_y(0), inlet_layers_z(0), - inlet_layers_phase(1),outlet_layers_phase(2), - sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), - sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), - sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), - sendList_x(NULL), sendList_y(NULL), sendList_z(NULL), sendList_X(NULL), sendList_Y(NULL), sendList_Z(NULL), - sendList_xy(NULL), sendList_yz(NULL), sendList_xz(NULL), sendList_Xy(NULL), sendList_Yz(NULL), sendList_xZ(NULL), - sendList_xY(NULL), sendList_yZ(NULL), sendList_Xz(NULL), sendList_XY(NULL), sendList_YZ(NULL), sendList_XZ(NULL), - sendBuf_x(NULL), sendBuf_y(NULL), sendBuf_z(NULL), sendBuf_X(NULL), sendBuf_Y(NULL), sendBuf_Z(NULL), - sendBuf_xy(NULL), sendBuf_yz(NULL), sendBuf_xz(NULL), sendBuf_Xy(NULL), sendBuf_Yz(NULL), sendBuf_xZ(NULL), - sendBuf_xY(NULL), sendBuf_yZ(NULL), sendBuf_Xz(NULL), sendBuf_XY(NULL), sendBuf_YZ(NULL), sendBuf_XZ(NULL), - recvCount_x(0), recvCount_y(0), recvCount_z(0), recvCount_X(0), recvCount_Y(0), recvCount_Z(0), - recvCount_xy(0), recvCount_yz(0), recvCount_xz(0), recvCount_Xy(0), recvCount_Yz(0), recvCount_xZ(0), - recvCount_xY(0), recvCount_yZ(0), recvCount_Xz(0), recvCount_XY(0), recvCount_YZ(0), recvCount_XZ(0), - recvList_x(NULL), recvList_y(NULL), recvList_z(NULL), recvList_X(NULL), recvList_Y(NULL), recvList_Z(NULL), - recvList_xy(NULL), recvList_yz(NULL), recvList_xz(NULL), recvList_Xy(NULL), recvList_Yz(NULL), recvList_xZ(NULL), - recvList_xY(NULL), recvList_yZ(NULL), recvList_Xz(NULL), recvList_XY(NULL), recvList_YZ(NULL), recvList_XZ(NULL), - recvBuf_x(NULL), recvBuf_y(NULL), recvBuf_z(NULL), recvBuf_X(NULL), recvBuf_Y(NULL), recvBuf_Z(NULL), - recvBuf_xy(NULL), recvBuf_yz(NULL), recvBuf_xz(NULL), recvBuf_Xy(NULL), recvBuf_Yz(NULL), recvBuf_xZ(NULL), - recvBuf_xY(NULL), recvBuf_yZ(NULL), recvBuf_Xz(NULL), recvBuf_XY(NULL), recvBuf_YZ(NULL), recvBuf_XZ(NULL), - sendData_x(NULL), sendData_y(NULL), sendData_z(NULL), sendData_X(NULL), sendData_Y(NULL), sendData_Z(NULL), - sendData_xy(NULL), sendData_yz(NULL), sendData_xz(NULL), sendData_Xy(NULL), sendData_Yz(NULL), sendData_xZ(NULL), - sendData_xY(NULL), sendData_yZ(NULL), sendData_Xz(NULL), sendData_XY(NULL), sendData_YZ(NULL), sendData_XZ(NULL), - recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), - recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), - recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), - id(NULL) + inlet_layers_phase(1),outlet_layers_phase(2) { NULL_USE( rnk ); NULL_USE( npy ); @@ -80,32 +55,7 @@ Domain::Domain( std::shared_ptr db, const Utilities::MPI& Communicator Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), inlet_layers_x(0), inlet_layers_y(0), inlet_layers_z(0), outlet_layers_x(0), outlet_layers_y(0), outlet_layers_z(0), - inlet_layers_phase(1),outlet_layers_phase(2), - sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), - sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), - sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), - sendList_x(NULL), sendList_y(NULL), sendList_z(NULL), sendList_X(NULL), sendList_Y(NULL), sendList_Z(NULL), - sendList_xy(NULL), sendList_yz(NULL), sendList_xz(NULL), sendList_Xy(NULL), sendList_Yz(NULL), sendList_xZ(NULL), - sendList_xY(NULL), sendList_yZ(NULL), sendList_Xz(NULL), sendList_XY(NULL), sendList_YZ(NULL), sendList_XZ(NULL), - sendBuf_x(NULL), sendBuf_y(NULL), sendBuf_z(NULL), sendBuf_X(NULL), sendBuf_Y(NULL), sendBuf_Z(NULL), - sendBuf_xy(NULL), sendBuf_yz(NULL), sendBuf_xz(NULL), sendBuf_Xy(NULL), sendBuf_Yz(NULL), sendBuf_xZ(NULL), - sendBuf_xY(NULL), sendBuf_yZ(NULL), sendBuf_Xz(NULL), sendBuf_XY(NULL), sendBuf_YZ(NULL), sendBuf_XZ(NULL), - recvCount_x(0), recvCount_y(0), recvCount_z(0), recvCount_X(0), recvCount_Y(0), recvCount_Z(0), - recvCount_xy(0), recvCount_yz(0), recvCount_xz(0), recvCount_Xy(0), recvCount_Yz(0), recvCount_xZ(0), - recvCount_xY(0), recvCount_yZ(0), recvCount_Xz(0), recvCount_XY(0), recvCount_YZ(0), recvCount_XZ(0), - recvList_x(NULL), recvList_y(NULL), recvList_z(NULL), recvList_X(NULL), recvList_Y(NULL), recvList_Z(NULL), - recvList_xy(NULL), recvList_yz(NULL), recvList_xz(NULL), recvList_Xy(NULL), recvList_Yz(NULL), recvList_xZ(NULL), - recvList_xY(NULL), recvList_yZ(NULL), recvList_Xz(NULL), recvList_XY(NULL), recvList_YZ(NULL), recvList_XZ(NULL), - recvBuf_x(NULL), recvBuf_y(NULL), recvBuf_z(NULL), recvBuf_X(NULL), recvBuf_Y(NULL), recvBuf_Z(NULL), - recvBuf_xy(NULL), recvBuf_yz(NULL), recvBuf_xz(NULL), recvBuf_Xy(NULL), recvBuf_Yz(NULL), recvBuf_xZ(NULL), - recvBuf_xY(NULL), recvBuf_yZ(NULL), recvBuf_Xz(NULL), recvBuf_XY(NULL), recvBuf_YZ(NULL), recvBuf_XZ(NULL), - sendData_x(NULL), sendData_y(NULL), sendData_z(NULL), sendData_X(NULL), sendData_Y(NULL), sendData_Z(NULL), - sendData_xy(NULL), sendData_yz(NULL), sendData_xz(NULL), sendData_Xy(NULL), sendData_Yz(NULL), sendData_xZ(NULL), - sendData_xY(NULL), sendData_yZ(NULL), sendData_Xz(NULL), sendData_XY(NULL), sendData_YZ(NULL), sendData_XZ(NULL), - recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), - recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), - recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), - id(NULL) + inlet_layers_phase(1),outlet_layers_phase(2) { Comm = Communicator.dup(); @@ -116,54 +66,18 @@ Domain::Domain( std::shared_ptr db, const Utilities::MPI& Communicator Comm.barrier(); } + +/******************************************************** + * Destructor * + ********************************************************/ Domain::~Domain() { - // Free sendList - delete [] sendList_x; delete [] sendList_y; delete [] sendList_z; - delete [] sendList_X; delete [] sendList_Y; delete [] sendList_Z; - delete [] sendList_xy; delete [] sendList_yz; delete [] sendList_xz; - delete [] sendList_Xy; delete [] sendList_Yz; delete [] sendList_xZ; - delete [] sendList_xY; delete [] sendList_yZ; delete [] sendList_Xz; - delete [] sendList_XY; delete [] sendList_YZ; delete [] sendList_XZ; - // Free sendBuf - delete [] sendBuf_x; delete [] sendBuf_y; delete [] sendBuf_z; - delete [] sendBuf_X; delete [] sendBuf_Y; delete [] sendBuf_Z; - delete [] sendBuf_xy; delete [] sendBuf_yz; delete [] sendBuf_xz; - delete [] sendBuf_Xy; delete [] sendBuf_Yz; delete [] sendBuf_xZ; - delete [] sendBuf_xY; delete [] sendBuf_yZ; delete [] sendBuf_Xz; - delete [] sendBuf_XY; delete [] sendBuf_YZ; delete [] sendBuf_XZ; - // Free recvList - delete [] recvList_x; delete [] recvList_y; delete [] recvList_z; - delete [] recvList_X; delete [] recvList_Y; delete [] recvList_Z; - delete [] recvList_xy; delete [] recvList_yz; delete [] recvList_xz; - delete [] recvList_Xy; delete [] recvList_Yz; delete [] recvList_xZ; - delete [] recvList_xY; delete [] recvList_yZ; delete [] recvList_Xz; - delete [] recvList_XY; delete [] recvList_YZ; delete [] recvList_XZ; - // Free recvBuf - delete [] recvBuf_x; delete [] recvBuf_y; delete [] recvBuf_z; - delete [] recvBuf_X; delete [] recvBuf_Y; delete [] recvBuf_Z; - delete [] recvBuf_xy; delete [] recvBuf_yz; delete [] recvBuf_xz; - delete [] recvBuf_Xy; delete [] recvBuf_Yz; delete [] recvBuf_xZ; - delete [] recvBuf_xY; delete [] recvBuf_yZ; delete [] recvBuf_Xz; - delete [] recvBuf_XY; delete [] recvBuf_YZ; delete [] recvBuf_XZ; - // Free sendData - delete [] sendData_x; delete [] sendData_y; delete [] sendData_z; - delete [] sendData_X; delete [] sendData_Y; delete [] sendData_Z; - delete [] sendData_xy; delete [] sendData_xY; delete [] sendData_Xy; - delete [] sendData_XY; delete [] sendData_xz; delete [] sendData_xZ; - delete [] sendData_Xz; delete [] sendData_XZ; delete [] sendData_yz; - delete [] sendData_yZ; delete [] sendData_Yz; delete [] sendData_YZ; - // Free recvData - delete [] recvData_x; delete [] recvData_y; delete [] recvData_z; - delete [] recvData_X; delete [] recvData_Y; delete [] recvData_Z; - delete [] recvData_xy; delete [] recvData_xY; delete [] recvData_Xy; - delete [] recvData_XY; delete [] recvData_xz; delete [] recvData_xZ; - delete [] recvData_Xz; delete [] recvData_XZ; delete [] recvData_yz; - delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ; - // Free id - delete [] id; } + +/******************************************************** + * Initialization * + ********************************************************/ void Domain::initialize( std::shared_ptr db ) { d_db = db; @@ -228,13 +142,115 @@ void Domain::initialize( std::shared_ptr db ) if (myrank==0) printf("voxel length = %f micron \n", voxel_length); - id = new signed char[N]; - memset(id,0,N); + id = std::vector( N, 0 ); BoundaryCondition = d_db->getScalar("BC"); int nprocs = Comm.getSize(); INSIST(nprocs == nproc[0]*nproc[1]*nproc[2],"Fatal error in processor count!"); } + +/******************************************************** + * Get send/recv lists * + ********************************************************/ +const std::vector& Domain::getRecvList( const char* dir ) const +{ + if ( dir[0] == 'x' ) { + if ( dir[1] == 0 ) + return recvList_x; + else if ( dir[1] == 'y' ) + return recvList_xy; + else if ( dir[1] == 'Y' ) + return recvList_xY; + else if ( dir[1] == 'z' ) + return recvList_xz; + else if ( dir[1] == 'Z' ) + return recvList_xZ; + } else if ( dir[0] == 'y' ) { + if ( dir[1] == 0 ) + return recvList_y; + else if ( dir[1] == 'z' ) + return recvList_yz; + else if ( dir[1] == 'Z' ) + return recvList_yZ; + } else if ( dir[0] == 'z' ) { + if ( dir[1] == 0 ) + return recvList_z; + } else if ( dir[0] == 'X' ) { + if ( dir[1] == 0 ) + return recvList_X; + else if ( dir[1] == 'y' ) + return recvList_Xy; + else if ( dir[1] == 'Y' ) + return recvList_XY; + else if ( dir[1] == 'z' ) + return recvList_Xz; + else if ( dir[1] == 'Z' ) + return recvList_XZ; + } else if ( dir[0] == 'Y' ) { + if ( dir[1] == 0 ) + return recvList_Y; + else if ( dir[1] == 'z' ) + return recvList_Yz; + else if ( dir[1] == 'Z' ) + return recvList_YZ; + } else if ( dir[0] == 'Z' ) { + if ( dir[1] == 0 ) + return recvList_Z; + } + throw std::logic_error("Internal error"); +} +const std::vector& Domain::getSendList( const char* dir ) const +{ + if ( dir[0] == 'x' ) { + if ( dir[1] == 0 ) + return sendList_x; + else if ( dir[1] == 'y' ) + return sendList_xy; + else if ( dir[1] == 'Y' ) + return sendList_xY; + else if ( dir[1] == 'z' ) + return sendList_xz; + else if ( dir[1] == 'Z' ) + return sendList_xZ; + } else if ( dir[0] == 'y' ) { + if ( dir[1] == 0 ) + return sendList_y; + else if ( dir[1] == 'z' ) + return sendList_yz; + else if ( dir[1] == 'Z' ) + return sendList_yZ; + } else if ( dir[0] == 'z' ) { + if ( dir[1] == 0 ) + return sendList_z; + } else if ( dir[0] == 'X' ) { + if ( dir[1] == 0 ) + return sendList_X; + else if ( dir[1] == 'y' ) + return sendList_Xy; + else if ( dir[1] == 'Y' ) + return sendList_XY; + else if ( dir[1] == 'z' ) + return sendList_Xz; + else if ( dir[1] == 'Z' ) + return sendList_XZ; + } else if ( dir[0] == 'Y' ) { + if ( dir[1] == 0 ) + return sendList_Y; + else if ( dir[1] == 'z' ) + return sendList_Yz; + else if ( dir[1] == 'Z' ) + return sendList_YZ; + } else if ( dir[0] == 'Z' ) { + if ( dir[1] == 0 ) + return sendList_Z; + } + throw std::logic_error("Internal error"); +} + + +/******************************************************** + * Decomp * + ********************************************************/ void Domain::Decomp( const std::string& Filename ) { //....................................................................... @@ -319,7 +335,7 @@ void Domain::Decomp( const std::string& Filename ) global_Ny = SIZE[1]; global_Nz = SIZE[2]; nprocs=nprocx*nprocy*nprocz; - char *SegData = NULL; + char *SegData = nullptr; if (RANK==0){ printf("Input media: %s\n",Filename.c_str()); @@ -337,7 +353,7 @@ void Domain::Decomp( const std::string& Filename ) if (ReadType == "8bit"){ printf("Reading 8-bit input data \n"); FILE *SEGDAT = fopen(Filename.c_str(),"rb"); - if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading segmented data"); + if (!SEGDAT) ERROR("Domain.cpp: Error reading segmented data"); size_t ReadSeg; ReadSeg=fread(SegData,1,SIZE,SEGDAT); if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading segmented data \n"); @@ -348,7 +364,7 @@ void Domain::Decomp( const std::string& Filename ) short int *InputData; InputData = new short int[SIZE]; FILE *SEGDAT = fopen(Filename.c_str(),"rb"); - if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading segmented data"); + if (!SEGDAT) ERROR("Domain.cpp: Error reading segmented data"); size_t ReadSeg; ReadSeg=fread(InputData,2,SIZE,SEGDAT); if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading segmented data \n"); @@ -575,7 +591,7 @@ void Domain::Decomp( const std::string& Filename ) else{ // Recieve the subdomain from rank = 0 //printf("Ready to recieve data %i at process %i \n", N,rank); - Comm.recv(id,N,0,15); + Comm.recv(id.data(),N,0,15); } Comm.barrier(); } @@ -680,6 +696,9 @@ void Domain::CommInit() int sendtag = 21; int recvtag = 21; //...................................................................................... + int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z; + int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ; + int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ; sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0; sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0; sendCount_xY = sendCount_yZ = sendCount_Xz = sendCount_XY = sendCount_YZ = sendCount_XZ = 0; @@ -717,24 +736,24 @@ void Domain::CommInit() } // allocate send lists - sendList_x = new int [sendCount_x]; - sendList_y = new int [sendCount_y]; - sendList_z = new int [sendCount_z]; - sendList_X = new int [sendCount_X]; - sendList_Y = new int [sendCount_Y]; - sendList_Z = new int [sendCount_Z]; - sendList_xy = new int [sendCount_xy]; - sendList_yz = new int [sendCount_yz]; - sendList_xz = new int [sendCount_xz]; - sendList_Xy = new int [sendCount_Xy]; - sendList_Yz = new int [sendCount_Yz]; - sendList_xZ = new int [sendCount_xZ]; - sendList_xY = new int [sendCount_xY]; - sendList_yZ = new int [sendCount_yZ]; - sendList_Xz = new int [sendCount_Xz]; - sendList_XY = new int [sendCount_XY]; - sendList_YZ = new int [sendCount_YZ]; - sendList_XZ = new int [sendCount_XZ]; + sendList_x.resize( sendCount_x, 0 ); + sendList_y.resize( sendCount_y, 0 ); + sendList_z.resize( sendCount_z, 0 ); + sendList_X.resize( sendCount_X, 0 ); + sendList_Y.resize( sendCount_Y, 0 ); + sendList_Z.resize( sendCount_Z, 0 ); + sendList_xy.resize( sendCount_xy, 0 ); + sendList_yz.resize( sendCount_yz, 0 ); + sendList_xz.resize( sendCount_xz, 0 ); + sendList_Xy.resize( sendCount_Xy, 0 ); + sendList_Yz.resize( sendCount_Yz, 0 ); + sendList_xZ.resize( sendCount_xZ, 0 ); + sendList_xY.resize( sendCount_xY, 0 ); + sendList_yZ.resize( sendCount_yZ, 0 ); + sendList_Xz.resize( sendCount_Xz, 0 ); + sendList_XY.resize( sendCount_XY, 0 ); + sendList_YZ.resize( sendCount_YZ, 0 ); + sendList_XZ.resize( sendCount_XZ, 0 ); // Populate the send list sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0; sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0; @@ -772,26 +791,10 @@ void Domain::CommInit() } } - // allocate send buffers - sendBuf_x = new int [sendCount_x]; - sendBuf_y = new int [sendCount_y]; - sendBuf_z = new int [sendCount_z]; - sendBuf_X = new int [sendCount_X]; - sendBuf_Y = new int [sendCount_Y]; - sendBuf_Z = new int [sendCount_Z]; - sendBuf_xy = new int [sendCount_xy]; - sendBuf_yz = new int [sendCount_yz]; - sendBuf_xz = new int [sendCount_xz]; - sendBuf_Xy = new int [sendCount_Xy]; - sendBuf_Yz = new int [sendCount_Yz]; - sendBuf_xZ = new int [sendCount_xZ]; - sendBuf_xY = new int [sendCount_xY]; - sendBuf_yZ = new int [sendCount_yZ]; - sendBuf_Xz = new int [sendCount_Xz]; - sendBuf_XY = new int [sendCount_XY]; - sendBuf_YZ = new int [sendCount_YZ]; - sendBuf_XZ = new int [sendCount_XZ]; //...................................................................................... + int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z; + int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ; + int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ; req1[0] = Comm.Isend(&sendCount_x,1,rank_x(),sendtag+0); req2[0] = Comm.Irecv(&recvCount_X,1,rank_X(),recvtag+0); req1[1] = Comm.Isend(&sendCount_X,1,rank_X(),sendtag+1); @@ -831,63 +834,62 @@ void Domain::CommInit() Comm.waitAll(18,req1); Comm.waitAll(18,req2); Comm.barrier(); + // allocate recv lists + recvList_x.resize( recvCount_x, 0 ); + recvList_y.resize( recvCount_y, 0 ); + recvList_z.resize( recvCount_z, 0 ); + recvList_X.resize( recvCount_X, 0 ); + recvList_Y.resize( recvCount_Y, 0 ); + recvList_Z.resize( recvCount_Z, 0 ); + recvList_xy.resize( recvCount_xy, 0 ); + recvList_yz.resize( recvCount_yz, 0 ); + recvList_xz.resize( recvCount_xz, 0 ); + recvList_Xy.resize( recvCount_Xy, 0 ); + recvList_Yz.resize( recvCount_Yz, 0 ); + recvList_xZ.resize( recvCount_xZ, 0 ); + recvList_xY.resize( recvCount_xY, 0 ); + recvList_yZ.resize( recvCount_yZ, 0 ); + recvList_Xz.resize( recvCount_Xz, 0 ); + recvList_XY.resize( recvCount_XY, 0 ); + recvList_YZ.resize( recvCount_YZ, 0 ); + recvList_XZ.resize( recvCount_XZ, 0 ); //...................................................................................... - // recv buffers - recvList_x = new int [recvCount_x]; - recvList_y = new int [recvCount_y]; - recvList_z = new int [recvCount_z]; - recvList_X = new int [recvCount_X]; - recvList_Y = new int [recvCount_Y]; - recvList_Z = new int [recvCount_Z]; - recvList_xy = new int [recvCount_xy]; - recvList_yz = new int [recvCount_yz]; - recvList_xz = new int [recvCount_xz]; - recvList_Xy = new int [recvCount_Xy]; - recvList_Yz = new int [recvCount_Yz]; - recvList_xZ = new int [recvCount_xZ]; - recvList_xY = new int [recvCount_xY]; - recvList_yZ = new int [recvCount_yZ]; - recvList_Xz = new int [recvCount_Xz]; - recvList_XY = new int [recvCount_XY]; - recvList_YZ = new int [recvCount_YZ]; - recvList_XZ = new int [recvCount_XZ]; - //...................................................................................... - req1[0] = Comm.Isend(sendList_x,sendCount_x,rank_x(),sendtag); - req2[0] = Comm.Irecv(recvList_X,recvCount_X,rank_X(),recvtag); - req1[1] = Comm.Isend(sendList_X,sendCount_X,rank_X(),sendtag); - req2[1] = Comm.Irecv(recvList_x,recvCount_x,rank_x(),recvtag); - req1[2] = Comm.Isend(sendList_y,sendCount_y,rank_y(),sendtag); - req2[2] = Comm.Irecv(recvList_Y,recvCount_Y,rank_Y(),recvtag); - req1[3] = Comm.Isend(sendList_Y,sendCount_Y,rank_Y(),sendtag); - req2[3] = Comm.Irecv(recvList_y,recvCount_y,rank_y(),recvtag); - req1[4] = Comm.Isend(sendList_z,sendCount_z,rank_z(),sendtag); - req2[4] = Comm.Irecv(recvList_Z,recvCount_Z,rank_Z(),recvtag); - req1[5] = Comm.Isend(sendList_Z,sendCount_Z,rank_Z(),sendtag); - req2[5] = Comm.Irecv(recvList_z,recvCount_z,rank_z(),recvtag); - req1[6] = Comm.Isend(sendList_xy,sendCount_xy,rank_xy(),sendtag); - req2[6] = Comm.Irecv(recvList_XY,recvCount_XY,rank_XY(),recvtag); - req1[7] = Comm.Isend(sendList_XY,sendCount_XY,rank_XY(),sendtag); - req2[7] = Comm.Irecv(recvList_xy,recvCount_xy,rank_xy(),recvtag); - req1[8] = Comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy(),sendtag); - req2[8] = Comm.Irecv(recvList_xY,recvCount_xY,rank_xY(),recvtag); - req1[9] = Comm.Isend(sendList_xY,sendCount_xY,rank_xY(),sendtag); - req2[9] = Comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy(),recvtag); - req1[10] = Comm.Isend(sendList_xz,sendCount_xz,rank_xz(),sendtag); - req2[10] = Comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ(),recvtag); - req1[11] = Comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ(),sendtag); - req2[11] = Comm.Irecv(recvList_xz,recvCount_xz,rank_xz(),recvtag); - req1[12] = Comm.Isend(sendList_Xz,sendCount_Xz,rank_Xz(),sendtag); - req2[12] = Comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ(),recvtag); - req1[13] = Comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ(),sendtag); - req2[13] = Comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz(),recvtag); - req1[14] = Comm.Isend(sendList_yz,sendCount_yz,rank_yz(),sendtag); - req2[14] = Comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ(),recvtag); - req1[15] = Comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ(),sendtag); - req2[15] = Comm.Irecv(recvList_yz,recvCount_yz,rank_yz(),recvtag); - req1[16] = Comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz(),sendtag); - req2[16] = Comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ(),recvtag); - req1[17] = Comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ(),sendtag); - req2[17] = Comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz(),recvtag); + req1[0] = Comm.Isend(sendList_x.data(),sendCount_x,rank_x(),sendtag); + req2[0] = Comm.Irecv(recvList_X.data(),recvCount_X,rank_X(),recvtag); + req1[1] = Comm.Isend(sendList_X.data(),sendCount_X,rank_X(),sendtag); + req2[1] = Comm.Irecv(recvList_x.data(),recvCount_x,rank_x(),recvtag); + req1[2] = Comm.Isend(sendList_y.data(),sendCount_y,rank_y(),sendtag); + req2[2] = Comm.Irecv(recvList_Y.data(),recvCount_Y,rank_Y(),recvtag); + req1[3] = Comm.Isend(sendList_Y.data(),sendCount_Y,rank_Y(),sendtag); + req2[3] = Comm.Irecv(recvList_y.data(),recvCount_y,rank_y(),recvtag); + req1[4] = Comm.Isend(sendList_z.data(),sendCount_z,rank_z(),sendtag); + req2[4] = Comm.Irecv(recvList_Z.data(),recvCount_Z,rank_Z(),recvtag); + req1[5] = Comm.Isend(sendList_Z.data(),sendCount_Z,rank_Z(),sendtag); + req2[5] = Comm.Irecv(recvList_z.data(),recvCount_z,rank_z(),recvtag); + req1[6] = Comm.Isend(sendList_xy.data(),sendCount_xy,rank_xy(),sendtag); + req2[6] = Comm.Irecv(recvList_XY.data(),recvCount_XY,rank_XY(),recvtag); + req1[7] = Comm.Isend(sendList_XY.data(),sendCount_XY,rank_XY(),sendtag); + req2[7] = Comm.Irecv(recvList_xy.data(),recvCount_xy,rank_xy(),recvtag); + req1[8] = Comm.Isend(sendList_Xy.data(),sendCount_Xy,rank_Xy(),sendtag); + req2[8] = Comm.Irecv(recvList_xY.data(),recvCount_xY,rank_xY(),recvtag); + req1[9] = Comm.Isend(sendList_xY.data(),sendCount_xY,rank_xY(),sendtag); + req2[9] = Comm.Irecv(recvList_Xy.data(),recvCount_Xy,rank_Xy(),recvtag); + req1[10] = Comm.Isend(sendList_xz.data(),sendCount_xz,rank_xz(),sendtag); + req2[10] = Comm.Irecv(recvList_XZ.data(),recvCount_XZ,rank_XZ(),recvtag); + req1[11] = Comm.Isend(sendList_XZ.data(),sendCount_XZ,rank_XZ(),sendtag); + req2[11] = Comm.Irecv(recvList_xz.data(),recvCount_xz,rank_xz(),recvtag); + req1[12] = Comm.Isend(sendList_Xz.data(),sendCount_Xz,rank_Xz(),sendtag); + req2[12] = Comm.Irecv(recvList_xZ.data(),recvCount_xZ,rank_xZ(),recvtag); + req1[13] = Comm.Isend(sendList_xZ.data(),sendCount_xZ,rank_xZ(),sendtag); + req2[13] = Comm.Irecv(recvList_Xz.data(),recvCount_Xz,rank_Xz(),recvtag); + req1[14] = Comm.Isend(sendList_yz.data(),sendCount_yz,rank_yz(),sendtag); + req2[14] = Comm.Irecv(recvList_YZ.data(),recvCount_YZ,rank_YZ(),recvtag); + req1[15] = Comm.Isend(sendList_YZ.data(),sendCount_YZ,rank_YZ(),sendtag); + req2[15] = Comm.Irecv(recvList_yz.data(),recvCount_yz,rank_yz(),recvtag); + req1[16] = Comm.Isend(sendList_Yz.data(),sendCount_Yz,rank_Yz(),sendtag); + req2[16] = Comm.Irecv(recvList_yZ.data(),recvCount_yZ,rank_yZ(),recvtag); + req1[17] = Comm.Isend(sendList_yZ.data(),sendCount_yZ,rank_yZ(),sendtag); + req2[17] = Comm.Irecv(recvList_Yz.data(),recvCount_Yz,rank_Yz(),recvtag); Comm.waitAll(18,req1); Comm.waitAll(18,req2); //...................................................................................... @@ -910,65 +912,7 @@ void Domain::CommInit() for (int idx=0; idx #include #include +#include #include #include #include @@ -155,24 +156,14 @@ public: // Public variables (need to create accessors instead) // Get the actual D3Q19 communication counts (based on location of solid phase) // Discrete velocity set symmetry implies the sendcount = recvcount //...................................................................................... - int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z; - int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ; - int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ; - //...................................................................................... - int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z; - int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ; - int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ; - //...................................................................................... - int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z; - int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ; - int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ; - //...................................................................................... - int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; - int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; - int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ; + inline int recvCount( const char* dir ) const { return getRecvList( dir ).size(); } + inline int sendCount( const char* dir ) const { return getSendList( dir ).size(); } + inline const int* recvList( const char* dir ) const { return getRecvList( dir ).data(); } + inline const int* sendList( const char* dir ) const { return getSendList( dir ).data(); } + //...................................................................................... // Solid indicator function - signed char *id; + std::vector id; void ReadIDs(); void Decomp( const std::string& filename ); @@ -189,21 +180,18 @@ private: //...................................................................................... MPI_Request req1[18], req2[18]; + //...................................................................................... + std::vector sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z; + std::vector sendList_xy, sendList_yz, sendList_xz, sendList_Xy, sendList_Yz, sendList_xZ; + std::vector sendList_xY, sendList_yZ, sendList_Xz, sendList_XY, sendList_YZ, sendList_XZ; + //...................................................................................... + std::vector recvList_x, recvList_y, recvList_z, recvList_X, recvList_Y, recvList_Z; + std::vector recvList_xy, recvList_yz, recvList_xz, recvList_Xy, recvList_Yz, recvList_xZ; + std::vector recvList_xY, recvList_yZ, recvList_Xz, recvList_XY, recvList_YZ, recvList_XZ; + //...................................................................................... + const std::vector& getRecvList( const char* dir ) const; + const std::vector& getSendList( const char* dir ) const; - int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z; - int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ; - int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ; - //...................................................................................... - int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z; - int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ; - int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ; - //...................................................................................... - double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z; - double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ; - double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ; - double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z; - double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ; - double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ; }; diff --git a/common/MPI.cpp b/common/MPI.cpp index 8b09bc49..7604ae27 100644 --- a/common/MPI.cpp +++ b/common/MPI.cpp @@ -3491,7 +3491,8 @@ void MPI_CLASS::wait( MPI_Request request ) { PROFILE_START( "wait", profile_level ); MPI_Status status; - int flag = 0; + MPI_Wait( &request, &status ); + /*int flag = 0; int err = MPI_Test( &request, &flag, &status ); MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid while ( !flag ) { @@ -3499,7 +3500,7 @@ void MPI_CLASS::wait( MPI_Request request ) sched_yield(); // Check if the request has finished MPI_Test( &request, &flag, &status ); - } + }*/ PROFILE_STOP( "wait", profile_level ); } int MPI_CLASS::waitAny( int count, MPI_Request *request ) @@ -3508,8 +3509,9 @@ int MPI_CLASS::waitAny( int count, MPI_Request *request ) return -1; PROFILE_START( "waitAny", profile_level ); int index = -1; - int flag = 0; auto status = new MPI_Status[count]; + MPI_Waitany( count, request, &index, status ); + /*int flag = 0; int err = MPI_Testany( count, request, &index, &flag, status ); MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid while ( !flag ) { @@ -3518,7 +3520,7 @@ int MPI_CLASS::waitAny( int count, MPI_Request *request ) // Check if the request has finished MPI_Testany( count, request, &index, &flag, status ); } - MPI_ASSERT( index >= 0 ); // Check that the index is valid + MPI_ASSERT( index >= 0 ); // Check that the index is valid*/ delete[] status; PROFILE_STOP( "waitAny", profile_level ); return index; @@ -3528,8 +3530,9 @@ void MPI_CLASS::waitAll( int count, MPI_Request *request ) if ( count == 0 ) return; PROFILE_START( "waitAll", profile_level ); - int flag = 0; auto status = new MPI_Status[count]; + MPI_Waitall( count, request, status ); + /*int flag = 0; int err = MPI_Testall( count, request, &flag, status ); MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid while ( !flag ) { @@ -3537,7 +3540,7 @@ void MPI_CLASS::waitAll( int count, MPI_Request *request ) sched_yield(); // Check if the request has finished MPI_Testall( count, request, &flag, status ); - } + }*/ PROFILE_STOP( "waitAll", profile_level ); delete[] status; } @@ -3549,7 +3552,8 @@ std::vector MPI_CLASS::waitSome( int count, MPI_Request *request ) std::vector indicies( count, -1 ); auto *status = new MPI_Status[count]; int outcount = 0; - int err = MPI_Testsome( count, request, &outcount, &indicies[0], status ); + MPI_Waitsome( count, request, &outcount, indicies.data(), status ); + /*int err = MPI_Testsome( count, request, &outcount, &indicies[0], status ); MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid MPI_ASSERT( outcount != MPI_UNDEFINED ); // Check that the first call is valid while ( outcount == 0 ) { @@ -3557,7 +3561,7 @@ std::vector MPI_CLASS::waitSome( int count, MPI_Request *request ) sched_yield(); // Check if the request has finished MPI_Testsome( count, request, &outcount, &indicies[0], status ); - } + }*/ indicies.resize( outcount ); delete[] status; PROFILE_STOP( "waitSome", profile_level ); diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 6f2966e7..51821176 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -32,42 +32,42 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ rank_YZ=Dm->rank_YZ(); rank_yZ=Dm->rank_yZ(); rank_Yz=Dm->rank_Yz(); - sendCount_x=Dm->sendCount_x; - sendCount_y=Dm->sendCount_y; - sendCount_z=Dm->sendCount_z; - sendCount_X=Dm->sendCount_X; - sendCount_Y=Dm->sendCount_Y; - sendCount_Z=Dm->sendCount_Z; - sendCount_xy=Dm->sendCount_xy; - sendCount_yz=Dm->sendCount_yz; - sendCount_xz=Dm->sendCount_xz; - sendCount_Xy=Dm->sendCount_Xy; - sendCount_Yz=Dm->sendCount_Yz; - sendCount_xZ=Dm->sendCount_xZ; - sendCount_xY=Dm->sendCount_xY; - sendCount_yZ=Dm->sendCount_yZ; - sendCount_Xz=Dm->sendCount_Xz; - sendCount_XY=Dm->sendCount_XY; - sendCount_YZ=Dm->sendCount_YZ; - sendCount_XZ=Dm->sendCount_XZ; - recvCount_x=Dm->recvCount_x; - recvCount_y=Dm->recvCount_y; - recvCount_z=Dm->recvCount_z; - recvCount_X=Dm->recvCount_X; - recvCount_Y=Dm->recvCount_Y; - recvCount_Z=Dm->recvCount_Z; - recvCount_xy=Dm->recvCount_xy; - recvCount_yz=Dm->recvCount_yz; - recvCount_xz=Dm->recvCount_xz; - recvCount_Xy=Dm->recvCount_Xy; - recvCount_Yz=Dm->recvCount_Yz; - recvCount_xZ=Dm->recvCount_xZ; - recvCount_xY=Dm->recvCount_xY; - recvCount_yZ=Dm->recvCount_yZ; - recvCount_Xz=Dm->recvCount_Xz; - recvCount_XY=Dm->recvCount_XY; - recvCount_YZ=Dm->recvCount_YZ; - recvCount_XZ=Dm->recvCount_XZ; + sendCount_x=Dm->sendCount("x"); + sendCount_y=Dm->sendCount("y"); + sendCount_z=Dm->sendCount("z"); + sendCount_X=Dm->sendCount("X"); + sendCount_Y=Dm->sendCount("Y"); + sendCount_Z=Dm->sendCount("Z"); + sendCount_xy=Dm->sendCount("xy"); + sendCount_yz=Dm->sendCount("yz"); + sendCount_xz=Dm->sendCount("xz"); + sendCount_Xy=Dm->sendCount("Xy"); + sendCount_Yz=Dm->sendCount("Yz"); + sendCount_xZ=Dm->sendCount("xZ"); + sendCount_xY=Dm->sendCount("xY"); + sendCount_yZ=Dm->sendCount("yZ"); + sendCount_Xz=Dm->sendCount("Xz"); + sendCount_XY=Dm->sendCount("XY"); + sendCount_YZ=Dm->sendCount("YZ"); + sendCount_XZ=Dm->sendCount("XZ"); + recvCount_x=Dm->recvCount("x"); + recvCount_y=Dm->recvCount("y"); + recvCount_z=Dm->recvCount("z"); + recvCount_X=Dm->recvCount("X"); + recvCount_Y=Dm->recvCount("Y"); + recvCount_Z=Dm->recvCount("Z"); + recvCount_xy=Dm->recvCount("xy"); + recvCount_yz=Dm->recvCount("yz"); + recvCount_xz=Dm->recvCount("xz"); + recvCount_Xy=Dm->recvCount("Xy"); + recvCount_Yz=Dm->recvCount("Yz"); + recvCount_xZ=Dm->recvCount("xZ"); + recvCount_xY=Dm->recvCount("xY"); + recvCount_yZ=Dm->recvCount("yZ"); + recvCount_Xz=Dm->recvCount("Xz"); + recvCount_XY=Dm->recvCount("XY"); + recvCount_YZ=Dm->recvCount("YZ"); + recvCount_XZ=Dm->recvCount("XZ"); iproc = Dm->iproc(); jproc = Dm->jproc(); @@ -174,43 +174,43 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ ScaLBL_AllocateZeroCopy((void **) &dvcRecvDist_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory //...................................................................................... - ScaLBL_CopyToZeroCopy(dvcSendList_x,Dm->sendList_x,sendCount_x*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_X,Dm->sendList_X,sendCount_X*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_y,Dm->sendList_y,sendCount_y*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Y,Dm->sendList_Y,sendCount_Y*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_z,Dm->sendList_z,sendCount_z*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Z,Dm->sendList_Z,sendCount_Z*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_xy,Dm->sendList_xy,sendCount_xy*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_XY,Dm->sendList_XY,sendCount_XY*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_xY,Dm->sendList_xY,sendCount_xY*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Xy,Dm->sendList_Xy,sendCount_Xy*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_xz,Dm->sendList_xz,sendCount_xz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_XZ,Dm->sendList_XZ,sendCount_XZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_xZ,Dm->sendList_xZ,sendCount_xZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Xz,Dm->sendList_Xz,sendCount_Xz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_yz,Dm->sendList_yz,sendCount_yz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_YZ,Dm->sendList_YZ,sendCount_YZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_yZ,Dm->sendList_yZ,sendCount_yZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Yz,Dm->sendList_Yz,sendCount_Yz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_x,Dm->sendList("x"),sendCount_x*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_X,Dm->sendList("X"),sendCount_X*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_y,Dm->sendList("y"),sendCount_y*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Y,Dm->sendList("Y"),sendCount_Y*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_z,Dm->sendList("z"),sendCount_z*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Z,Dm->sendList("Z"),sendCount_Z*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_xy,Dm->sendList("xy"),sendCount_xy*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_XY,Dm->sendList("XY"),sendCount_XY*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_xY,Dm->sendList("xY"),sendCount_xY*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Xy,Dm->sendList("Xy"),sendCount_Xy*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_xz,Dm->sendList("xz"),sendCount_xz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_XZ,Dm->sendList("XZ"),sendCount_XZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_xZ,Dm->sendList("xZ"),sendCount_xZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Xz,Dm->sendList("Xz"),sendCount_Xz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_yz,Dm->sendList("yz"),sendCount_yz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_YZ,Dm->sendList("YZ"),sendCount_YZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_yZ,Dm->sendList("yZ"),sendCount_yZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Yz,Dm->sendList("Yz"),sendCount_Yz*sizeof(int)); //...................................................................................... - ScaLBL_CopyToZeroCopy(dvcRecvList_x,Dm->recvList_x,recvCount_x*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_X,Dm->recvList_X,recvCount_X*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_y,Dm->recvList_y,recvCount_y*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Y,Dm->recvList_Y,recvCount_Y*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_z,Dm->recvList_z,recvCount_z*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Z,Dm->recvList_Z,recvCount_Z*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_xy,Dm->recvList_xy,recvCount_xy*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_XY,Dm->recvList_XY,recvCount_XY*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_xY,Dm->recvList_xY,recvCount_xY*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Xy,Dm->recvList_Xy,recvCount_Xy*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_xz,Dm->recvList_xz,recvCount_xz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_XZ,Dm->recvList_XZ,recvCount_XZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_xZ,Dm->recvList_xZ,recvCount_xZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Xz,Dm->recvList_Xz,recvCount_Xz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_yz,Dm->recvList_yz,recvCount_yz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_YZ,Dm->recvList_YZ,recvCount_YZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_yZ,Dm->recvList_yZ,recvCount_yZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm->recvList_Yz,recvCount_Yz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_x,Dm->recvList("x"),recvCount_x*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_X,Dm->recvList("X"),recvCount_X*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_y,Dm->recvList("y"),recvCount_y*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Y,Dm->recvList("Y"),recvCount_Y*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_z,Dm->recvList("z"),recvCount_z*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Z,Dm->recvList("Z"),recvCount_Z*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_xy,Dm->recvList("xy"),recvCount_xy*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_XY,Dm->recvList("XY"),recvCount_XY*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_xY,Dm->recvList("xY"),recvCount_xY*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Xy,Dm->recvList("Xy"),recvCount_Xy*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_xz,Dm->recvList("xz"),recvCount_xz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_XZ,Dm->recvList("XZ"),recvCount_XZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_xZ,Dm->recvList("xZ"),recvCount_xZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Xz,Dm->recvList("Xz"),recvCount_Xz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_yz,Dm->recvList("yz"),recvCount_yz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_YZ,Dm->recvList("YZ"),recvCount_YZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_yZ,Dm->recvList("yZ"),recvCount_yZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm->recvList("Yz"),recvCount_Yz*sizeof(int)); //...................................................................................... MPI_COMM_SCALBL.barrier(); @@ -219,70 +219,70 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ // Set up the recieve distribution lists //................................................................................... //...Map recieve list for the X face: q=2,8,10,12,14 ................................. - D3Q19_MapRecv(-1,0,0,Dm->recvList_X,0,recvCount_X,dvcRecvDist_X); - D3Q19_MapRecv(-1,-1,0,Dm->recvList_X,recvCount_X,recvCount_X,dvcRecvDist_X); - D3Q19_MapRecv(-1,1,0,Dm->recvList_X,2*recvCount_X,recvCount_X,dvcRecvDist_X); - D3Q19_MapRecv(-1,0,-1,Dm->recvList_X,3*recvCount_X,recvCount_X,dvcRecvDist_X); - D3Q19_MapRecv(-1,0,1,Dm->recvList_X,4*recvCount_X,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,0,0, Dm->recvList("X"),0,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,-1,0,Dm->recvList("X"),recvCount_X,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,1,0, Dm->recvList("X"),2*recvCount_X,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,0,-1,Dm->recvList("X"),3*recvCount_X,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,0,1, Dm->recvList("X"),4*recvCount_X,recvCount_X,dvcRecvDist_X); //................................................................................... //...Map recieve list for the x face: q=1,7,9,11,13.................................. - D3Q19_MapRecv(1,0,0,Dm->recvList_x,0,recvCount_x,dvcRecvDist_x); - D3Q19_MapRecv(1,1,0,Dm->recvList_x,recvCount_x,recvCount_x,dvcRecvDist_x); - D3Q19_MapRecv(1,-1,0,Dm->recvList_x,2*recvCount_x,recvCount_x,dvcRecvDist_x); - D3Q19_MapRecv(1,0,1,Dm->recvList_x,3*recvCount_x,recvCount_x,dvcRecvDist_x); - D3Q19_MapRecv(1,0,-1,Dm->recvList_x,4*recvCount_x,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,0,0, Dm->recvList("x"),0,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,1,0, Dm->recvList("x"),recvCount_x,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,-1,0,Dm->recvList("x"),2*recvCount_x,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,0,1, Dm->recvList("x"),3*recvCount_x,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,0,-1,Dm->recvList("x"),4*recvCount_x,recvCount_x,dvcRecvDist_x); //................................................................................... //...Map recieve list for the y face: q=4,8,9,16,18 ................................... - D3Q19_MapRecv(0,-1,0,Dm->recvList_Y,0,recvCount_Y,dvcRecvDist_Y); - D3Q19_MapRecv(-1,-1,0,Dm->recvList_Y,recvCount_Y,recvCount_Y,dvcRecvDist_Y); - D3Q19_MapRecv(1,-1,0,Dm->recvList_Y,2*recvCount_Y,recvCount_Y,dvcRecvDist_Y); - D3Q19_MapRecv(0,-1,-1,Dm->recvList_Y,3*recvCount_Y,recvCount_Y,dvcRecvDist_Y); - D3Q19_MapRecv(0,-1,1,Dm->recvList_Y,4*recvCount_Y,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(0,-1,0, Dm->recvList("Y"),0,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(-1,-1,0,Dm->recvList("Y"),recvCount_Y,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(1,-1,0, Dm->recvList("Y"),2*recvCount_Y,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(0,-1,-1,Dm->recvList("Y"),3*recvCount_Y,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(0,-1,1, Dm->recvList("Y"),4*recvCount_Y,recvCount_Y,dvcRecvDist_Y); //................................................................................... //...Map recieve list for the Y face: q=3,7,10,15,17 .................................. - D3Q19_MapRecv(0,1,0,Dm->recvList_y,0,recvCount_y,dvcRecvDist_y); - D3Q19_MapRecv(1,1,0,Dm->recvList_y,recvCount_y,recvCount_y,dvcRecvDist_y); - D3Q19_MapRecv(-1,1,0,Dm->recvList_y,2*recvCount_y,recvCount_y,dvcRecvDist_y); - D3Q19_MapRecv(0,1,1,Dm->recvList_y,3*recvCount_y,recvCount_y,dvcRecvDist_y); - D3Q19_MapRecv(0,1,-1,Dm->recvList_y,4*recvCount_y,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(0,1,0, Dm->recvList("y"),0,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(1,1,0, Dm->recvList("y"),recvCount_y,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(-1,1,0,Dm->recvList("y"),2*recvCount_y,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(0,1,1, Dm->recvList("y"),3*recvCount_y,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(0,1,-1,Dm->recvList("y"),4*recvCount_y,recvCount_y,dvcRecvDist_y); //................................................................................... //...Map recieve list for the z face<<<6,12,13,16,17).............................................. - D3Q19_MapRecv(0,0,-1,Dm->recvList_Z,0,recvCount_Z,dvcRecvDist_Z); - D3Q19_MapRecv(-1,0,-1,Dm->recvList_Z,recvCount_Z,recvCount_Z,dvcRecvDist_Z); - D3Q19_MapRecv(1,0,-1,Dm->recvList_Z,2*recvCount_Z,recvCount_Z,dvcRecvDist_Z); - D3Q19_MapRecv(0,-1,-1,Dm->recvList_Z,3*recvCount_Z,recvCount_Z,dvcRecvDist_Z); - D3Q19_MapRecv(0,1,-1,Dm->recvList_Z,4*recvCount_Z,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(0,0,-1, Dm->recvList("Z"),0,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(-1,0,-1,Dm->recvList("Z"),recvCount_Z,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(1,0,-1, Dm->recvList("Z"),2*recvCount_Z,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(0,-1,-1,Dm->recvList("Z"),3*recvCount_Z,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(0,1,-1, Dm->recvList("Z"),4*recvCount_Z,recvCount_Z,dvcRecvDist_Z); //...Map recieve list for the Z face<<<5,11,14,15,18).............................................. - D3Q19_MapRecv(0,0,1,Dm->recvList_z,0,recvCount_z,dvcRecvDist_z); - D3Q19_MapRecv(1,0,1,Dm->recvList_z,recvCount_z,recvCount_z,dvcRecvDist_z); - D3Q19_MapRecv(-1,0,1,Dm->recvList_z,2*recvCount_z,recvCount_z,dvcRecvDist_z); - D3Q19_MapRecv(0,1,1,Dm->recvList_z,3*recvCount_z,recvCount_z,dvcRecvDist_z); - D3Q19_MapRecv(0,-1,1,Dm->recvList_z,4*recvCount_z,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(0,0,1, Dm->recvList("z"),0,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(1,0,1, Dm->recvList("z"),recvCount_z,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(-1,0,1,Dm->recvList("z"),2*recvCount_z,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(0,1,1, Dm->recvList("z"),3*recvCount_z,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(0,-1,1,Dm->recvList("z"),4*recvCount_z,recvCount_z,dvcRecvDist_z); //.................................................................................. //...Map recieve list for the xy edge <<<8)................................ - D3Q19_MapRecv(-1,-1,0,Dm->recvList_XY,0,recvCount_XY,dvcRecvDist_XY); + D3Q19_MapRecv(-1,-1,0,Dm->recvList("XY"),0,recvCount_XY,dvcRecvDist_XY); //...Map recieve list for the Xy edge <<<9)................................ - D3Q19_MapRecv(1,-1,0,Dm->recvList_xY,0,recvCount_xY,dvcRecvDist_xY); + D3Q19_MapRecv(1,-1,0,Dm->recvList("xY"),0,recvCount_xY,dvcRecvDist_xY); //...Map recieve list for the xY edge <<<10)................................ - D3Q19_MapRecv(-1,1,0,Dm->recvList_Xy,0,recvCount_Xy,dvcRecvDist_Xy); + D3Q19_MapRecv(-1,1,0,Dm->recvList("Xy"),0,recvCount_Xy,dvcRecvDist_Xy); //...Map recieve list for the XY edge <<<7)................................ - D3Q19_MapRecv(1,1,0,Dm->recvList_xy,0,recvCount_xy,dvcRecvDist_xy); + D3Q19_MapRecv(1,1,0,Dm->recvList("xy"),0,recvCount_xy,dvcRecvDist_xy); //...Map recieve list for the xz edge <<<12)................................ - D3Q19_MapRecv(-1,0,-1,Dm->recvList_XZ,0,recvCount_XZ,dvcRecvDist_XZ); + D3Q19_MapRecv(-1,0,-1,Dm->recvList("XZ"),0,recvCount_XZ,dvcRecvDist_XZ); //...Map recieve list for the xZ edge <<<14)................................ - D3Q19_MapRecv(-1,0,1,Dm->recvList_Xz,0,recvCount_Xz,dvcRecvDist_Xz); + D3Q19_MapRecv(-1,0,1,Dm->recvList("Xz"),0,recvCount_Xz,dvcRecvDist_Xz); //...Map recieve list for the Xz edge <<<13)................................ - D3Q19_MapRecv(1,0,-1,Dm->recvList_xZ,0,recvCount_xZ,dvcRecvDist_xZ); + D3Q19_MapRecv(1,0,-1,Dm->recvList("xZ"),0,recvCount_xZ,dvcRecvDist_xZ); //...Map recieve list for the XZ edge <<<11)................................ - D3Q19_MapRecv(1,0,1,Dm->recvList_xz,0,recvCount_xz,dvcRecvDist_xz); + D3Q19_MapRecv(1,0,1,Dm->recvList("xz"),0,recvCount_xz,dvcRecvDist_xz); //...Map recieve list for the yz edge <<<16)................................ - D3Q19_MapRecv(0,-1,-1,Dm->recvList_YZ,0,recvCount_YZ,dvcRecvDist_YZ); + D3Q19_MapRecv(0,-1,-1,Dm->recvList("YZ"),0,recvCount_YZ,dvcRecvDist_YZ); //...Map recieve list for the yZ edge <<<18)................................ - D3Q19_MapRecv(0,-1,1,Dm->recvList_Yz,0,recvCount_Yz,dvcRecvDist_Yz); + D3Q19_MapRecv(0,-1,1,Dm->recvList("Yz"),0,recvCount_Yz,dvcRecvDist_Yz); //...Map recieve list for the Yz edge <<<17)................................ - D3Q19_MapRecv(0,1,-1,Dm->recvList_yZ,0,recvCount_yZ,dvcRecvDist_yZ); + D3Q19_MapRecv(0,1,-1,Dm->recvList("yZ"),0,recvCount_yZ,dvcRecvDist_yZ); //...Map recieve list for the YZ edge <<<15)................................ - D3Q19_MapRecv(0,1,1,Dm->recvList_yz,0,recvCount_yz,dvcRecvDist_yz); + D3Q19_MapRecv(0,1,1,Dm->recvList("yz"),0,recvCount_yz,dvcRecvDist_yz); //................................................................................... //...................................................................................... @@ -319,7 +319,7 @@ int ScaLBL_Communicator::LastInterior(){ return last_interior; } -void ScaLBL_Communicator::D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list, int start, int count, +void ScaLBL_Communicator::D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, const int *list, int start, int count, int *d3q19_recvlist){ int i,j,k,n,nn,idx; int * ReturnDist; diff --git a/common/ScaLBL.h b/common/ScaLBL.h index d7f012d1..50eabdd3 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -195,7 +195,7 @@ public: private: //void D3Q19_MapRecv_OLD(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, int *d3q19_recvlist); - void D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list, int start, int count, int *d3q19_recvlist); + void D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, const int *list, int start, int count, int *d3q19_recvlist); bool Lock; // use Lock to make sure only one call at a time to protect data in transit // only one set of Send requests can be active at any time (per instance) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 5a9c56d4..f085c0b2 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -199,7 +199,7 @@ void ScaLBL_ColorModel::ReadInput(){ ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); Array id_view; - id_view.viewRaw( size1, Mask->id ); + id_view.viewRaw( size1, Mask->id.data() ); fill.copy( input_id, id_view ); fill.fill( id_view ); } @@ -332,7 +332,7 @@ void ScaLBL_ColorModel::Create(){ if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np); comm.barrier(); //........................................................................... diff --git a/models/DFHModel.cpp b/models/DFHModel.cpp index 9709b107..718ce8f1 100644 --- a/models/DFHModel.cpp +++ b/models/DFHModel.cpp @@ -205,7 +205,7 @@ void ScaLBL_DFHModel::Create(){ if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np); comm.barrier(); //........................................................................... diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index 76d54571..95b5090e 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -151,7 +151,7 @@ void ScaLBL_MRTModel::Create(){ if (rank==0) printf ("Set up memory efficient layout \n"); Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np); comm.barrier(); //........................................................................... // MAIN VARIABLES ALLOCATED HERE diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index 0c84287e..9e4cdfda 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -20,7 +20,7 @@ using namespace std; -inline void PackID(int *list, int count, char *sendbuf, char *ID){ +inline void PackID(const int *list, int count, char *sendbuf, char *ID){ // Fill in the phase ID values from neighboring processors // This packs up the values that need to be sent from one processor to another int idx,n; @@ -32,7 +32,7 @@ inline void PackID(int *list, int count, char *sendbuf, char *ID){ } //*************************************************************************************** -inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ +inline void UnpackID(const int *list, int count, char *recvbuf, char *ID){ // Fill in the phase ID values from neighboring processors // This unpacks the values once they have been recieved from neighbors int idx,n; @@ -91,44 +91,44 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ; char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ; // send buffers - sendID_x = new char [Dm.sendCount_x]; - sendID_y = new char [Dm.sendCount_y]; - sendID_z = new char [Dm.sendCount_z]; - sendID_X = new char [Dm.sendCount_X]; - sendID_Y = new char [Dm.sendCount_Y]; - sendID_Z = new char [Dm.sendCount_Z]; - sendID_xy = new char [Dm.sendCount_xy]; - sendID_yz = new char [Dm.sendCount_yz]; - sendID_xz = new char [Dm.sendCount_xz]; - sendID_Xy = new char [Dm.sendCount_Xy]; - sendID_Yz = new char [Dm.sendCount_Yz]; - sendID_xZ = new char [Dm.sendCount_xZ]; - sendID_xY = new char [Dm.sendCount_xY]; - sendID_yZ = new char [Dm.sendCount_yZ]; - sendID_Xz = new char [Dm.sendCount_Xz]; - sendID_XY = new char [Dm.sendCount_XY]; - sendID_YZ = new char [Dm.sendCount_YZ]; - sendID_XZ = new char [Dm.sendCount_XZ]; + sendID_x = new char [Dm.sendCount("x")]; + sendID_y = new char [Dm.sendCount("y")]; + sendID_z = new char [Dm.sendCount("z")]; + sendID_X = new char [Dm.sendCount("X")]; + sendID_Y = new char [Dm.sendCount("Y")]; + sendID_Z = new char [Dm.sendCount("Z")]; + sendID_xy = new char [Dm.sendCount("xy")]; + sendID_yz = new char [Dm.sendCount("yz")]; + sendID_xz = new char [Dm.sendCount("xz")]; + sendID_Xy = new char [Dm.sendCount("Xy")]; + sendID_Yz = new char [Dm.sendCount("Yz")]; + sendID_xZ = new char [Dm.sendCount("xZ")]; + sendID_xY = new char [Dm.sendCount("xY")]; + sendID_yZ = new char [Dm.sendCount("yZ")]; + sendID_Xz = new char [Dm.sendCount("Xz")]; + sendID_XY = new char [Dm.sendCount("XY")]; + sendID_YZ = new char [Dm.sendCount("YZ")]; + sendID_XZ = new char [Dm.sendCount("XZ")]; //...................................................................................... // recv buffers - recvID_x = new char [Dm.recvCount_x]; - recvID_y = new char [Dm.recvCount_y]; - recvID_z = new char [Dm.recvCount_z]; - recvID_X = new char [Dm.recvCount_X]; - recvID_Y = new char [Dm.recvCount_Y]; - recvID_Z = new char [Dm.recvCount_Z]; - recvID_xy = new char [Dm.recvCount_xy]; - recvID_yz = new char [Dm.recvCount_yz]; - recvID_xz = new char [Dm.recvCount_xz]; - recvID_Xy = new char [Dm.recvCount_Xy]; - recvID_xZ = new char [Dm.recvCount_xZ]; - recvID_xY = new char [Dm.recvCount_xY]; - recvID_yZ = new char [Dm.recvCount_yZ]; - recvID_Yz = new char [Dm.recvCount_Yz]; - recvID_Xz = new char [Dm.recvCount_Xz]; - recvID_XY = new char [Dm.recvCount_XY]; - recvID_YZ = new char [Dm.recvCount_YZ]; - recvID_XZ = new char [Dm.recvCount_XZ]; + recvID_x = new char [Dm.recvCount("x")]; + recvID_y = new char [Dm.recvCount("y")]; + recvID_z = new char [Dm.recvCount("z")]; + recvID_X = new char [Dm.recvCount("X")]; + recvID_Y = new char [Dm.recvCount("Y")]; + recvID_Z = new char [Dm.recvCount("Z")]; + recvID_xy = new char [Dm.recvCount("xy")]; + recvID_yz = new char [Dm.recvCount("yz")]; + recvID_xz = new char [Dm.recvCount("xz")]; + recvID_Xy = new char [Dm.recvCount("Xy")]; + recvID_xZ = new char [Dm.recvCount("xZ")]; + recvID_xY = new char [Dm.recvCount("xY")]; + recvID_yZ = new char [Dm.recvCount("yZ")]; + recvID_Yz = new char [Dm.recvCount("Yz")]; + recvID_Xz = new char [Dm.recvCount("Xz")]; + recvID_XY = new char [Dm.recvCount("XY")]; + recvID_YZ = new char [Dm.recvCount("YZ")]; + recvID_XZ = new char [Dm.recvCount("XZ")]; //...................................................................................... int sendtag,recvtag; sendtag = recvtag = 7; @@ -194,62 +194,62 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny // Pack and send the updated ID values - PackID(Dm.sendList_x, Dm.sendCount_x ,sendID_x, id); - PackID(Dm.sendList_X, Dm.sendCount_X ,sendID_X, id); - PackID(Dm.sendList_y, Dm.sendCount_y ,sendID_y, id); - PackID(Dm.sendList_Y, Dm.sendCount_Y ,sendID_Y, id); - PackID(Dm.sendList_z, Dm.sendCount_z ,sendID_z, id); - PackID(Dm.sendList_Z, Dm.sendCount_Z ,sendID_Z, id); - PackID(Dm.sendList_xy, Dm.sendCount_xy ,sendID_xy, id); - PackID(Dm.sendList_Xy, Dm.sendCount_Xy ,sendID_Xy, id); - PackID(Dm.sendList_xY, Dm.sendCount_xY ,sendID_xY, id); - PackID(Dm.sendList_XY, Dm.sendCount_XY ,sendID_XY, id); - PackID(Dm.sendList_xz, Dm.sendCount_xz ,sendID_xz, id); - PackID(Dm.sendList_Xz, Dm.sendCount_Xz ,sendID_Xz, id); - PackID(Dm.sendList_xZ, Dm.sendCount_xZ ,sendID_xZ, id); - PackID(Dm.sendList_XZ, Dm.sendCount_XZ ,sendID_XZ, id); - PackID(Dm.sendList_yz, Dm.sendCount_yz ,sendID_yz, id); - PackID(Dm.sendList_Yz, Dm.sendCount_Yz ,sendID_Yz, id); - PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id); - PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id); + PackID(Dm.sendList("x"), Dm.sendCount("x") ,sendID_x, id); + PackID(Dm.sendList("X"), Dm.sendCount("X") ,sendID_X, id); + PackID(Dm.sendList("y"), Dm.sendCount("y") ,sendID_y, id); + PackID(Dm.sendList("Y"), Dm.sendCount("Y") ,sendID_Y, id); + PackID(Dm.sendList("z"), Dm.sendCount("z") ,sendID_z, id); + PackID(Dm.sendList("Z"), Dm.sendCount("Z") ,sendID_Z, id); + PackID(Dm.sendList("xy"), Dm.sendCount("xy") ,sendID_xy, id); + PackID(Dm.sendList("Xy"), Dm.sendCount("Xy") ,sendID_Xy, id); + PackID(Dm.sendList("xY"), Dm.sendCount("xY") ,sendID_xY, id); + PackID(Dm.sendList("XY"), Dm.sendCount("XY") ,sendID_XY, id); + PackID(Dm.sendList("xz"), Dm.sendCount("xz") ,sendID_xz, id); + PackID(Dm.sendList("Xz"), Dm.sendCount("Xz") ,sendID_Xz, id); + PackID(Dm.sendList("xZ"), Dm.sendCount("xZ") ,sendID_xZ, id); + PackID(Dm.sendList("XZ"), Dm.sendCount("XZ") ,sendID_XZ, id); + PackID(Dm.sendList("yz"), Dm.sendCount("yz") ,sendID_yz, id); + PackID(Dm.sendList("Yz"), Dm.sendCount("Yz") ,sendID_Yz, id); + PackID(Dm.sendList("yZ"), Dm.sendCount("yZ") ,sendID_yZ, id); + PackID(Dm.sendList("YZ"), Dm.sendCount("YZ") ,sendID_YZ, id); //...................................................................................... - Dm.Comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag); - Dm.Comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag); - Dm.Comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag); - Dm.Comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag); - Dm.Comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag); - Dm.Comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag); - Dm.Comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag); - Dm.Comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag); - Dm.Comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag); - Dm.Comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag); - Dm.Comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag); - Dm.Comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag); - Dm.Comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag); - Dm.Comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag); - Dm.Comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag); - Dm.Comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag); - Dm.Comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag); - Dm.Comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag); + Dm.Comm.sendrecv(sendID_x,Dm.sendCount("x"),Dm.rank_x(),sendtag,recvID_X,Dm.recvCount("X"),Dm.rank_X(),recvtag); + Dm.Comm.sendrecv(sendID_X,Dm.sendCount("X"),Dm.rank_X(),sendtag,recvID_x,Dm.recvCount("x"),Dm.rank_x(),recvtag); + Dm.Comm.sendrecv(sendID_y,Dm.sendCount("y"),Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount("Y"),Dm.rank_Y(),recvtag); + Dm.Comm.sendrecv(sendID_Y,Dm.sendCount("Y"),Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount("y"),Dm.rank_y(),recvtag); + Dm.Comm.sendrecv(sendID_z,Dm.sendCount("z"),Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount("Z"),Dm.rank_Z(),recvtag); + Dm.Comm.sendrecv(sendID_Z,Dm.sendCount("Z"),Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount("z"),Dm.rank_z(),recvtag); + Dm.Comm.sendrecv(sendID_xy,Dm.sendCount("xy"),Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount("XY"),Dm.rank_XY(),recvtag); + Dm.Comm.sendrecv(sendID_XY,Dm.sendCount("XY"),Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount("xy"),Dm.rank_xy(),recvtag); + Dm.Comm.sendrecv(sendID_Xy,Dm.sendCount("Xy"),Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount("xY"),Dm.rank_xY(),recvtag); + Dm.Comm.sendrecv(sendID_xY,Dm.sendCount("xY"),Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount("Xy"),Dm.rank_Xy(),recvtag); + Dm.Comm.sendrecv(sendID_xz,Dm.sendCount("xz"),Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount("XZ"),Dm.rank_XZ(),recvtag); + Dm.Comm.sendrecv(sendID_XZ,Dm.sendCount("XZ"),Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount("xz"),Dm.rank_xz(),recvtag); + Dm.Comm.sendrecv(sendID_Xz,Dm.sendCount("Xz"),Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount("xZ"),Dm.rank_xZ(),recvtag); + Dm.Comm.sendrecv(sendID_xZ,Dm.sendCount("xZ"),Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount("Xz"),Dm.rank_Xz(),recvtag); + Dm.Comm.sendrecv(sendID_yz,Dm.sendCount("yz"),Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount("YZ"),Dm.rank_YZ(),recvtag); + Dm.Comm.sendrecv(sendID_YZ,Dm.sendCount("YZ"),Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount("yz"),Dm.rank_yz(),recvtag); + Dm.Comm.sendrecv(sendID_Yz,Dm.sendCount("Yz"),Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount("yZ"),Dm.rank_yZ(),recvtag); + Dm.Comm.sendrecv(sendID_yZ,Dm.sendCount("yZ"),Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount("Yz"),Dm.rank_Yz(),recvtag); //...................................................................................... - UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id); - UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id); - UnpackID(Dm.recvList_y, Dm.recvCount_y ,recvID_y, id); - UnpackID(Dm.recvList_Y, Dm.recvCount_Y ,recvID_Y, id); - UnpackID(Dm.recvList_z, Dm.recvCount_z ,recvID_z, id); - UnpackID(Dm.recvList_Z, Dm.recvCount_Z ,recvID_Z, id); - UnpackID(Dm.recvList_xy, Dm.recvCount_xy ,recvID_xy, id); - UnpackID(Dm.recvList_Xy, Dm.recvCount_Xy ,recvID_Xy, id); - UnpackID(Dm.recvList_xY, Dm.recvCount_xY ,recvID_xY, id); - UnpackID(Dm.recvList_XY, Dm.recvCount_XY ,recvID_XY, id); - UnpackID(Dm.recvList_xz, Dm.recvCount_xz ,recvID_xz, id); - UnpackID(Dm.recvList_Xz, Dm.recvCount_Xz ,recvID_Xz, id); - UnpackID(Dm.recvList_xZ, Dm.recvCount_xZ ,recvID_xZ, id); - UnpackID(Dm.recvList_XZ, Dm.recvCount_XZ ,recvID_XZ, id); - UnpackID(Dm.recvList_yz, Dm.recvCount_yz ,recvID_yz, id); - UnpackID(Dm.recvList_Yz, Dm.recvCount_Yz ,recvID_Yz, id); - UnpackID(Dm.recvList_yZ, Dm.recvCount_yZ ,recvID_yZ, id); - UnpackID(Dm.recvList_YZ, Dm.recvCount_YZ ,recvID_YZ, id); + UnpackID(Dm.recvList("x"), Dm.recvCount("x") ,recvID_x, id); + UnpackID(Dm.recvList("X"), Dm.recvCount("X") ,recvID_X, id); + UnpackID(Dm.recvList("y"), Dm.recvCount("y") ,recvID_y, id); + UnpackID(Dm.recvList("Y"), Dm.recvCount("Y") ,recvID_Y, id); + UnpackID(Dm.recvList("z"), Dm.recvCount("z") ,recvID_z, id); + UnpackID(Dm.recvList("Z"), Dm.recvCount("Z") ,recvID_Z, id); + UnpackID(Dm.recvList("xy"), Dm.recvCount("xy") ,recvID_xy, id); + UnpackID(Dm.recvList("Xy"), Dm.recvCount("Xy") ,recvID_Xy, id); + UnpackID(Dm.recvList("xY"), Dm.recvCount("xY") ,recvID_xY, id); + UnpackID(Dm.recvList("XY"), Dm.recvCount("XY") ,recvID_XY, id); + UnpackID(Dm.recvList("xz"), Dm.recvCount("xz") ,recvID_xz, id); + UnpackID(Dm.recvList("Xz"), Dm.recvCount("Xz") ,recvID_Xz, id); + UnpackID(Dm.recvList("xZ"), Dm.recvCount("xZ") ,recvID_xZ, id); + UnpackID(Dm.recvList("XZ"), Dm.recvCount("XZ") ,recvID_XZ, id); + UnpackID(Dm.recvList("yz"), Dm.recvCount("yz") ,recvID_yz, id); + UnpackID(Dm.recvList("Yz"), Dm.recvCount("Yz") ,recvID_Yz, id); + UnpackID(Dm.recvList("yZ"), Dm.recvCount("yZ") ,recvID_yZ, id); + UnpackID(Dm.recvList("YZ"), Dm.recvCount("YZ") ,recvID_YZ, id); //...................................................................................... //double GlobalNumber = Dm.Comm.sumReduce( LocalNumber ); diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index 8b4f1a9b..3eecb13f 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -248,7 +248,7 @@ int main(int argc, char **argv) if (rank==0) printf ("Set up memory efficient layout Npad=%i \n",Npad); IntArray Map(Nx,Ny,Nz); auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np); comm.barrier(); //........................................................................... diff --git a/tests/TestColorGradDFH.cpp b/tests/TestColorGradDFH.cpp index b04aebce..02c0dc9d 100644 --- a/tests/TestColorGradDFH.cpp +++ b/tests/TestColorGradDFH.cpp @@ -100,7 +100,7 @@ int main(int argc, char **argv) int *neighborList; IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np); comm.barrier(); //......................device distributions................................. diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index c4a045ae..1ffa2465 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -283,7 +283,7 @@ int main(int argc, char **argv) auto neighborList= new int[18*Npad]; IntArray Map(Nx,Ny,Nz); Map.fill(-2); - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); + Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np); comm.barrier(); int neighborSize=18*Np*sizeof(int); //......................device distributions................................. diff --git a/tests/TestFluxBC.cpp b/tests/TestFluxBC.cpp index 3e999715..0798a481 100644 --- a/tests/TestFluxBC.cpp +++ b/tests/TestFluxBC.cpp @@ -88,7 +88,7 @@ int main (int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np); comm.barrier(); //......................device distributions................................. diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index b10954b1..fab5fe68 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -162,7 +162,7 @@ int main(int argc, char **argv) char *ID; ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device - ScaLBL_CopyToDevice(ID, Dm->id, N); + ScaLBL_CopyToDevice(ID, Dm->id.data(), N); //........................................................................... if (rank==0){ @@ -179,7 +179,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Np]; - ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); + ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np); if (rank == 0) PrintNeighborList(neighborList,Np, rank); diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index f3010081..3ba403a7 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -88,7 +88,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np); comm.barrier(); // Check the neighborlist diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index 25afd226..c19bdcef 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -106,13 +106,13 @@ int main(int argc, char **argv) if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device - auto ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Dm)); + auto ScaLBL_Comm = std::make_shared( Dm ); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device - ScaLBL_CopyToDevice(ID, Dm->id, N); + ScaLBL_CopyToDevice(ID, Dm->id.data(), N); //........................................................................... if (rank==0){ @@ -127,7 +127,7 @@ int main(int argc, char **argv) int *neighborList; IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np); comm.barrier(); //......................device distributions................................. diff --git a/tests/lbpm_minkowski_scalar.cpp b/tests/lbpm_minkowski_scalar.cpp index 721207a1..c5c27426 100644 --- a/tests/lbpm_minkowski_scalar.cpp +++ b/tests/lbpm_minkowski_scalar.cpp @@ -159,7 +159,7 @@ int main(int argc, char **argv) else{ // Recieve the subdomain from rank = 0 printf("Ready to recieve data %i at process %i \n", N,rank); - comm.recv(Dm->id,N,0,15); + comm.recv(Dm->id.data(),N,0,15); } comm.barrier();