From 9f5b44dfe4b4b18d4e8308e19df5e5edd0f1ee00 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Tue, 17 Mar 2020 21:23:18 -0400 Subject: [PATCH] Revert "Moving more MPI calls to the wrapper" This reverts commit 0f91767b6c870101084fbae0978280c04c85a004. --- IO/netcdf.cpp | 2 +- StackTrace/ErrorHandlers.h | 2 +- StackTrace/Utilities.cpp | 2 +- analysis/TwoPhase.cpp | 7 +- analysis/morphology.cpp | 132 ++--- common/Communication.h | 216 ++++---- common/Domain.cpp | 206 ++++---- common/Domain.h | 3 + common/MPI.I | 33 -- common/MPI.cpp | 48 -- common/MPI.h | 7 - common/ScaLBL.h | 1 + common/Utilities.cpp | 2 +- cpu/exe/lb2_Color_mpi.cpp | 538 +++++++++---------- cpu/exe/lb2_Color_wia_mpi_bubble.cpp | 711 ++++++++++++++------------ gpu/exe/lb1_MRT_mpi.cpp | 348 +++++++------ gpu/exe/lb1_MRT_mpi.cu | 352 +++++++------ gpu/exe/lb2_Color.cu | 65 ++- gpu/exe/lb2_Color_mpi.cpp | 541 ++++++++++---------- gpu/exe/lb2_Color_pBC_wia_mpi.cpp | 621 ++++++++++++---------- models/ColorModel.cpp | 8 +- models/DFHModel.cpp | 4 +- models/MRTModel.cpp | 4 +- tests/BlobAnalyzeParallel.cpp | 22 +- tests/GenerateSphereTest.cpp | 54 +- tests/TestBlobAnalyze.cpp | 28 +- tests/TestBubble.cpp | 41 +- tests/TestBubbleDFH.cpp | 4 +- tests/TestColorGrad.cpp | 20 +- tests/TestCommD3Q19.cpp | 4 +- tests/TestForceD3Q19.cpp | 4 +- tests/TestForceMoments.cpp | 4 +- tests/TestMRT.cpp | 28 +- tests/TestMicroCTReader.cpp | 1 + tests/TestMomentsD3Q19.cpp | 2 +- tests/TestNetcdf.cpp | 2 +- tests/TestSegDist.cpp | 4 +- tests/lb2_CMT_wia.cpp | 30 +- tests/lb2_Color_blob_wia_mpi.cpp | 427 ++++++++-------- tests/lbpm_BGK_simulator.cpp | 48 +- tests/lbpm_color_macro_simulator.cpp | 61 +-- tests/lbpm_disc_pp.cpp | 34 +- tests/lbpm_inkbottle_pp.cpp | 22 +- tests/lbpm_juanes_bench_disc_pp.cpp | 35 +- tests/lbpm_nondarcy_simulator.cpp | 52 +- tests/lbpm_nonnewtonian_simulator.cpp | 26 +- tests/lbpm_plates_pp.cpp | 24 +- tests/lbpm_porenetwork_pp.cpp | 25 +- tests/lbpm_random_pp.cpp | 92 ++-- tests/lbpm_segmented_decomp.cpp | 48 +- tests/lbpm_segmented_pp.cpp | 2 +- tests/lbpm_sphere_pp.cpp | 16 +- tests/lbpm_squaretube_pp.cpp | 25 +- 53 files changed, 2678 insertions(+), 2360 deletions(-) diff --git a/IO/netcdf.cpp b/IO/netcdf.cpp index 6c3773e3..e061579a 100644 --- a/IO/netcdf.cpp +++ b/IO/netcdf.cpp @@ -119,7 +119,7 @@ std::string VariableTypeName( VariableType type ) int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm ) { int fid = 0; - if ( comm.isNull() ) { + if ( comm == MPI_COMM_NULL ) { if ( mode == READ ) { int err = nc_open( filename.c_str(), NC_NOWRITE, &fid ); CHECK_NC_ERR( err ); diff --git a/StackTrace/ErrorHandlers.h b/StackTrace/ErrorHandlers.h index e43a4688..12b8d7de 100644 --- a/StackTrace/ErrorHandlers.h +++ b/StackTrace/ErrorHandlers.h @@ -6,7 +6,7 @@ #include -#include "common/MPI.h" +#include "mpi.h" namespace StackTrace diff --git a/StackTrace/Utilities.cpp b/StackTrace/Utilities.cpp index 5fb8e9b8..11f05777 100644 --- a/StackTrace/Utilities.cpp +++ b/StackTrace/Utilities.cpp @@ -14,7 +14,7 @@ #include #ifdef USE_MPI -#include "common/MPI.h" +#include "mpi.h" #endif #ifdef USE_TIMER diff --git a/analysis/TwoPhase.cpp b/analysis/TwoPhase.cpp index d878a663..812490e7 100644 --- a/analysis/TwoPhase.cpp +++ b/analysis/TwoPhase.cpp @@ -890,14 +890,14 @@ void TwoPhase::ComponentAverages() RecvBuffer.resize(BLOB_AVG_COUNT,NumberComponents_NWP); /* for (int b=0; bComm.barrier(); - Dm->Comm.sumReduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT); + MPI_Barrier(Dm->Comm); + MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm); for (int idx=0; idxComm.barrier(); Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP); - // Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT); + // MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm); if (Dm->rank()==0){ printf("rescaling... \n"); @@ -994,6 +994,7 @@ void TwoPhase::ComponentAverages() // reduce the wetting phase averages for (int b=0; bComm.barrier(); +// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm); Dm->Comm.sumReduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT); for (int idx=0; idx PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id); PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id); //...................................................................................... - Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag); - Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag); - Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag); - Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag); - Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag); - Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag); - Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag); - Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag); - Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag); - Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag); - Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag); - Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag); - Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag); - Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag); - Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag); - Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag); - Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag); - Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag); + MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag, + recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag, + recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag, + recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag, + recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag, + recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag, + recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag, + recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag, + recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag, + recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag, + recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag, + recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag, + recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag, + recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag, + recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag, + recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag, + recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag, + recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag, + recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id); UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id); @@ -285,7 +303,7 @@ double morph_open() fillHalo fillChar(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1); - GlobalNumber = Dm->Comm.sumReduce( LocalNumber ); + MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); count = 0.f; for (int k=1; kComm.sumReduce( count ); + MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); return countGlobal; } */ @@ -488,42 +506,42 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrsendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id); PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id); //...................................................................................... - Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag, - recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag); - Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag, - recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag); - Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag, - recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag); - Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag, - recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag); - Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag, - recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag); - Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag, - recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag); - Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag, - recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag); - Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag, - recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag); - Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag, - recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag); - Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag, - recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag); - Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag, - recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag); - Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag, - recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag); - Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag, - recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag); - Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag, - recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag); - Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag, - recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag); - Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag, - recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag); - Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag, - recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag); - Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag, - recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag); + MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag, + recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag, + recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag, + recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag, + recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag, + recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag, + recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag, + recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag, + recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag, + recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag, + recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag, + recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag, + recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag, + recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag, + recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag, + recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag, + recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag, + recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag, + recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id); UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id); @@ -599,7 +617,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrrank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm); - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); for (int k=1; k -void MPI_CLASS::sendrecv( const char*, int, int, int, char*, int, int, int ) const; -template<> -void MPI_CLASS::sendrecv( const int*, int, int, int, int*, int, int, int ) const; -template<> -void MPI_CLASS::sendrecv( const float*, int, int, int, float*, int, int, int ) const; -template<> -void MPI_CLASS::sendrecv( const double*, int, int, int, double*, int, int, int ) const; -template -void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag, - TYPE *recvbuf, int recvcount, int source, int recvtag ) const -{ - ERROR( "Not implimented" ); -} -#else -template -void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag, - TYPE *recvbuf, int recvcount, int source, int recvtag ) const -{ - ASSERT( dest == 0 ); - ASSERT( source == 0 ); - ASSERT( sendcount == recvcount ); - ASSERT( sendtag == recvtag ); - memcpy( recvbuf, sendbuf, sendcount * sizeof( TYPE ) ); -} -#endif - - - /************************************************************************ * allGather * ************************************************************************/ diff --git a/common/MPI.cpp b/common/MPI.cpp index 8b09bc49..73932d03 100644 --- a/common/MPI.cpp +++ b/common/MPI.cpp @@ -2805,54 +2805,6 @@ MPI_Request MPI_CLASS::IrecvBytes( } - -/************************************************************************ - * sendrecv * - ************************************************************************/ -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::sendrecv( const char* sendbuf, int sendcount, int dest, int sendtag, - char* recvbuf, int recvcount, int source, int recvtag ) const -{ - PROFILE_START( "sendrecv", profile_level ); - MPI_Sendrecv( sendbuf, sendcount, MPI_CHAR, dest, sendtag, - recvbuf, recvcount, MPI_CHAR, source, recvtag, - communicator, MPI_STATUS_IGNORE ); - PROFILE_STOP( "sendrecv", profile_level ); -} -template<> -void MPI_CLASS::sendrecv( const int* sendbuf, int sendcount, int dest, int sendtag, - int* recvbuf, int recvcount, int source, int recvtag ) const -{ - PROFILE_START( "sendrecv", profile_level ); - MPI_Sendrecv( sendbuf, sendcount, MPI_INT, dest, sendtag, - recvbuf, recvcount, MPI_INT, source, recvtag, - communicator, MPI_STATUS_IGNORE ); - PROFILE_STOP( "sendrecv", profile_level ); -} -template<> -void MPI_CLASS::sendrecv( const float* sendbuf, int sendcount, int dest, int sendtag, - float* recvbuf, int recvcount, int source, int recvtag ) const -{ - PROFILE_START( "sendrecv", profile_level ); - MPI_Sendrecv( sendbuf, sendcount, MPI_FLOAT, dest, sendtag, - recvbuf, recvcount, MPI_FLOAT, source, recvtag, - communicator, MPI_STATUS_IGNORE ); - PROFILE_STOP( "sendrecv", profile_level ); -} -template<> -void MPI_CLASS::sendrecv( const double* sendbuf, int sendcount, int dest, int sendtag, - double* recvbuf, int recvcount, int source, int recvtag ) const -{ - PROFILE_START( "sendrecv", profile_level ); - MPI_Sendrecv( sendbuf, sendcount, MPI_DOUBLE, dest, sendtag, - recvbuf, recvcount, MPI_DOUBLE, source, recvtag, - communicator, MPI_STATUS_IGNORE ); - PROFILE_STOP( "sendrecv", profile_level ); -} -#endif - - /************************************************************************ * allGather * * Note: these specializations are only called when using MPI. * diff --git a/common/MPI.h b/common/MPI.h index 4161d6a7..e3fd3e13 100644 --- a/common/MPI.h +++ b/common/MPI.h @@ -792,13 +792,6 @@ public: // Member functions void *buf, const int N_bytes, const int send_proc, const int tag ) const; - /*! - * @brief This function sends and recieves data using a blocking call - */ - template - void sendrecv( const type *sendbuf, int sendcount, int dest, int sendtag, type *recvbuf, int recvcount, int source, int recvtag ) const; - - /*! * Each processor sends every other processor a single value. * @param[in] x Input value for allGather diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 610fce5d..51195f5a 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -201,6 +201,7 @@ private: int sendtag,recvtag; // Give the object it's own MPI communicator RankInfoStruct rank_info; + MPI_Group Group; // Group of processors associated with this domain Utilities::MPI MPI_COMM_SCALBL; // MPI Communicator for this domain MPI_Request req1[18],req2[18]; //...................................................................................... diff --git a/common/Utilities.cpp b/common/Utilities.cpp index 723b34f8..9c89e024 100644 --- a/common/Utilities.cpp +++ b/common/Utilities.cpp @@ -8,7 +8,7 @@ #endif #ifdef USE_MPI -#include "common/MPI.h" +#include "mpi.h" #endif #include diff --git a/cpu/exe/lb2_Color_mpi.cpp b/cpu/exe/lb2_Color_mpi.cpp index cdf56af9..0cade21e 100644 --- a/cpu/exe/lb2_Color_mpi.cpp +++ b/cpu/exe/lb2_Color_mpi.cpp @@ -36,11 +36,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -54,6 +58,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -110,30 +115,31 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** // ************************************************************** @@ -163,7 +169,7 @@ int main(int argc, char **argv) } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -445,7 +451,7 @@ int main(int argc, char **argv) PM.close(); // printf("File porosity = %f\n", double(sum)/N); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... // Write the communcation structure into a file for debugging @@ -582,7 +588,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -593,46 +599,46 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - comm.Send(&sendCount_x,1,rank_X,sendtag); - comm.Recv(&recvCount_X,1,rank_x,recvtag); - comm.Send(&sendCount_X,1,rank_x,sendtag); - comm.Recv(&recvCount_x,1,rank_X,recvtag); - comm.Send(&sendCount_y,1,rank_Y,sendtag); - comm.Recv(&recvCount_Y,1,rank_y,recvtag); - comm.Send(&sendCount_Y,1,rank_y,sendtag); - comm.Recv(&recvCount_y,1,rank_Y,recvtag); - comm.Send(&sendCount_z,1,rank_Z,sendtag); - comm.Recv(&recvCount_Z,1,rank_z,recvtag); - comm.Send(&sendCount_Z,1,rank_z,sendtag); - comm.Recv(&recvCount_z,1,rank_Z,recvtag); + MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_xy,1,rank_XY,sendtag); - comm.Recv(&recvCount_XY,1,rank_xy,recvtag); - comm.Send(&sendCount_XY,1,rank_xy,sendtag); - comm.Recv(&recvCount_xy,1,rank_XY,recvtag); - comm.Send(&sendCount_Xy,1,rank_xY,sendtag); - comm.Recv(&recvCount_xY,1,rank_Xy,recvtag); - comm.Send(&sendCount_xY,1,rank_Xy,sendtag); - comm.Recv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_xz,1,rank_XZ,sendtag); - comm.Recv(&recvCount_XZ,1,rank_xz,recvtag); - comm.Send(&sendCount_XZ,1,rank_xz,sendtag); - comm.Recv(&recvCount_xz,1,rank_XZ,recvtag); - comm.Send(&sendCount_Xz,1,rank_xZ,sendtag); - comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag); - comm.Send(&sendCount_xZ,1,rank_Xz,sendtag); - comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_yz,1,rank_YZ,sendtag); - comm.Recv(&recvCount_YZ,1,rank_yz,recvtag); - comm.Send(&sendCount_YZ,1,rank_yz,sendtag); - comm.Recv(&recvCount_yz,1,rank_YZ,recvtag); - comm.Send(&sendCount_Yz,1,rank_yZ,sendtag); - comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag); - comm.Send(&sendCount_yZ,1,rank_Yz,sendtag); - comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.barrier(); + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); //********************************************************************************** //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; @@ -663,48 +669,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... for (int idx=0; idx #include #include -#include "common/MPI.h" +#include #include using namespace std; @@ -64,11 +64,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -82,6 +86,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; //********************************** //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -131,23 +136,24 @@ int main(int argc, char **argv) // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** double rlx_setA = 1.f/tau; @@ -170,7 +176,7 @@ int main(int argc, char **argv) printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz); } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -451,7 +457,7 @@ int main(int argc, char **argv) PM.close(); // printf("File porosity = %f\n", double(sum)/N); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... // Write the communcation structure into a file for debugging @@ -588,7 +594,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -599,46 +605,46 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - comm.send(&sendCount_x,1,rank_X,sendtag); - comm.recv(&recvCount_X,1,rank_x,recvtag); - comm.send(&sendCount_X,1,rank_x,sendtag); - comm.recv(&recvCount_x,1,rank_X,recvtag); - comm.send(&sendCount_y,1,rank_Y,sendtag); - comm.recv(&recvCount_Y,1,rank_y,recvtag); - comm.send(&sendCount_Y,1,rank_y,sendtag); - comm.recv(&recvCount_y,1,rank_Y,recvtag); - comm.send(&sendCount_z,1,rank_Z,sendtag); - comm.recv(&recvCount_Z,1,rank_z,recvtag); - comm.send(&sendCount_Z,1,rank_z,sendtag); - comm.recv(&recvCount_z,1,rank_Z,recvtag); + MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_xy,1,rank_XY,sendtag); - comm.recv(&recvCount_XY,1,rank_xy,recvtag); - comm.send(&sendCount_XY,1,rank_xy,sendtag); - comm.recv(&recvCount_xy,1,rank_XY,recvtag); - comm.send(&sendCount_Xy,1,rank_xY,sendtag); - comm.recv(&recvCount_xY,1,rank_Xy,recvtag); - comm.send(&sendCount_xY,1,rank_Xy,sendtag); - comm.recv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_xz,1,rank_XZ,sendtag); - comm.recv(&recvCount_XZ,1,rank_xz,recvtag); - comm.send(&sendCount_XZ,1,rank_xz,sendtag); - comm.recv(&recvCount_xz,1,rank_XZ,recvtag); - comm.send(&sendCount_Xz,1,rank_xZ,sendtag); - comm.recv(&recvCount_xZ,1,rank_Xz,recvtag); - comm.send(&sendCount_xZ,1,rank_Xz,sendtag); - comm.recv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_yz,1,rank_YZ,sendtag); - comm.recv(&recvCount_YZ,1,rank_yz,recvtag); - comm.send(&sendCount_YZ,1,rank_yz,sendtag); - comm.recv(&recvCount_yz,1,rank_YZ,recvtag); - comm.send(&sendCount_Yz,1,rank_yZ,sendtag); - comm.recv(&recvCount_yZ,1,rank_Yz,recvtag); - comm.send(&sendCount_yZ,1,rank_Yz,sendtag); - comm.recv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.barrier(); + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); //********************************************************************************** //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; @@ -669,48 +675,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z; double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ; @@ -909,24 +915,42 @@ int main(int argc, char **argv) PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id); PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id); //...................................................................................... - comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag); - comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag); - comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag); - comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag); - comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag); - comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag); - comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag); - comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag); - comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag); - comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag); - comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag); - comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag); - comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag); - comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag); - comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag); - comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag); - comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag); - comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag); + MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag, + recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag, + recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag, + recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag, + recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag, + recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag, + recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag, + recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag, + recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag, + recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag, + recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag, + recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag, + recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag, + recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag, + recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag, + recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag, + recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag, + recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag, + recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(recvList_x, recvCount_x ,recvID_x, id); UnpackID(recvList_X, recvCount_X ,recvID_X, id); @@ -959,7 +983,7 @@ int main(int argc, char **argv) free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz); //...................................................................................... if (rank==0) printf ("Devices are ready to communicate. \n"); - comm.barrier(); + MPI_Barrier(comm); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); @@ -999,8 +1023,8 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - comm.barrier(); - starttime = Utilities::MPI::time(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); // Old cuda timer is below // cudaEvent_t start, stop; // float time; @@ -1112,48 +1136,48 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag); - req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag); - req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag); - req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag); + MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]); //................................................................................... //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); //................................................................................... // Unpack the distributions on the device //................................................................................... @@ -1236,7 +1260,7 @@ int main(int argc, char **argv) //***************************************************************************** //***************************************************************************** - comm.barrier(); + MPI_Barrier(comm); // Iteration completed! timestep++; //................................................................... @@ -1245,8 +1269,8 @@ int main(int argc, char **argv) // cudaThreadSynchronize(); dvc_Barrier(); - comm.barrier(); - stoptime = Utilities::MPI::time(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; @@ -1280,7 +1304,7 @@ int main(int argc, char **argv) // dvc_CopyToDevice(velocity, vel, 3*dist_mem_size, dvc_CopyToDeviceDeviceToHost); //.............................................................................. // cudaThreadSynchronize(); -// comm.barrier(); +// MPI_Barrier(comm); //............................................................ //....Write the z-velocity to test poiseuille flow............ // double vz,vz_avg; @@ -1309,7 +1333,7 @@ int main(int argc, char **argv) // free (velocity); free(id); // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/gpu/exe/lb1_MRT_mpi.cu b/gpu/exe/lb1_MRT_mpi.cu index 776ea29f..0c0863c7 100644 --- a/gpu/exe/lb1_MRT_mpi.cu +++ b/gpu/exe/lb1_MRT_mpi.cu @@ -1,10 +1,8 @@ -#include "common/MPI.h" - #include #include #include #include - +#include inline void PackID(int *list, int count, char *sendbuf, char *ID){ // Fill in the phase ID values from neighboring processors @@ -555,11 +553,15 @@ void Write_Out(double *array, int Nx, int Ny, int Nz){ int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -573,6 +575,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; //********************************** //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -622,21 +625,24 @@ int main(int argc, char **argv) // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(&iterMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&iterMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + //................................................. + MPI_Barrier(comm); // ************************************************************** double rlx_setA = 1.f/tau; @@ -659,7 +665,7 @@ int main(int argc, char **argv) printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz); } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -940,7 +946,7 @@ int main(int argc, char **argv) PM.close(); // printf("File porosity = %f\n", double(sum)/N); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... // Write the communcation structure into a file for debugging @@ -1077,7 +1083,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -1088,46 +1094,46 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - comm.send(&sendCount_x,1,rank_X,sendtag); - comm.recv(&recvCount_X,1,rank_x,recvtag); - comm.send(&sendCount_X,1,rank_x,sendtag); - comm.recv(&recvCount_x,1,rank_X,recvtag); - comm.send(&sendCount_y,1,rank_Y,sendtag); - comm.recv(&recvCount_Y,1,rank_y,recvtag); - comm.send(&sendCount_Y,1,rank_y,sendtag); - comm.recv(&recvCount_y,1,rank_Y,recvtag); - comm.send(&sendCount_z,1,rank_Z,sendtag); - comm.recv(&recvCount_Z,1,rank_z,recvtag); - comm.send(&sendCount_Z,1,rank_z,sendtag); - comm.recv(&recvCount_z,1,rank_Z,recvtag); + MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_xy,1,rank_XY,sendtag); - comm.recv(&recvCount_XY,1,rank_xy,recvtag); - comm.send(&sendCount_XY,1,rank_xy,sendtag); - comm.recv(&recvCount_xy,1,rank_XY,recvtag); - comm.send(&sendCount_Xy,1,rank_xY,sendtag); - comm.recv(&recvCount_xY,1,rank_Xy,recvtag); - comm.send(&sendCount_xY,1,rank_Xy,sendtag); - comm.recv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_xz,1,rank_XZ,sendtag); - comm.recv(&recvCount_XZ,1,rank_xz,recvtag); - comm.send(&sendCount_XZ,1,rank_xz,sendtag); - comm.recv(&recvCount_xz,1,rank_XZ,recvtag); - comm.send(&sendCount_Xz,1,rank_xZ,sendtag); - comm.recv(&recvCount_xZ,1,rank_Xz,recvtag); - comm.send(&sendCount_xZ,1,rank_Xz,sendtag); - comm.recv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_yz,1,rank_YZ,sendtag); - comm.recv(&recvCount_YZ,1,rank_yz,recvtag); - comm.send(&sendCount_YZ,1,rank_yz,sendtag); - comm.recv(&recvCount_yz,1,rank_YZ,recvtag); - comm.send(&sendCount_Yz,1,rank_yZ,sendtag); - comm.recv(&recvCount_yZ,1,rank_Yz,recvtag); - comm.send(&sendCount_yZ,1,rank_Yz,sendtag); - comm.recv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.barrier(); + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); //********************************************************************************** //recvCount_x = sendCount_x; //recvCount_X = sendCount_X; @@ -1151,7 +1157,7 @@ int main(int argc, char **argv) //...................................................................................... // Use MPI to fill in the appropriate values // int tag = 5; - // Mcomm.sendrecv(sendCount_x,1,rank_x,tag,sendCount_X,1); + // MPI_Sendrecv(sendCount_x,1,MPI_INT,rank_x,tag,sendCount_X,1,MPI_INT,comm,req); //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; @@ -1181,48 +1187,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z; double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ; @@ -1421,24 +1427,42 @@ int main(int argc, char **argv) PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id); PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id); //...................................................................................... - comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag); - comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag); - comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag); - comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag); - comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag); - comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag); - comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag); - comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag); - comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag); - comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag); - comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag); - comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag); - comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag); - comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag); - comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag); - comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag); - comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag); - comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag); + MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag, + recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag, + recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag, + recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag, + recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag, + recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag, + recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag, + recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag, + recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag, + recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag, + recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag, + recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag, + recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag, + recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag, + recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag, + recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag, + recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag, + recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag, + recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(recvList_x, recvCount_x ,recvID_x, id); UnpackID(recvList_X, recvCount_X ,recvID_X, id); @@ -1471,7 +1495,7 @@ int main(int argc, char **argv) free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz); //...................................................................................... if (rank==0) printf ("Devices are ready to communicate. \n"); - comm.barrier(); + MPI_Barrier(comm); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); @@ -1511,8 +1535,8 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - comm.barrier(); - starttime = Utilities::MPI::time(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); // Old cuda timer is below // cudaEvent_t start, stop; // float time; @@ -1609,48 +1633,48 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag); - req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag); - req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag); - req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag); + MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]); //................................................................................... //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); //................................................................................... // Unpack the distributions on the device //................................................................................... @@ -1734,7 +1758,7 @@ int main(int argc, char **argv) //***************************************************************************** //***************************************************************************** - comm.barrier(); + MPI_Barrier(comm); // Iteration completed! iter++; //................................................................... @@ -1742,8 +1766,8 @@ int main(int argc, char **argv) //************************************************************************/ cudaThreadSynchronize(); - comm.barrier(); - stoptime = Utilities::MPI::time(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*iter)/cputime/1000000 << " MLUPS" << endl; @@ -1778,7 +1802,7 @@ int main(int argc, char **argv) cudaMemcpy(velocity, vel, 3*dist_mem_size, cudaMemcpyDeviceToHost); //.............................................................................. cudaThreadSynchronize(); - comm.barrier(); + MPI_Barrier(comm); //............................................................ //....Write the z-velocity to test poiseuille flow............ double vz,vz_avg; @@ -1807,7 +1831,7 @@ int main(int argc, char **argv) free (velocity); free(id); // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/gpu/exe/lb2_Color.cu b/gpu/exe/lb2_Color.cu index 1f227d08..1871b23c 100644 --- a/gpu/exe/lb2_Color.cu +++ b/gpu/exe/lb2_Color.cu @@ -1,4 +1,6 @@ -#include "common/MPI.h" +#ifdef useMPI +#include +#endif #include #include @@ -60,10 +62,18 @@ int main(int argc, char *argv[]) { //********** Initialize MPI **************** + int numprocs,rank; +#ifdef useMPI + MPI_Status stat; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int numprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_size(comm,&numprocs); + MPI_Comm_rank(comm,&rank); +#else + MPI_Comm comm = MPI_COMM_WORLD; + numprocs = 1; + rank = 0; +#endif //****************************************** if (rank == 0){ @@ -113,31 +123,32 @@ int main(int argc, char *argv[]) input >> tol; // error tolerance //............................................................. } +#ifdef useMPI // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** +#endif double rlxA = 1.f/tau; double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA); @@ -232,7 +243,11 @@ int main(int argc, char *argv[]) if (k==4) k=Nz-5; } } - comm.bcast(&id[0],N,0); +#ifdef useMPI //............................................................ + MPI_Barrier(comm); + MPI_Bcast(&id[0],N,MPI_CHAR,0,comm); + MPI_Barrier(comm); +#endif if (rank == 0) printf("Domain set.\n"); //........................................................................... diff --git a/gpu/exe/lb2_Color_mpi.cpp b/gpu/exe/lb2_Color_mpi.cpp index a2f3d8a9..fe11d32f 100644 --- a/gpu/exe/lb2_Color_mpi.cpp +++ b/gpu/exe/lb2_Color_mpi.cpp @@ -2,7 +2,7 @@ #include #include #include -#include "common/MPI.h" +#include using namespace std; @@ -98,11 +98,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -116,6 +120,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -172,30 +177,31 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** // ************************************************************** @@ -225,7 +231,7 @@ int main(int argc, char **argv) } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -507,7 +513,7 @@ int main(int argc, char **argv) PM.close(); // printf("File porosity = %f\n", double(sum)/N); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... // Write the communcation structure into a file for debugging @@ -644,7 +650,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -655,46 +661,46 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - comm.Send(&sendCount_x,1,rank_X,sendtag); - comm.Recv(&recvCount_X,1,rank_x,recvtag); - comm.Send(&sendCount_X,1,rank_x,sendtag); - comm.Recv(&recvCount_x,1,rank_X,recvtag); - comm.Send(&sendCount_y,1,rank_Y,sendtag); - comm.Recv(&recvCount_Y,1,rank_y,recvtag); - comm.Send(&sendCount_Y,1,rank_y,sendtag); - comm.Recv(&recvCount_y,1,rank_Y,recvtag); - comm.Send(&sendCount_z,1,rank_Z,sendtag); - comm.Recv(&recvCount_Z,1,rank_z,recvtag); - comm.Send(&sendCount_Z,1,rank_z,sendtag); - comm.Recv(&recvCount_z,1,rank_Z,recvtag); + MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_xy,1,rank_XY,sendtag); - comm.Recv(&recvCount_XY,1,rank_xy,recvtag); - comm.Send(&sendCount_XY,1,rank_xy,sendtag); - comm.Recv(&recvCount_xy,1,rank_XY,recvtag); - comm.Send(&sendCount_Xy,1,rank_xY,sendtag); - comm.Recv(&recvCount_xY,1,rank_Xy,recvtag); - comm.Send(&sendCount_xY,1,rank_Xy,sendtag); - comm.Recv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_xz,1,rank_XZ,sendtag); - comm.Recv(&recvCount_XZ,1,rank_xz,recvtag); - comm.Send(&sendCount_XZ,1,rank_xz,sendtag); - comm.Recv(&recvCount_xz,1,rank_XZ,recvtag); - comm.Send(&sendCount_Xz,1,rank_xZ,sendtag); - comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag); - comm.Send(&sendCount_xZ,1,rank_Xz,sendtag); - comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_yz,1,rank_YZ,sendtag); - comm.Recv(&recvCount_YZ,1,rank_yz,recvtag); - comm.Send(&sendCount_YZ,1,rank_yz,sendtag); - comm.Recv(&recvCount_yz,1,rank_YZ,recvtag); - comm.Send(&sendCount_Yz,1,rank_yZ,sendtag); - comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag); - comm.Send(&sendCount_yZ,1,rank_Yz,sendtag); - comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.barrier(); + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); //********************************************************************************** //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; @@ -725,48 +731,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... for (int idx=0; idx #include #include -#include "common/MPI.h" +#include #include "pmmc.h" #include "Domain.h" @@ -101,11 +101,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -119,6 +123,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -198,35 +203,35 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** // ************************************************************** double Ps = -(das-dbs)/(das+dbs); @@ -258,7 +263,7 @@ int main(int argc, char **argv) printf("********************************************************\n"); } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -556,14 +561,14 @@ int main(int argc, char **argv) //....................................................................... if (rank == 0) printf("Reading the sphere packing \n"); if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad); - comm.barrier(); + MPI_Barrier(comm); // Broadcast the sphere packing to all processes - comm.bcast(cx,nspheres,0); - comm.bcast(cy,nspheres,0); - comm.bcast(cz,nspheres,0); - comm.bcast(rad,nspheres,0); + MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //....................................................................... // sprintf(LocalRankString,"%05d",rank); @@ -713,7 +718,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -724,49 +729,89 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - req1[0] = comm.Isend(&sendCount_x,1,rank_X,sendtag); - req2[0] = comm.Irecv(&recvCount_X,1,rank_x,recvtag); - req1[1] = comm.Isend(&sendCount_X,1,rank_x,sendtag); - req2[1] = comm.Irecv(&recvCount_x,1,rank_X,recvtag); - req1[2] = comm.Isend(&sendCount_y,1,rank_Y,sendtag); - req2[2] = comm.Irecv(&recvCount_Y,1,rank_y,recvtag); - req1[3] = comm.Isend(&sendCount_Y,1,rank_y,sendtag); - req2[3] = comm.Irecv(&recvCount_y,1,rank_Y,recvtag); - req1[4] = comm.Isend(&sendCount_z,1,rank_Z,sendtag); - req2[4] = comm.Irecv(&recvCount_Z,1,rank_z,recvtag); - req1[5] = comm.Isend(&sendCount_Z,1,rank_z,sendtag); - req2[5] = comm.Irecv(&recvCount_z,1,rank_Z,recvtag); + MPI_Isend(&sendCount_x, 1,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(&sendCount_X, 1,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(&sendCount_y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(&sendCount_z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(&sendCount_xy,1,rank_XY,sendtag); - req2[6] = comm.Irecv(&recvCount_XY,1,rank_xy,recvtag); - req1[7] = comm.Isend(&sendCount_XY,1,rank_xy,sendtag); - req2[7] = comm.Irecv(&recvCount_xy,1,rank_XY,recvtag); - req1[8] = comm.Isend(&sendCount_Xy,1,rank_xY,sendtag); - req2[8] = comm.Irecv(&recvCount_xY,1,rank_Xy,recvtag); - req1[9] = comm.Isend(&sendCount_xY,1,rank_Xy,sendtag); - req2[9] = comm.Irecv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(&sendCount_xz,1,rank_XZ,sendtag); - req2[10] = comm.Irecv(&recvCount_XZ,1,rank_xz,recvtag); - req1[11] = comm.Isend(&sendCount_XZ,1,rank_xz,sendtag); - req2[11] = comm.Irecv(&recvCount_xz,1,rank_XZ,recvtag); - req1[12] = comm.Isend(&sendCount_Xz,1,rank_xZ,sendtag); - req2[12] = comm.Irecv(&recvCount_xZ,1,rank_Xz,recvtag); - req1[13] = comm.Isend(&sendCount_xZ,1,rank_Xz,sendtag); - req2[13] = comm.Irecv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(&sendCount_yz,1,rank_YZ,sendtag); - req2[14] = comm.Irecv(&recvCount_YZ,1,rank_yz,recvtag); - req1[15] = comm.Isend(&sendCount_YZ,1,rank_yz,sendtag); - req2[15] = comm.Irecv(&recvCount_yz,1,rank_YZ,recvtag); - req1[16] = comm.Isend(&sendCount_Yz,1,rank_yZ,sendtag); - req2[16] = comm.Irecv(&recvCount_yZ,1,rank_Yz,recvtag); - req1[17] = comm.Isend(&sendCount_yZ,1,rank_Yz,sendtag); - req2[17] = comm.Irecv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); - //********************************************************************************** + MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); +/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); + + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); + + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); + + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); +*/ //********************************************************************************** //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; @@ -796,48 +841,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... for (int idx=0; idxkeyExists( "GridFile" )){ // Read the local domain data - auto input_id = readMicroCT( *domain_db, comm ); + auto input_id = readMicroCT( *domain_db, MPI_COMM_WORLD ); // Fill the halo (assuming GCW of 1) array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); - fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + fillHalo fill( MPI_COMM_WORLD, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); Array id_view; id_view.viewRaw( size1, Mask->id ); fill.copy( input_id, id_view ); @@ -652,7 +652,7 @@ void ScaLBL_ColorModel::Run(){ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... //************ MAIN ITERATION LOOP ***************************************/ @@ -991,7 +991,7 @@ void ScaLBL_ColorModel::Run(){ //************************************************************************ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/models/DFHModel.cpp b/models/DFHModel.cpp index 9709b107..ced5853f 100644 --- a/models/DFHModel.cpp +++ b/models/DFHModel.cpp @@ -487,7 +487,7 @@ void ScaLBL_DFHModel::Run(){ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... //************ MAIN ITERATION LOOP ***************************************/ @@ -583,7 +583,7 @@ void ScaLBL_DFHModel::Run(){ //************************************************************************ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index 60847e54..23925930 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -227,7 +227,7 @@ void ScaLBL_MRTModel::Run(){ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax); if (rank==0) printf("********************************************************\n"); timestep=0; @@ -325,7 +325,7 @@ void ScaLBL_MRTModel::Run(){ } } //************************************************************************/ - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/BlobAnalyzeParallel.cpp b/tests/BlobAnalyzeParallel.cpp index 773309f9..48e9e230 100644 --- a/tests/BlobAnalyzeParallel.cpp +++ b/tests/BlobAnalyzeParallel.cpp @@ -138,16 +138,16 @@ int main(int argc, char **argv) } comm.barrier(); // Computational domain - comm.bcast(&nx,1,0); - comm.bcast(&ny,1,0); - comm.bcast(&nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&nx,1,MPI_INT,0,comm); + MPI_Bcast(&ny,1,MPI_INT,0,comm); + MPI_Bcast(&nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -291,7 +291,7 @@ int main(int argc, char **argv) } Dm.CommInit(); // Initialize communications for domains - sum_global = comm.sumReduce( sum ); + MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum_global/Dm.Volume; if (rank==0) printf("Porosity = %f \n",porosity); diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index d4340964..43434092 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -213,24 +213,42 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id); PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id); //...................................................................................... - Dm.Comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag); - Dm.Comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag); - Dm.Comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag); - Dm.Comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag); - Dm.Comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag); - Dm.Comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag); - Dm.Comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag); - Dm.Comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag); - Dm.Comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag); - Dm.Comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag); - Dm.Comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag); - Dm.Comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag); - Dm.Comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag); - Dm.Comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag); - Dm.Comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag); - Dm.Comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag); - Dm.Comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag); - Dm.Comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag); + MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag, + recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag, + recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag, + recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag, + recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag, + recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag, + recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag, + recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag, + recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag, + recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag, + recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag, + recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag, + recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag, + recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag, + recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag, + recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag, + recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag, + recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag, + recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id); UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id); diff --git a/tests/TestBlobAnalyze.cpp b/tests/TestBlobAnalyze.cpp index 19360fe3..63d928c1 100644 --- a/tests/TestBlobAnalyze.cpp +++ b/tests/TestBlobAnalyze.cpp @@ -190,16 +190,16 @@ int main(int argc, char **argv) } comm.barrier(); // Computational domain - comm.bcast(&nx,1,0); - comm.bcast(&ny,1,0); - comm.bcast(&nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&nx,1,MPI_INT,0,comm); + MPI_Bcast(&ny,1,MPI_INT,0,comm); + MPI_Bcast(&nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -255,10 +255,10 @@ int main(int argc, char **argv) comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,nspheres,0); - comm.bcast(cy,nspheres,0); - comm.bcast(cz,nspheres,0); - comm.bcast(rad,nspheres,0); + MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); //....................................................................... diff --git a/tests/TestBubble.cpp b/tests/TestBubble.cpp index 6eb74b37..e7e0ced8 100644 --- a/tests/TestBubble.cpp +++ b/tests/TestBubble.cpp @@ -45,6 +45,7 @@ int main(int argc, char **argv) int nprocx,nprocy,nprocz; MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -433,7 +434,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... //........................................................................... // MAIN VARIABLES INITIALIZED HERE @@ -808,25 +809,25 @@ int main(int argc, char **argv) } //........................................................................... comm.barrier(); - nwp_volume_global = comm.sumReduce( nwp_volume ); - awn_global = comm.sumReduce( awn ); - ans_global = comm.sumReduce( ans ); - aws_global = comm.sumReduce( aws ); - lwns_global = comm.sumReduce( lwns ); - As_global = comm.sumReduce( As ); - Jwn_global = comm.sumReduce( Jwn ); - efawns_global = comm.sumReduce( efawns ); + MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm); // Phase averages - vol_w_global = comm.sumReduce( vol_w ); - vol_n_global = comm.sumReduce( vol_n ); - paw_global = comm.sumReduce( paw ); - pan_global = comm.sumReduce( pan ); - vaw_global(0) = comm.sumReduce( vaw(0) ); - van_global(0) = comm.sumReduce( van(0) ); - vawn_global(0) = comm.sumReduce( vawn(0) ); - Gwn_global(0) = comm.sumReduce( Gwn(0) ); - Gns_global(0) = comm.sumReduce( Gns(0) ); - Gws_global(0) = comm.sumReduce( Gws(0) ); + MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm); comm.barrier(); //......................................................................... // Compute the change in the total surface energy based on the defined interval @@ -951,7 +952,7 @@ int main(int argc, char **argv) //************************************************************************/ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index 8b4f1a9b..7f5d0047 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -387,7 +387,7 @@ int main(int argc, char **argv) double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... err = 1.0; @@ -487,7 +487,7 @@ int main(int argc, char **argv) //************************************************************************ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/TestColorGrad.cpp b/tests/TestColorGrad.cpp index 2566f8c0..df1c1daf 100644 --- a/tests/TestColorGrad.cpp +++ b/tests/TestColorGrad.cpp @@ -114,16 +114,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); // ************************************************************** diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index c4a045ae..d2799355 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -378,7 +378,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... @@ -403,7 +403,7 @@ int main(int argc, char **argv) //................................................................... } //************************************************************************/ - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; diff --git a/tests/TestForceD3Q19.cpp b/tests/TestForceD3Q19.cpp index 31151584..f8569624 100644 --- a/tests/TestForceD3Q19.cpp +++ b/tests/TestForceD3Q19.cpp @@ -450,7 +450,7 @@ int main (int argc, char **argv) for (int i=0; iSendD3Q19(dist, &dist[10*Np]); @@ -244,7 +244,7 @@ int main(int argc, char **argv) //************************************************************************/ - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; diff --git a/tests/TestMRT.cpp b/tests/TestMRT.cpp index e4acba99..5f2c4449 100644 --- a/tests/TestMRT.cpp +++ b/tests/TestMRT.cpp @@ -580,16 +580,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); // ************************************************************** @@ -668,7 +668,7 @@ int main(int argc, char **argv) } } comm.barrier(); - sum = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -731,7 +731,7 @@ int main(int argc, char **argv) double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); while (timestep < timesteps) { @@ -752,7 +752,7 @@ int main(int argc, char **argv) } //************************************************************************/ - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; @@ -795,7 +795,7 @@ int main(int argc, char **argv) } } } - sum = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); double PoreVel = sum*iVol_global; if (rank==0) printf("Velocity = %f \n",PoreVel); diff --git a/tests/TestMicroCTReader.cpp b/tests/TestMicroCTReader.cpp index 52a5b9d3..9a54610c 100644 --- a/tests/TestMicroCTReader.cpp +++ b/tests/TestMicroCTReader.cpp @@ -62,6 +62,7 @@ int main(int argc, char **argv) int N_errors = ut.NumFailGlobal(); // Close MPI + MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return N_errors; } diff --git a/tests/TestMomentsD3Q19.cpp b/tests/TestMomentsD3Q19.cpp index 2660ed26..6bd3e8ff 100644 --- a/tests/TestMomentsD3Q19.cpp +++ b/tests/TestMomentsD3Q19.cpp @@ -539,7 +539,7 @@ int main (int argc, char **argv) error=count; // Finished - comm.barrier(); + MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return error; } diff --git a/tests/TestNetcdf.cpp b/tests/TestNetcdf.cpp index 3d0498d2..8768c9ea 100644 --- a/tests/TestNetcdf.cpp +++ b/tests/TestNetcdf.cpp @@ -116,7 +116,7 @@ int main(int argc, char **argv) PROFILE_SAVE("TestNetcdf"); // Close MPI - comm.barrier(); + MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return N_errors; } diff --git a/tests/TestSegDist.cpp b/tests/TestSegDist.cpp index ecb6d6b9..b5e23ec8 100644 --- a/tests/TestSegDist.cpp +++ b/tests/TestSegDist.cpp @@ -100,10 +100,10 @@ int main(int argc, char **argv) comm.barrier(); if (rank==0) printf("Initialized! Converting to Signed Distance function \n"); - double t1 = Utilities::MPI::time(); + double t1 = MPI_Wtime(); DoubleArray Distance(nx,ny,nz); CalcDist(Distance,id,Dm,{false,false,false}); - double t2 = Utilities::MPI::time(); + double t2 = MPI_Wtime(); if (rank==0) printf("Total time: %f seconds \n",t2-t1); diff --git a/tests/lb2_CMT_wia.cpp b/tests/lb2_CMT_wia.cpp index 389bc8a8..820428a3 100644 --- a/tests/lb2_CMT_wia.cpp +++ b/tests/lb2_CMT_wia.cpp @@ -292,18 +292,18 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x,2*sendCount_x,rank_x,sendtag); - req2[0] = comm.Irecv(recvbuf_X,2*recvCount_X,rank_X,recvtag); - req1[1] = comm.Isend(sendbuf_X,2*sendCount_X,rank_X,sendtag); - req2[1] = comm.Irecv(recvbuf_x,2*recvCount_x,rank_x,recvtag); - req1[2] = comm.Isend(sendbuf_y,2*sendCount_y,rank_y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y,2*recvCount_Y,rank_Y,recvtag); - req1[3] = comm.Isend(sendbuf_Y,2*sendCount_Y,rank_Y,sendtag); - req2[3] = comm.Irecv(recvbuf_y,2*recvCount_y,rank_y,recvtag); - req1[4] = comm.Isend(sendbuf_z,2*sendCount_z,rank_z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z,2*recvCount_Z,rank_Z,recvtag); - req1[5] = comm.Isend(sendbuf_Z,2*sendCount_Z,rank_Z,sendtag); - req2[5] = comm.Irecv(recvbuf_z,2*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); */ //................................................................................... ScaLBL_D3Q7_Swap(ID, &packed_even[0], &packed_odd[0], Nx, Ny, Nz); @@ -311,8 +311,8 @@ int main(int argc, char **argv) /* //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(6,req1); - comm.waitAll(6,req2); + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); //................................................................................... // Unpack the distributions on the device //................................................................................... @@ -358,7 +358,7 @@ int main(int argc, char **argv) fclose(PHASE); // Close MPI - comm.barrier(); + MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return 0; } diff --git a/tests/lb2_Color_blob_wia_mpi.cpp b/tests/lb2_Color_blob_wia_mpi.cpp index e3323612..70342176 100644 --- a/tests/lb2_Color_blob_wia_mpi.cpp +++ b/tests/lb2_Color_blob_wia_mpi.cpp @@ -114,6 +114,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -206,36 +207,36 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&phi_s,1,0); - comm.bcast(&wp_saturation,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&Restart,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); -// comm.bcast(&nBlocks,1,0); -// comm.bcast(&nthreads,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); +// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); +// MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -398,10 +399,10 @@ int main(int argc, char **argv) if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad); comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,nspheres,0); - comm.bcast(cy,nspheres,0); - comm.bcast(cz,nspheres,0); - comm.bcast(rad,nspheres,0); + MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); if (rank == 0) cout << "Domain set." << endl; @@ -417,7 +418,7 @@ int main(int argc, char **argv) D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx; printf("Sauter Mean Diameter (computed from sphere packing) = %f \n ",D); } - comm.bcast(&D,1,0); + MPI_Bcast(&D,1,MPI_DOUBLE,0,comm); //....................................................................... // sprintf(LocalRankString,"%05d",rank); @@ -477,7 +478,7 @@ int main(int argc, char **argv) id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; //......................................................... sum_local = 1.0*sum; - porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = porosity*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -885,24 +886,42 @@ int main(int argc, char **argv) PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id); PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id); //...................................................................................... - comm.sendrecv(sendID_x,sendCount_x,rank_x,sendtag,recvID_X,recvCount_X,rank_X,recvtag); - comm.sendrecv(sendID_X,sendCount_X,rank_X,sendtag,recvID_x,recvCount_x,rank_x,recvtag); - comm.sendrecv(sendID_y,sendCount_y,rank_y,sendtag,recvID_Y,recvCount_Y,rank_Y,recvtag); - comm.sendrecv(sendID_Y,sendCount_Y,rank_Y,sendtag,recvID_y,recvCount_y,rank_y,recvtag); - comm.sendrecv(sendID_z,sendCount_z,rank_z,sendtag,recvID_Z,recvCount_Z,rank_Z,recvtag); - comm.sendrecv(sendID_Z,sendCount_Z,rank_Z,sendtag,recvID_z,recvCount_z,rank_z,recvtag); - comm.sendrecv(sendID_xy,sendCount_xy,rank_xy,sendtag,recvID_XY,recvCount_XY,rank_XY,recvtag); - comm.sendrecv(sendID_XY,sendCount_XY,rank_XY,sendtag,recvID_xy,recvCount_xy,rank_xy,recvtag); - comm.sendrecv(sendID_Xy,sendCount_Xy,rank_Xy,sendtag,recvID_xY,recvCount_xY,rank_xY,recvtag); - comm.sendrecv(sendID_xY,sendCount_xY,rank_xY,sendtag,recvID_Xy,recvCount_Xy,rank_Xy,recvtag); - comm.sendrecv(sendID_xz,sendCount_xz,rank_xz,sendtag,recvID_XZ,recvCount_XZ,rank_XZ,recvtag); - comm.sendrecv(sendID_XZ,sendCount_XZ,rank_XZ,sendtag,recvID_xz,recvCount_xz,rank_xz,recvtag); - comm.sendrecv(sendID_Xz,sendCount_Xz,rank_Xz,sendtag,recvID_xZ,recvCount_xZ,rank_xZ,recvtag); - comm.sendrecv(sendID_xZ,sendCount_xZ,rank_xZ,sendtag,recvID_Xz,recvCount_Xz,rank_Xz,recvtag); - comm.sendrecv(sendID_yz,sendCount_yz,rank_yz,sendtag,recvID_YZ,recvCount_YZ,rank_YZ,recvtag); - comm.sendrecv(sendID_YZ,sendCount_YZ,rank_YZ,sendtag,recvID_yz,recvCount_yz,rank_yz,recvtag); - comm.sendrecv(sendID_Yz,sendCount_Yz,rank_Yz,sendtag,recvID_yZ,recvCount_yZ,rank_yZ,recvtag); - comm.sendrecv(sendID_yZ,sendCount_yZ,rank_yZ,sendtag,recvID_Yz,recvCount_Yz,rank_Yz,recvtag); + MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag, + recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag, + recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag, + recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag, + recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag, + recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag, + recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag, + recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag, + recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag, + recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag, + recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag, + recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag, + recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag, + recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag, + recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag, + recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag, + recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag, + recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag, + recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(recvList_x, recvCount_x ,recvID_x, id); UnpackID(recvList_X, recvCount_X ,recvID_X, id); @@ -1361,48 +1380,48 @@ int main(int argc, char **argv) //................................................................................... // Send / Recv all the phase indcator field values //................................................................................... - req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_x,sendtag); - req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag); - req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_X,sendtag); - req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag); - req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag); - req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag); - req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag); - req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag); - req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag); - req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag); - req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag); - req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag); - req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag); - req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag); - req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag); - req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag); - req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag); - req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag); - req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag); - req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag); - req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag); - req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag); - req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag); - req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag); - req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag); - req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag); - req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag); - req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag); - req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag); - req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag); - req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag); - req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag); - req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag); - req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag); + MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]); //................................................................................... //................................................................................... // Wait for completion of Indicator Field communication //................................................................................... - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... @@ -1478,7 +1497,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... sendtag = recvtag = 5; @@ -1574,42 +1593,42 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x, 5*sendCount_x,rank_x,sendtag); - req2[0] = comm.Irecv(recvbuf_X, 5*recvCount_X,rank_X,recvtag); - req1[1] = comm.Isend(sendbuf_X, 5*sendCount_X,rank_X,sendtag); - req2[1] = comm.Irecv(recvbuf_x, 5*recvCount_x,rank_x,recvtag); - req1[2] = comm.Isend(sendbuf_y, 5*sendCount_y,rank_y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y, 5*recvCount_Y,rank_Y,recvtag); - req1[3] = comm.Isend(sendbuf_Y, 5*sendCount_Y,rank_Y,sendtag); - req2[3] = comm.Irecv(recvbuf_y, 5*recvCount_y,rank_y,recvtag); - req1[4] = comm.Isend(sendbuf_z, 5*sendCount_z,rank_z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z, 5*recvCount_Z,rank_Z,recvtag); - req1[5] = comm.Isend(sendbuf_Z, 5*sendCount_Z,rank_Z,sendtag); - req2[5] = comm.Irecv(recvbuf_z, 5*recvCount_z,rank_z,recvtag); - req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag); - req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag); - req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag); - req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag); - req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag); - req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag); - req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag); - req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag); - req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag); - req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag); - req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag); - req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag); - req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag); - req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag); - req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag); - req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag); - req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag); - req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag); - req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag); - req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag); - req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag); - req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag); - req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag); - req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag); + MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]); //................................................................................... //************************************************************************* @@ -1629,8 +1648,8 @@ int main(int argc, char **argv) //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); //................................................................................... // Unpack the distributions on the device @@ -1724,18 +1743,18 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x, 2*sendCount_x,rank_x,sendtag); - req2[0] = comm.Irecv(recvbuf_X, 2*recvCount_X,rank_X,recvtag); - req1[1] = comm.Isend(sendbuf_X, 2*sendCount_X,rank_X,sendtag); - req2[1] = comm.Irecv(recvbuf_x, 2*recvCount_x,rank_x,recvtag); - req1[2] = comm.Isend(sendbuf_y, 2*sendCount_y,rank_y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y, 2*recvCount_Y,rank_Y,recvtag); - req1[3] = comm.Isend(sendbuf_Y, 2*sendCount_Y,rank_Y,sendtag); - req2[3] = comm.Irecv(recvbuf_y, 2*recvCount_y,rank_y,recvtag); - req1[4] = comm.Isend(sendbuf_z, 2*sendCount_z,rank_z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z, 2*recvCount_Z,rank_Z,recvtag); - req1[5] = comm.Isend(sendbuf_Z, 2*sendCount_Z,rank_Z,sendtag); - req2[5] = comm.Irecv(recvbuf_z, 2*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); //................................................................................... ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz); @@ -1743,8 +1762,8 @@ int main(int argc, char **argv) //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(6,req1); - comm.waitAll(6,req2); + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); //................................................................................... // Unpack the distributions on the device //................................................................................... @@ -1805,48 +1824,48 @@ int main(int argc, char **argv) //................................................................................... // Send / Recv all the phase indcator field values //................................................................................... - req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_x,sendtag,comm,&req1[0]); - req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag,comm,&req2[0]); - req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_X,sendtag,comm,&req1[1]); - req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag,comm,&req2[1]); - req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_y,sendtag,comm,&req1[2]); - req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag,comm,&req2[2]); - req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag,comm,&req1[3]); - req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag,comm,&req2[3]); - req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_z,sendtag,comm,&req1[4]); - req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag,comm,&req2[4]); - req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag,comm,&req1[5]); - req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag,comm,&req1[6]); - req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag,comm,&req2[6]); - req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag,comm,&req1[7]); - req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag,comm,&req2[7]); - req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag,comm,&req1[8]); - req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag,comm,&req2[8]); - req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag,comm,&req1[9]); - req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag,comm,&req1[10]); - req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag,comm,&req2[10]); - req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag,comm,&req1[11]); - req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag,comm,&req2[11]); - req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag,comm,&req1[12]); - req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag,comm,&req2[12]); - req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag,comm,&req1[13]); - req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag,comm,&req1[14]); - req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag,comm,&req2[14]); - req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag,comm,&req1[15]); - req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag,comm,&req2[15]); - req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag,comm,&req1[16]); - req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag,comm,&req2[16]); - req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag,comm,&req1[17]); - req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag,comm,&req2[17]); + MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]); //................................................................................... //................................................................................... // Wait for completion of Indicator Field communication //................................................................................... - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... @@ -2423,28 +2442,28 @@ int main(int argc, char **argv) //........................................................................... comm.barrier(); - nwp_volume_global = comm.sumReduce( nwp_volume ); - awn_global = comm.sumReduce( awn ); - ans_global = comm.sumReduce( ans ); - aws_global = comm.sumReduce( aws ); - lwns_global = comm.sumReduce( lwns ); - As_global = comm.sumReduce( As ); - Jwn_global = comm.sumReduce( Jwn ); - Kwn_global = comm.sumReduce( Kwn ); - efawns_global = comm.sumReduce( efawns ); + MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm); // Phase averages - vol_w_global = comm.sumReduce( vol_w ); - vol_n_global = comm.sumReduce( vol_n ); - paw_global = comm.sumReduce( paw ); - pan_global = comm.sumReduce( pan ); - vaw_global(0) = comm.sumReduce( vaw(0) ); - van_global(0) = comm.sumReduce( van(0) ); - vawn_global(0) = comm.sumReduce( vawn(0) ); - Gwn_global(0) = comm.sumReduce( Gwn(0) ); - Gns_global(0) = comm.sumReduce( Gns(0) ); - Gws_global(0) = comm.sumReduce( Gws(0) ); - trawn_global = comm.sumReduce( trawn ); - trJwn_global = comm.sumReduce( trJwn ); + MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,comm); comm.barrier(); //......................................................................... // Compute the change in the total surface energy based on the defined interval @@ -2670,7 +2689,7 @@ int main(int argc, char **argv) //************************************************************************/ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/lbpm_BGK_simulator.cpp b/tests/lbpm_BGK_simulator.cpp index 1ac61853..8b079900 100644 --- a/tests/lbpm_BGK_simulator.cpp +++ b/tests/lbpm_BGK_simulator.cpp @@ -97,28 +97,28 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&tau,1,0); - //comm.bcast(&pBC,1,0); - //comm.bcast(&Restart,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + //MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + // MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - //comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + //MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -249,7 +249,7 @@ int main(int argc, char **argv) } } } - sum = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -331,7 +331,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... double D32,Fo,Re,velocity,err1D,mag_force,vel_prev; @@ -410,7 +410,7 @@ int main(int argc, char **argv) //************************************************************************/ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/lbpm_color_macro_simulator.cpp b/tests/lbpm_color_macro_simulator.cpp index c92b0c45..97df6812 100644 --- a/tests/lbpm_color_macro_simulator.cpp +++ b/tests/lbpm_color_macro_simulator.cpp @@ -39,6 +39,9 @@ int main(int argc, char **argv) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; + if (rank == 0){ printf("********************************************************\n"); printf("Running Color LBM \n"); @@ -169,32 +172,32 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&tauA,1,0); - comm.bcast(&tauB,1,0); - comm.bcast(&rhoA,1,0); - comm.bcast(&rhoB,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&BoundaryCondition,1,0); - comm.bcast(&InitialCondition,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&RESTART_INTERVAL,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tauA,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&tauB,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&rhoA,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&rhoB,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm); + MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&RESTART_INTERVAL,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. flux = 0.f; @@ -319,7 +322,7 @@ int main(int argc, char **argv) timestep=0; } } - comm.bcast(×tep,1,0); + MPI_Bcast(×tep,1,MPI_INT,0,comm); FILE *RESTART = fopen(LocalRestartFile,"rb"); if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx"); readID=fread(id,1,N,RESTART); @@ -358,7 +361,7 @@ int main(int argc, char **argv) } } } - sum - comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); //......................................................... @@ -534,7 +537,7 @@ int main(int argc, char **argv) double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... err = 1.0; @@ -634,7 +637,7 @@ int main(int argc, char **argv) //************************************************************************ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/lbpm_disc_pp.cpp b/tests/lbpm_disc_pp.cpp index 41825c7d..20d41884 100644 --- a/tests/lbpm_disc_pp.cpp +++ b/tests/lbpm_disc_pp.cpp @@ -9,7 +9,7 @@ #include "analysis/pmmc.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI.h" // This includes mpi.h #include "common/SpherePack.h" /* @@ -147,6 +147,8 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; int depth; @@ -187,16 +189,16 @@ int main(int argc, char **argv) comm.barrier(); //................................................. // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&ndiscs,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&ndiscs,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -273,9 +275,9 @@ int main(int argc, char **argv) if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad); comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,ndiscs,0); - comm.bcast(cy,ndiscs,0); - comm.bcast(rad,ndiscs,0); + MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); if (rank == 0){ @@ -344,7 +346,7 @@ int main(int argc, char **argv) } } sum_local = 1.0*sum; - porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = porosity*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -360,7 +362,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_inkbottle_pp.cpp b/tests/lbpm_inkbottle_pp.cpp index ca188633..669ab8c0 100644 --- a/tests/lbpm_inkbottle_pp.cpp +++ b/tests/lbpm_inkbottle_pp.cpp @@ -81,16 +81,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -197,7 +197,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_juanes_bench_disc_pp.cpp b/tests/lbpm_juanes_bench_disc_pp.cpp index a90d43f8..47d8cb84 100644 --- a/tests/lbpm_juanes_bench_disc_pp.cpp +++ b/tests/lbpm_juanes_bench_disc_pp.cpp @@ -9,7 +9,7 @@ #include "analysis/pmmc.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI.h" // This includes mpi.h #include "common/SpherePack.h" /* @@ -147,6 +147,9 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; + if (rank == 0){ printf("********************************************************\n"); @@ -190,16 +193,16 @@ int main(int argc, char **argv) comm.barrier(); //................................................. // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&ndiscs,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&ndiscs,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -289,9 +292,9 @@ int main(int argc, char **argv) if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad); comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,ndiscs,0); - comm.bcast(cy,ndiscs,0); - comm.bcast(rad,ndiscs,0); + MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); /* if (rank == 0){ @@ -433,7 +436,7 @@ int main(int argc, char **argv) } } sum_local = 1.0*sum; - porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = porosity*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -449,7 +452,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_nondarcy_simulator.cpp b/tests/lbpm_nondarcy_simulator.cpp index a25fef69..096dc790 100644 --- a/tests/lbpm_nondarcy_simulator.cpp +++ b/tests/lbpm_nondarcy_simulator.cpp @@ -94,6 +94,8 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; double REYNOLDS_NUMBER = 100.f; if (argc > 1){ @@ -156,28 +158,28 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&tau,1,0); - //comm.bcast(&pBC,1,0); - //comm.bcast(&Restart,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + //MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + // MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -306,8 +308,8 @@ int main(int argc, char **argv) } } } - por_vol = comm.sumReduce( sum_local ); - //porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); + // MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = pore_vol*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); //......................................................... @@ -431,7 +433,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... double D32,vawx,vawy,vawz,Fo,Re,velocity,err1D,mag_force,vel_prev; @@ -552,7 +554,7 @@ int main(int argc, char **argv) fclose(NONDARCY); ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/lbpm_nonnewtonian_simulator.cpp b/tests/lbpm_nonnewtonian_simulator.cpp index bea3a814..ff8792e7 100644 --- a/tests/lbpm_nonnewtonian_simulator.cpp +++ b/tests/lbpm_nonnewtonian_simulator.cpp @@ -124,6 +124,8 @@ int main(int argc, char **argv) // int rank_xz,rank_XZ,rank_xZ,rank_Xz; // int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -426,8 +428,8 @@ int main(int argc, char **argv) } } - pore_vol = comm.sumReduce( sum_local ); /* 6 */ - //porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); /* 6 */ + //MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = pore_vol*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -572,7 +574,7 @@ int main(int argc, char **argv) timestep=5; } } - comm.bcast(×tep,1,0); + MPI_Bcast(×tep,1,MPI_INT,0,comm); // Read in the restart file to CPU buffers double *cDen = new double[2*N]; @@ -660,7 +662,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); /* * Create the thread pool @@ -808,7 +810,7 @@ int main(int argc, char **argv) //************************************************************************/ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; @@ -833,6 +835,20 @@ int main(int argc, char **argv) + + + + + + + + + + + + + + // Scrap // if (rank==0){ diff --git a/tests/lbpm_plates_pp.cpp b/tests/lbpm_plates_pp.cpp index 37191979..acd64f52 100644 --- a/tests/lbpm_plates_pp.cpp +++ b/tests/lbpm_plates_pp.cpp @@ -31,6 +31,8 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; double TubeRadius =15.0; double WIDTH; @@ -75,16 +77,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -174,7 +176,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_porenetwork_pp.cpp b/tests/lbpm_porenetwork_pp.cpp index 1715811f..4a6ccda7 100644 --- a/tests/lbpm_porenetwork_pp.cpp +++ b/tests/lbpm_porenetwork_pp.cpp @@ -24,6 +24,9 @@ int main(int argc, char **argv) int iproc,jproc,kproc; int sendtag,recvtag; //***************************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; + //********************************** int nsph,ncyl, BC; nsph = atoi(argv[1]); @@ -64,16 +67,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -266,7 +269,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); if (rank==0) printf("Pore volume = %f \n",pore_vol/double(Nx*Ny*Nz)); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_random_pp.cpp b/tests/lbpm_random_pp.cpp index 8318f50f..ad4b83cc 100644 --- a/tests/lbpm_random_pp.cpp +++ b/tests/lbpm_random_pp.cpp @@ -98,16 +98,16 @@ int main(int argc, char **argv) } comm.barrier(); // Computational domain - comm.bcast(&nx,1,0); - comm.bcast(&ny,1,0); - comm.bcast(&nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&nx,1,MPI_INT,0,comm); + MPI_Bcast(&ny,1,MPI_INT,0,comm); + MPI_Bcast(&nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -166,7 +166,7 @@ int main(int argc, char **argv) } } // total Global is the number of nodes in the pore-space - totalGlobal = sumReduce( count ); + MPI_Allreduce(&count,&totalGlobal,1,MPI_INT,MPI_SUM,comm); float porosity=float(totalGlobal)/(nprocx*nprocy*nprocz*(nx-2)*(ny-2)*(nz-2)); if (rank==0) printf("Media Porosity: %f \n",porosity); @@ -216,12 +216,12 @@ int main(int argc, char **argv) sizeY = SizeY[bin]; sizeZ = SizeZ[bin]; } - comm.bcast(&x,1,0); - comm.bcast(&y,1,0); - comm.bcast(&z,1,0); - comm.bcast(&sizeX,1,0); - comm.bcast(&sizeY,1,0); - comm.bcast(&sizeZ,1,0); + MPI_Bcast(&x,1,MPI_INT,0,comm); + MPI_Bcast(&y,1,MPI_INT,0,comm); + MPI_Bcast(&z,1,MPI_INT,0,comm); + MPI_Bcast(&sizeX,1,MPI_INT,0,comm); + MPI_Bcast(&sizeY,1,MPI_INT,0,comm); + MPI_Bcast(&sizeZ,1,MPI_INT,0,comm); //if (rank==0) printf("Broadcast block at %i,%i,%i \n",x,y,z); @@ -269,7 +269,7 @@ int main(int argc, char **argv) } } } - countGlobal = sumReduce( count ); + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); sat = float(countGlobal)/totalGlobal; //if (rank==0) printf("New count=%i\n",countGlobal); //if (rank==0) printf("New saturation=%f\n",sat); @@ -345,24 +345,42 @@ int main(int argc, char **argv) PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id); PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id); //...................................................................................... - comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag); - comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag); - comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag); - comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag); - comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag); - comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag); - comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag); - comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag); - comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag); - comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag); - comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag); - comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag); - comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag); - comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag); - comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag); - comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag); - comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag); - comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag); + MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag, + recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag, + recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag, + recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag, + recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag, + recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag, + recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag, + recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag, + recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag, + recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag, + recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag, + recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag, + recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag, + recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag, + recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag, + recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag, + recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag, + recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag, + recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id); UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id); @@ -394,7 +412,7 @@ int main(int argc, char **argv) } } } - countGlobal = comm.sumReduce( count ); + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); sat = float(countGlobal)/totalGlobal; if (rank==0) printf("Final saturation=%f\n",sat); diff --git a/tests/lbpm_segmented_decomp.cpp b/tests/lbpm_segmented_decomp.cpp index 65b8576f..1bc89adb 100644 --- a/tests/lbpm_segmented_decomp.cpp +++ b/tests/lbpm_segmented_decomp.cpp @@ -85,23 +85,23 @@ int main(int argc, char **argv) comm.barrier(); // Computational domain //................................................. - comm.bcast(&nx,1,0); - comm.bcast(&ny,1,0); - comm.bcast(&nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&nx,1,MPI_INT,0,comm); + MPI_Bcast(&ny,1,MPI_INT,0,comm); + MPI_Bcast(&nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&xStart,1,0); - comm.bcast(&yStart,1,0); - comm.bcast(&zStart,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&xStart,1,MPI_INT,0,comm); + MPI_Bcast(&yStart,1,MPI_INT,0,comm); + MPI_Bcast(&zStart,1,MPI_INT,0,comm); //................................................. comm.barrier(); @@ -191,7 +191,7 @@ int main(int argc, char **argv) } else{ printf("Sending data to process %i \n", rnk); - comm.send(tmp,N,rnk,15); + MPI_Send(tmp,N,MPI_CHAR,rnk,15,comm); } } } @@ -200,7 +200,7 @@ int main(int argc, char **argv) else{ // Recieve the subdomain from rank = 0 printf("Ready to recieve data %i at process %i \n", N,rank); - comm.recv(Dm.id,N,0,15); + MPI_Recv(Dm.id,N,MPI_CHAR,0,15,comm,MPI_STATUS_IGNORE); } comm.barrier(); @@ -243,8 +243,8 @@ int main(int argc, char **argv) printf("Original label=%i, New label=%i \n",oldlabel,newlabel); } } - comm.barrier(); - comm.bcast(LabelList,2*NLABELS,0); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Bcast(LabelList,2*NLABELS,MPI_INT,0,MPI_COMM_WORLD); char *newIDs; newIDs= new char [nx*ny*nz]; @@ -278,8 +278,8 @@ int main(int argc, char **argv) } } } - countGlobal = comm.sumReduce( count ); - totalGlobal = comm.sumReduce( total ); + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); + MPI_Allreduce(&total,&totalGlobal,1,MPI_INT,MPI_SUM,comm); float porosity = float(totalGlobal-countGlobal)/totalGlobal; @@ -321,8 +321,8 @@ int main(int argc, char **argv) } } } - countGlobal = comm.sumReduce( count ); - totalGlobal = comm.sumReduce( total ); + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); + MPI_Allreduce(&total,&totalGlobal,1,MPI_INT,MPI_SUM,comm); float saturation = float(countGlobal)/totalGlobal; if (rank==0) printf("wetting phase saturation=%f\n",saturation); diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index 484a11e2..39cf0bd1 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -180,7 +180,7 @@ int main(int argc, char **argv) fflush(stdout); porosity = ReadFromBlock(Dm->id,Dm->iproc(),Dm->jproc(),Dm->kproc(),nx,ny,nz); - comm.barrier(); + MPI_Barrier(MPI_COMM_WORLD); if (rank==0) printf("Writing local ID files (poros=%f) \n",porosity); fflush(stdout); FILE *ID = fopen(LocalRankFilename,"wb"); diff --git a/tests/lbpm_sphere_pp.cpp b/tests/lbpm_sphere_pp.cpp index 0df11b96..2e053eed 100644 --- a/tests/lbpm_sphere_pp.cpp +++ b/tests/lbpm_sphere_pp.cpp @@ -38,6 +38,8 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -123,10 +125,10 @@ int main(int argc, char **argv) if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad); comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,nspheres,0); - comm.bcast(cy,nspheres,0); - comm.bcast(cz,nspheres,0); - comm.bcast(rad,nspheres,0); + MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); if (rank == 0) cout << "Domain set." << endl; @@ -142,7 +144,7 @@ int main(int argc, char **argv) D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx; printf("Sauter Mean Diameter (computed from sphere packing) = %f \n",D); } - comm.bcast(&D,1,0); + MPI_Bcast(&D,1,MPI_DOUBLE,0,comm); //....................................................................... SignedDistance(SignDist.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz, @@ -175,7 +177,7 @@ int main(int argc, char **argv) } } sum_local = 1.0*sum; - porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = porosity*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -191,7 +193,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_squaretube_pp.cpp b/tests/lbpm_squaretube_pp.cpp index a4ee5f60..c1f05aee 100644 --- a/tests/lbpm_squaretube_pp.cpp +++ b/tests/lbpm_squaretube_pp.cpp @@ -30,6 +30,9 @@ int main(int argc, char **argv) int rank_xy,rank_XY,rank_xY,rank_Xy; int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; + //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; int ORIENTATION=2; //default: the tube is aligned with Z axis //ORIENTATION = 0: tube is aligned with X axis @@ -80,16 +83,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -232,7 +235,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners