OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring TestCommD3Q19

Browse Source
This commit is contained in:
James E McClure 2015-06-30 12:14:04 -04:00
parent cb6d89f9d3
commit 36ad8afe37
2 changed files with 39 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
common/ScaLBL.h
View File

@ -86,8 +86,33 @@ extern "C" void ColorBC_outlet(double *Phi, double *Den, double *A_even, double
double *B_even, double *B_odd, int Nx, int Ny, int Nz); double *B_even, double *B_odd, int Nx, int Ny, int Nz);
class ScaLBL_Communicator{ class ScaLBL_Communicator{
public:
//......................................................................................
ScaLBL_Communicator(Domain &Dm); ScaLBL_Communicator(Domain &Dm);
~ScaLBL_Communicator(); ~ScaLBL_Communicator();
//......................................................................................
unsigned long int CommunicationCount,SendCount,RecvCount;
//......................................................................................
// Set up for D319 distributions
// - determines how much memory is allocated
// - buffers are reused to send D3Q7 distributions and halo exchange as needed
//......................................................................................
// Buffers to store data sent and recieved by this MPI process
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
double *sendbuf_xY, *sendbuf_yZ, *sendbuf_Xz, *sendbuf_XY, *sendbuf_YZ, *sendbuf_XZ;
double *recvbuf_x, *recvbuf_y, *recvbuf_z, *recvbuf_X, *recvbuf_Y, *recvbuf_Z;
double *recvbuf_xy, *recvbuf_yz, *recvbuf_xz, *recvbuf_Xy, *recvbuf_Yz, *recvbuf_xZ;
double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ;
//......................................................................................
void SendD3Q19(double *f_even, double *f_odd);
void RecvD3Q19(double *f_even, double *f_odd);
void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
void SendHalo(double *data);
void RecvHalo(double *data);
private: private:
bool Lock; // use Lock to make sure only one call at a time to protect data in transit bool Lock; // use Lock to make sure only one call at a time to protect data in transit
// only one set of Send requests can be active at any time (per instance) // only one set of Send requests can be active at any time (per instance)
@ -129,27 +154,6 @@ private:
int *dvcRecvList_xY, *dvcRecvList_yZ, *dvcRecvList_Xz, *dvcRecvList_XY, *dvcRecvList_YZ, *dvcRecvList_XZ; int *dvcRecvList_xY, *dvcRecvList_yZ, *dvcRecvList_Xz, *dvcRecvList_XY, *dvcRecvList_YZ, *dvcRecvList_XZ;
//...................................................................................... //......................................................................................
public:
//......................................................................................
// Set up for D319 distributions
// - determines how much memory is allocated
// - buffers are reused to send D3Q7 distributions and halo exchange as needed
//......................................................................................
// Buffers to store data sent and recieved by this MPI process
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
double *sendbuf_xY, *sendbuf_yZ, *sendbuf_Xz, *sendbuf_XY, *sendbuf_YZ, *sendbuf_XZ;
double *recvbuf_x, *recvbuf_y, *recvbuf_z, *recvbuf_X, *recvbuf_Y, *recvbuf_Z;
double *recvbuf_xy, *recvbuf_yz, *recvbuf_xz, *recvbuf_Xy, *recvbuf_Yz, *recvbuf_xZ;
double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ;
//......................................................................................
void SendD3Q19(double *f_even, double *f_odd);
void RecvD3Q19(double *f_even, double *f_odd);
void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
void SendHalo(double *data);
void RecvHalo(double *data);
}; };
ScaLBL_Communicator::ScaLBL_Communicator(Domain &Dm){ ScaLBL_Communicator::ScaLBL_Communicator(Domain &Dm){
@ -436,6 +440,18 @@ void ScaLBL_Communicator::SendD3Q19(double *f_even, double *f_odd){
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]); MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]); MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]); MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]);
SendCount = sendCount_x+sendCount_X+sendCount_y+sendCount_Y+sendCount_z+sendCount_Z+
sendCount_xy+sendCount_Xy+sendCount_xY+sendCount_XY+
sendCount_xZ+sendCount_Xz+sendCount_xZ+sendCount_XZ+
sendCount_yz+sendCount_Yz+sendCount_yZ+sendCount_YZ;
RecvCount = recvCount_x+recvCount_X+recvCount_y+recvCount_Y+recvCount_z+recvCount_Z+
recvCount_xy+recvCount_Xy+recvCount_xY+recvCount_XY+
recvCount_xZ+recvCount_Xz+recvCount_xZ+recvCount_XZ+
recvCount_yz+recvCount_Yz+recvCount_yZ+recvCount_YZ;
CommunicationCount = SendCount+RecvCount;
} }
void ScaLBL_Communicator::RecvD3Q19(double *f_even, double *f_odd){ void ScaLBL_Communicator::RecvD3Q19(double *f_even, double *f_odd){

9
tests/TestCommD3Q19.cpp
View File

@ -453,11 +453,6 @@ int main(int argc, char **argv)
if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS); if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS);
if (rank==0) printf("********************************************************\n"); if (rank==0) printf("********************************************************\n");
// Number of double precision values sent and recieved by each process (per timestep)
double CommunicationCount = sendCount_x+sendCount_X+sendCount_y+sendCount_Y+sendCount_z+sendCount_Z+
sendCount_xy+sendCount_Xy+sendCount_xY+sendCount_XY+
sendCount_xZ+sendCount_Xz+sendCount_xZ+sendCount_XZ+
sendCount_yz+sendCount_Yz+sendCount_yZ+sendCount_YZ;
// Number of memory references from the swap algorithm (per timestep) // Number of memory references from the swap algorithm (per timestep)
// 18 reads and 18 writes for each lattice site // 18 reads and 18 writes for each lattice site
double MemoryRefs = (Nx-2)*(Ny-2)*(Nz-2)*36; double MemoryRefs = (Nx-2)*(Ny-2)*(Nz-2)*36;
@ -465,8 +460,8 @@ int main(int argc, char **argv)
if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*8*timestep/1e9); if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*8*timestep/1e9);
// Report bandwidth in Gigabits per second // Report bandwidth in Gigabits per second
// communication bandwidth includes both send and recieve // communication bandwidth includes both send and recieve
if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",CommunicationCount*128*timestep/1e9); if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9);
if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*CommunicationCount*128*timestep/1e9); if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9);
// **************************************************** // ****************************************************
MPI_Barrier(MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD);