OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Factoring some of the communication into common file. Fixing bug in domain decomposition for non-cube domains

Browse Source
This commit is contained in:
Mark Berrill
2014-02-01 10:33:03 -05:00
parent 4ab2084991
commit bd0e20a0db
7 changed files with 510 additions and 744 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
common/Communication.h
View File

@@ -1,6 +1,8 @@
#ifndef COMMUNICATION_H_INC
#define COMMUNICATION_H_INC
#include "Array.h"
// ********** COMMUNICTION **************************************
/*
//..............................................................
@@ -33,6 +35,184 @@ inline void UnpackMeshData(int *list, int count, double *recvbuf, DoubleArray &V
}
//***************************************************************************************
inline int getRankForBlock( int nprocx, int nprocy, int nprocz, int i, int j, int k )
{
int i2 = (i+nprocx)%nprocx;
int j2 = (j+nprocy)%nprocy;
int k2 = (k+nprocz)%nprocz;
return i2 + j2*nprocx + k2*nprocx*nprocy;
}
inline void InitializeRanks( const int rank, const int nprocx, const int nprocy, const int nprocz,
int& iproc, int& jproc, int& kproc,
int& rank_x, int& rank_y, int& rank_z,
int& rank_X, int& rank_Y, int& rank_Z,
int& rank_xy, int& rank_XY, int& rank_xY, int& rank_Xy,
int& rank_xz, int& rank_XZ, int& rank_xZ, int& rank_Xz,
int& rank_yz, int& rank_YZ, int& rank_yZ, int& rank_Yz )
{
// map the rank to the block index
iproc = rank%nprocx;
jproc = (rank/nprocx)%nprocy;
kproc = rank/(nprocx*nprocy);
// set up the neighbor ranks
int i = iproc;
int j = jproc;
int k = kproc;
rank_X = getRankForBlock(nprocx,nprocy,nprocz,i+1,j,k);
rank_x = getRankForBlock(nprocx,nprocy,nprocz,i-1,j,k);
rank_Y = getRankForBlock(nprocx,nprocy,nprocz,i,j+1,k);
rank_y = getRankForBlock(nprocx,nprocy,nprocz,i,j-1,k);
rank_Z = getRankForBlock(nprocx,nprocy,nprocz,i,j,k+1);
rank_z = getRankForBlock(nprocx,nprocy,nprocz,i,j,k-1);
rank_XY = getRankForBlock(nprocx,nprocy,nprocz,i+1,j+1,k);
rank_xy = getRankForBlock(nprocx,nprocy,nprocz,i-1,j-1,k);
rank_Xy = getRankForBlock(nprocx,nprocy,nprocz,i+1,j-1,k);
rank_xY = getRankForBlock(nprocx,nprocy,nprocz,i-1,j+1,k);
rank_XZ = getRankForBlock(nprocx,nprocy,nprocz,i+1,j,k+1);
rank_xz = getRankForBlock(nprocx,nprocy,nprocz,i-1,j,k-1);
rank_Xz = getRankForBlock(nprocx,nprocy,nprocz,i+1,j,k-1);
rank_xZ = getRankForBlock(nprocx,nprocy,nprocz,i-1,j,k+1);
rank_YZ = getRankForBlock(nprocx,nprocy,nprocz,i,j+1,k+1);
rank_yz = getRankForBlock(nprocx,nprocy,nprocz,i,j-1,k-1);
rank_Yz = getRankForBlock(nprocx,nprocy,nprocz,i,j+1,k-1);
rank_yZ = getRankForBlock(nprocx,nprocy,nprocz,i,j-1,k+1);
}
//***************************************************************************************
inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int recvtag,
int rank_x, int rank_y, int rank_z,
int rank_X, int rank_Y, int rank_Z,
int rank_xy, int rank_XY, int rank_xY, int rank_Xy,
int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz,
int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz,
int sendCount_x, int sendCount_y, int sendCount_z,
int sendCount_X, int sendCount_Y, int sendCount_Z,
int sendCount_xy, int sendCount_XY, int sendCount_xY, int sendCount_Xy,
int sendCount_xz, int sendCount_XZ, int sendCount_xZ, int sendCount_Xz,
int sendCount_yz, int sendCount_YZ, int sendCount_yZ, int sendCount_Yz,
int& recvCount_x, int& recvCount_y, int& recvCount_z,
int& recvCount_X, int& recvCount_Y, int& recvCount_Z,
int& recvCount_xy, int& recvCount_XY, int& recvCount_xY, int& recvCount_Xy,
int& recvCount_xz, int& recvCount_XZ, int& recvCount_xZ, int& recvCount_Xz,
int& recvCount_yz, int& recvCount_YZ, int& recvCount_yZ, int& recvCount_Yz )
{
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(3,req2,stat2);
MPI_Barrier(Communicator);
}
//***************************************************************************************
inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvtag,
int *sendList_x, int *sendList_y, int *sendList_z, int *sendList_X, int *sendList_Y, int *sendList_Z,
int *sendList_xy, int *sendList_XY, int *sendList_xY, int *sendList_Xy,
int *sendList_xz, int *sendList_XZ, int *sendList_xZ, int *sendList_Xz,
int *sendList_yz, int *sendList_YZ, int *sendList_yZ, int *sendList_Yz,
int sendCount_x, int sendCount_y, int sendCount_z, int sendCount_X, int sendCount_Y, int sendCount_Z,
int sendCount_xy, int sendCount_XY, int sendCount_xY, int sendCount_Xy,
int sendCount_xz, int sendCount_XZ, int sendCount_xZ, int sendCount_Xz,
int sendCount_yz, int sendCount_YZ, int sendCount_yZ, int sendCount_Yz,
int *recvList_x, int *recvList_y, int *recvList_z, int *recvList_X, int *recvList_Y, int *recvList_Z,
int *recvList_xy, int *recvList_XY, int *recvList_xY, int *recvList_Xy,
int *recvList_xz, int *recvList_XZ, int *recvList_xZ, int *recvList_Xz,
int *recvList_yz, int *recvList_YZ, int *recvList_yZ, int *recvList_Yz,
int recvCount_x, int recvCount_y, int recvCount_z, int recvCount_X, int recvCount_Y, int recvCount_Z,
int recvCount_xy, int recvCount_XY, int recvCount_xY, int recvCount_Xy,
int recvCount_xz, int recvCount_XZ, int recvCount_xZ, int recvCount_Xz,
int recvCount_yz, int recvCount_YZ, int recvCount_yZ, int recvCount_Yz,
int rank_x, int rank_y, int rank_z, int rank_X, int rank_Y, int rank_Z, int rank_xy, int rank_XY, int rank_xY,
int rank_Xy, int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz, int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz)
{
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
}
//***************************************************************************************
inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
@@ -43,24 +223,24 @@ inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
double *recvbuf_xy,double *recvbuf_XY,double *recvbuf_xY,double *recvbuf_Xy,
double *recvbuf_xz,double *recvbuf_XZ,double *recvbuf_xZ,double *recvbuf_Xz,
double *recvbuf_yz,double *recvbuf_YZ,double *recvbuf_yZ,double *recvbuf_Yz,
int *sendList_x,int *sendList_y,int *sendList_z,int *sendList_X,int *sendList_Y,int *sendList_Z,
int *sendList_xy,int *sendList_XY,int *sendList_xY,int *sendList_Xy,
int *sendList_xz,int *sendList_XZ,int *sendList_xZ,int *sendList_Xz,
int *sendList_yz,int *sendList_YZ,int *sendList_yZ,int *sendList_Yz,
int sendCount_x,int sendCount_y,int sendCount_z,int sendCount_X,int sendCount_Y,int sendCount_Z,
int sendCount_xy,int sendCount_XY,int sendCount_xY,int sendCount_Xy,
int sendCount_xz,int sendCount_XZ,int sendCount_xZ,int sendCount_Xz,
int sendCount_yz,int sendCount_YZ,int sendCount_yZ,int sendCount_Yz,
int *recvList_x,int *recvList_y,int *recvList_z,int *recvList_X,int *recvList_Y,int *recvList_Z,
int *recvList_xy,int *recvList_XY,int *recvList_xY,int *recvList_Xy,
int *recvList_xz,int *recvList_XZ,int *recvList_xZ,int *recvList_Xz,
int *recvList_yz,int *recvList_YZ,int *recvList_yZ,int *recvList_Yz,
int recvCount_x,int recvCount_y,int recvCount_z,int recvCount_X,int recvCount_Y,int recvCount_Z,
int recvCount_xy,int recvCount_XY,int recvCount_xY,int recvCount_Xy,
int recvCount_xz,int recvCount_XZ,int recvCount_xZ,int recvCount_Xz,
int recvCount_yz,int recvCount_YZ,int recvCount_yZ,int recvCount_Yz,
int rank_x,int rank_y,int rank_z,int rank_X,int rank_Y,int rank_Z,int rank_xy,int rank_XY,int rank_xY,
int rank_Xy,int rank_xz,int rank_XZ,int rank_xZ,int rank_Xz,int rank_yz,int rank_YZ,int rank_yZ,int rank_Yz)
int *sendList_x, int *sendList_y, int *sendList_z, int *sendList_X, int *sendList_Y, int *sendList_Z,
int *sendList_xy, int *sendList_XY, int *sendList_xY, int *sendList_Xy,
int *sendList_xz, int *sendList_XZ, int *sendList_xZ, int *sendList_Xz,
int *sendList_yz, int *sendList_YZ, int *sendList_yZ, int *sendList_Yz,
int sendCount_x, int sendCount_y, int sendCount_z, int sendCount_X, int sendCount_Y, int sendCount_Z,
int sendCount_xy, int sendCount_XY, int sendCount_xY, int sendCount_Xy,
int sendCount_xz, int sendCount_XZ, int sendCount_xZ, int sendCount_Xz,
int sendCount_yz, int sendCount_YZ, int sendCount_yZ, int sendCount_Yz,
int *recvList_x, int *recvList_y, int *recvList_z, int *recvList_X, int *recvList_Y, int *recvList_Z,
int *recvList_xy, int *recvList_XY, int *recvList_xY, int *recvList_Xy,
int *recvList_xz, int *recvList_XZ, int *recvList_xZ, int *recvList_Xz,
int *recvList_yz, int *recvList_YZ, int *recvList_yZ, int *recvList_Yz,
int recvCount_x, int recvCount_y, int recvCount_z, int recvCount_X, int recvCount_Y, int recvCount_Z,
int recvCount_xy, int recvCount_XY, int recvCount_xY, int recvCount_Xy,
int recvCount_xz, int recvCount_XZ, int recvCount_xZ, int recvCount_Xz,
int recvCount_yz, int recvCount_YZ, int recvCount_yZ, int recvCount_Yz,
int rank_x, int rank_y, int rank_z, int rank_X, int rank_Y, int rank_Z, int rank_xy, int rank_XY, int rank_xY,
int rank_Xy, int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz, int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz)
{
int sendtag, recvtag;
sendtag = recvtag = 7;

54
common/Domain.h
View File

@@ -6,46 +6,44 @@
#include <fstream>
#include <math.h>
#include <time.h>
#include <exception> // std::exception
#include <stdexcept>
#include <stdio.h>
using namespace std;
inline void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad)
{
// Read in the full sphere pack
int count;
double x,y,z,r;
//...... READ IN THE SPHERES...................................
char * trsh;
trsh = new char[100];
cout << "Reading the packing file..." << endl;
ifstream pack ("pack.out");
FILE *fid = fopen("pack.out","rb");
if ( fid==NULL) {
throw logic_error("Error opening pack.out");
}
//.........Trash the header lines..........
pack.getline(trsh, 100);
pack.getline(trsh, 100);
pack.getline(trsh, 100);
pack.getline(trsh, 100);
pack.getline(trsh, 100);
char * line = new char[100];
fgets(line, 100, fid);
fgets(line, 100, fid);
fgets(line, 100, fid);
fgets(line, 100, fid);
fgets(line, 100, fid);
//........read the spheres..................
count = 0;
pack >> x;
pack >> y;
pack >> z;
pack >> r;
while (! pack.eof()){
List_cx[count] = x;
List_cy[count] = y;
List_cz[count] = z;
List_rad[count] = r;
pack >> x;
pack >> y;
pack >> z;
pack >> r;
// We will read until a blank like or end-of-file is reached
int count = 0;
while ( !feof(fid) && fgets(line,100,fid)>0 ) {
char* line2 = line;
List_cx[count] = strtod(line2,&line2);
List_cy[count] = strtod(line2,&line2);
List_cz[count] = strtod(line2,&line2);
List_rad[count] = strtod(line2,&line2);
count++;
}
pack.close();
cout << "Number of spheres extracted is: " << count << endl;
if (count != nspheres){
printf("Specified number of spheres is probably incorrect!");
cout << "Number of spheres extracted is: " << count/4 << endl;
if ( count != nspheres ) {
throw logic_error("Specified number of spheres is probably incorrect!");
}
// .............................................................
}

1
tests/CMakeLists.txt
View File

@@ -6,6 +6,7 @@ CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cylindertest ${CMAKE_CURRENT_BINARY_
ADD_LBPM_TEST( pmmc_cylinder )
ADD_LBPM_TEST( TestCylinderAreas )
ADD_LBPM_TEST( TestSphereCurvature )
ADD_LBPM_TEST_1_2_4( testCommunication )
# Sample test that will run with 1, 2, and 4 processors, failing with 4 or more procs
ADD_LBPM_TEST_1_2_4( hello_world )

394
tests/cpu/lb2_Color_wia_mpi.cpp
View File

@@ -1,6 +1,8 @@
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include <mpi.h>
@@ -194,10 +196,10 @@ int main(int argc, char **argv)
double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA);
if (nprocs != nprocx*nprocy*nprocz){
printf("Fatal error in processor number! \n");
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
throw std::logic_error("Fatal error in processor number! \n");
}
if (rank==0){
@@ -218,230 +220,15 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
MPI_Barrier(MPI_COMM_WORLD);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
//..........................................
// set up the neighbor ranks
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j+=0;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_X = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j+=0;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_x = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j+=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_Y = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j-=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_y = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j+=0;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-= nprocy;
if (!(k<nprocz)) k-= nprocz;
rank_Z = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j+=0;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-= nprocy;
if (!(k<nprocz)) k-= nprocz;
rank_z = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j+=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_XY = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j-=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_xy = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j-=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_Xy = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j+=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_xY = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j+=0;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_XZ = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j+=0;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_xz = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j+=0;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_xZ = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j+=0;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_Xz = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j+=1;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_YZ = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j-=1;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_yz = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j-=1;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_yZ = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k=kproc;
i+=0;
j+=1;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_Yz = k*nprocx*nprocy+j*nprocx+i;
//..........................................
//**********************************
InitializeRanks( rank, nprocx, nprocy, nprocz,
iproc, jproc, kproc,
rank_x, rank_y, rank_z,
rank_X, rank_Y, rank_Z,
rank_xy, rank_XY, rank_xY, rank_Xy,
rank_xz, rank_XZ, rank_xZ, rank_Xz,
rank_yz, rank_YZ, rank_yZ, rank_Yz );
MPI_Barrier(MPI_COMM_WORLD);
Nz += 2;
Nx = Ny = Nz; // Cubic domain
@@ -547,7 +334,7 @@ int main(int argc, char **argv)
porosity = double(sum)/double(1.0*N);
#else
// Read in sphere pack
if (rank==1) printf("nspheres =%i \n",nspheres);
if (rank==0) printf("nspheres =%i \n",nspheres);
//.......................................................................
double *cx,*cy,*cz,*rad;
cx = new double[nspheres];
@@ -722,89 +509,20 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
CommunicateSendRecvCounts( MPI_COMM_WORLD, sendtag, recvtag,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
rank_xy, rank_XY, rank_xY, rank_Xy,
rank_xz, rank_XZ, rank_xZ, rank_Xz,
rank_yz, rank_YZ, rank_yZ, rank_Yz,
sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z,
sendCount_xy, sendCount_XY, sendCount_xY, sendCount_Xy,
sendCount_xz, sendCount_XZ, sendCount_xZ, sendCount_Xz,
sendCount_yz, sendCount_YZ, sendCount_yZ, sendCount_Yz,
recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z,
recvCount_xy, recvCount_XY, recvCount_xY, recvCount_Xy,
recvCount_xz, recvCount_XZ, recvCount_xZ, recvCount_Xz,
recvCount_yz, recvCount_YZ, recvCount_yZ, recvCount_Yz );
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(MPI_COMM_WORLD);
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Barrier(MPI_COMM_WORLD);
*/ //**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@@ -834,47 +552,25 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
CommunicateRecvLists( MPI_COMM_WORLD, sendtag, recvtag,
sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z,
sendList_xy, sendList_XY, sendList_xY, sendList_Xy,
sendList_xz, sendList_XZ, sendList_xZ, sendList_Xz,
sendList_yz, sendList_YZ, sendList_yZ, sendList_Yz,
sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z,
sendCount_xy, sendCount_XY, sendCount_xY, sendCount_Xy,
sendCount_xz, sendCount_XZ, sendCount_xZ, sendCount_Xz,
sendCount_yz, sendCount_YZ, sendCount_yZ, sendCount_Yz,
recvList_x, recvList_y, recvList_z, recvList_X, recvList_Y, recvList_Z,
recvList_xy, recvList_XY, recvList_xY, recvList_Xy,
recvList_xz, recvList_XZ, recvList_xZ, recvList_Xz,
recvList_yz, recvList_YZ, recvList_yZ, recvList_Yz,
recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z,
recvCount_xy, recvCount_XY, recvCount_xY, recvCount_Xy,
recvCount_xz, recvCount_XZ, recvCount_xZ, recvCount_Xz,
recvCount_yz, recvCount_YZ, recvCount_yZ, recvCount_Yz,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, rank_xy, rank_XY, rank_xY,
rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz );
MPI_Barrier(MPI_COMM_WORLD);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);

2
tests/gpu/CMakeLists.txt
View File

@@ -6,3 +6,5 @@ INSTALL_LBPM_EXE( lb2_Color_wia_mpi )
CONFIGURE_FILE( ${LBPM_SOURCE_DIR}/example/ConstrainedBubble/Color.in ${CMAKE_CURRENT_BINARY_DIR}/Color.in COPYONLY )
CONFIGURE_FILE( ${LBPM_SOURCE_DIR}/example/ConstrainedBubble/Domain.in ${CMAKE_CURRENT_BINARY_DIR}/Domain.in COPYONLY )
ADD_LBPM_WEEKLY_TEST( lb2_Color_wia_mpi 1 )

391
tests/gpu/lb2_Color_wia_mpi.cpp
View File

@@ -84,7 +84,6 @@ extern "C" void dvc_Barrier();
//*************************************************************************
// Implementation of Two-Phase Immiscible LBM using CUDA
//*************************************************************************
inline void PackID(int *list, int count, char *sendbuf, char *ID){
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
@@ -281,230 +280,15 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
MPI_Barrier(MPI_COMM_WORLD);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
//..........................................
// set up the neighbor ranks
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j+=0;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_X = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j+=0;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_x = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j+=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_Y = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j-=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_y = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j+=0;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-= nprocy;
if (!(k<nprocz)) k-= nprocz;
rank_Z = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j+=0;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-= nprocy;
if (!(k<nprocz)) k-= nprocz;
rank_z = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j+=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_XY = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j-=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_xy = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j-=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_Xy = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j+=1;
k+=0;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_xY = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j+=0;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_XZ = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j+=0;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_xz = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i-=1;
j+=0;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_xZ = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=1;
j+=0;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_Xz = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j+=1;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_YZ = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j-=1;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_yz = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k =kproc;
i+=0;
j-=1;
k+=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_yZ = k*nprocx*nprocy+j*nprocx+i;
//..........................................
i=iproc; j=jproc; k=kproc;
i+=0;
j+=1;
k-=1;
if (i<0) i+=nprocx;
if (j<0) j+=nprocy;
if (k<0) k+=nprocz;
if (!(i<nprocx)) i-= nprocx;
if (!(j<nprocy)) j-=nprocy;
if (!(k<nprocz)) k-=nprocz;
rank_Yz = k*nprocx*nprocy+j*nprocx+i;
//..........................................
//**********************************
InitializeRanks( rank, nprocx, nprocy, nprocz,
iproc, jproc, kproc,
rank_x, rank_y, rank_z,
rank_X, rank_Y, rank_Z,
rank_xy, rank_XY, rank_xY, rank_Xy,
rank_xz, rank_XZ, rank_xZ, rank_Xz,
rank_yz, rank_YZ, rank_yZ, rank_Yz );
MPI_Barrier(MPI_COMM_WORLD);
Nz += 2;
Nx = Ny = Nz; // Cubic domain
@@ -610,7 +394,7 @@ int main(int argc, char **argv)
porosity = double(sum)/double(1.0*N);
#else
// Read in sphere pack
if (rank==1) printf("nspheres =%i \n",nspheres);
if (rank==0) printf("nspheres =%i \n",nspheres);
//.......................................................................
double *cx,*cy,*cz,*rad;
cx = new double[nspheres];
@@ -797,89 +581,20 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
CommunicateSendRecvCounts( sendtag, recvtag,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
rank_xy, rank_XY, rank_xY, rank_Xy,
rank_xz, rank_XZ, rank_xZ, rank_Xz,
rank_yz, rank_YZ, rank_yZ, rank_Yz,
sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z,
sendCount_xy, sendCount_XY, sendCount_xY, sendCount_Xy,
sendCount_xz, sendCount_XZ, sendCount_xZ, sendCount_Xz,
sendCount_yz, sendCount_YZ, sendCount_yZ, sendCount_Yz,
recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z,
recvCount_xy, recvCount_XY, recvCount_xY, recvCount_Xy,
recvCount_xz, recvCount_XZ, recvCount_xZ, recvCount_Xz,
recvCount_yz, recvCount_YZ, recvCount_yZ, recvCount_Yz );
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(MPI_COMM_WORLD);
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
MPI_Barrier(MPI_COMM_WORLD);
*/ //**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@@ -909,47 +624,25 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,MPI_COMM_WORLD,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,MPI_COMM_WORLD,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,MPI_COMM_WORLD,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,MPI_COMM_WORLD,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,MPI_COMM_WORLD,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,MPI_COMM_WORLD,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,MPI_COMM_WORLD,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,MPI_COMM_WORLD,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,MPI_COMM_WORLD,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,MPI_COMM_WORLD,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,MPI_COMM_WORLD,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,MPI_COMM_WORLD,&req2[5]);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,MPI_COMM_WORLD,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,MPI_COMM_WORLD,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,MPI_COMM_WORLD,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,MPI_COMM_WORLD,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,MPI_COMM_WORLD,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,MPI_COMM_WORLD,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,MPI_COMM_WORLD,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,MPI_COMM_WORLD,&req2[9]);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,MPI_COMM_WORLD,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,MPI_COMM_WORLD,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,MPI_COMM_WORLD,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,MPI_COMM_WORLD,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,MPI_COMM_WORLD,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,MPI_COMM_WORLD,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,MPI_COMM_WORLD,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,MPI_COMM_WORLD,&req2[13]);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,MPI_COMM_WORLD,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,MPI_COMM_WORLD,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,MPI_COMM_WORLD,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,MPI_COMM_WORLD,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,MPI_COMM_WORLD,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,MPI_COMM_WORLD,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,MPI_COMM_WORLD,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,MPI_COMM_WORLD,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
CommunicateRecvLists( MPI_COMM_WORLD, sendtag, recvtag,
sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z,
sendList_xy, sendList_XY, sendList_xY, sendList_Xy,
sendList_xz, sendList_XZ, sendList_xZ, sendList_Xz,
sendList_yz, sendList_YZ, sendList_yZ, sendList_Yz,
sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z,
sendCount_xy, sendCount_XY, sendCount_xY, sendCount_Xy,
sendCount_xz, sendCount_XZ, sendCount_xZ, sendCount_Xz,
sendCount_yz, sendCount_YZ, sendCount_yZ, sendCount_Yz,
recvList_x, recvList_y, recvList_z, recvList_X, recvList_Y, recvList_Z,
recvList_xy, recvList_XY, recvList_xY, recvList_Xy,
recvList_xz, recvList_XZ, recvList_xZ, recvList_Xz,
recvList_yz, recvList_YZ, recvList_yZ, recvList_Yz,
recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z,
recvCount_xy, recvCount_XY, recvCount_xY, recvCount_Xy,
recvCount_xz, recvCount_XZ, recvCount_xZ, recvCount_Xz,
recvCount_yz, recvCount_YZ, recvCount_yZ, recvCount_Yz,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, rank_xy, rank_XY, rank_xY,
rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz );
MPI_Barrier(MPI_COMM_WORLD);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);

196
tests/testCommunication.cpp Normal file
View File

@@ -0,0 +1,196 @@
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include <mpi.h>
#include "Communication.h"
using namespace std;
//***************************************************************************************
int test_communication( MPI_Comm comm, int nprocx, int nprocy, int nprocz )
{
int rank,nprocs;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int iproc,jproc,kproc;
int sendtag,recvtag;
if (rank==0) printf("\nRunning test %i %i %i\n",nprocx,nprocy,nprocz);
//*****************************************
// MPI ranks for all 18 neighbors
//**********************************
int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z;
int rank_xy,rank_XY,rank_xY,rank_Xy;
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
InitializeRanks( rank, nprocx, nprocy, nprocz,
iproc, jproc, kproc,
rank_x, rank_y, rank_z,
rank_X, rank_Y, rank_Z,
rank_xy, rank_XY, rank_xY, rank_Xy,
rank_xz, rank_XZ, rank_xZ, rank_Xz,
rank_yz, rank_YZ, rank_yZ, rank_Yz );
MPI_Barrier(comm);
//**********************************
int Nx, Ny, Nz;
Nx = Ny = Nz = 10; // Cubic domain
// Set up MPI communication structurese
if (rank==0) printf ("Setting up communication control structures \n");
//......................................................................................
// Get the actual D3Q19 communication counts (based on location of solid phase)
// Discrete velocity set symmetry implies the sendcount = recvcount
int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0;
sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0;
sendCount_xY = sendCount_yZ = sendCount_Xz = sendCount_XY = sendCount_YZ = sendCount_XZ = 0;
//......................................................................................
int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z;
int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ;
int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ;
//......................................................................................
// send buffers
sendList_x = new int [sendCount_x];
sendList_y = new int [sendCount_y];
sendList_z = new int [sendCount_z];
sendList_X = new int [sendCount_X];
sendList_Y = new int [sendCount_Y];
sendList_Z = new int [sendCount_Z];
sendList_xy = new int [sendCount_xy];
sendList_yz = new int [sendCount_yz];
sendList_xz = new int [sendCount_xz];
sendList_Xy = new int [sendCount_Xy];
sendList_Yz = new int [sendCount_Yz];
sendList_xZ = new int [sendCount_xZ];
sendList_xY = new int [sendCount_xY];
sendList_yZ = new int [sendCount_yZ];
sendList_Xz = new int [sendCount_Xz];
sendList_XY = new int [sendCount_XY];
sendList_YZ = new int [sendCount_YZ];
sendList_XZ = new int [sendCount_XZ];
if (rank==0) printf ("Preparing the sendlists \n");
//......................................................................................
// Populate the send list
sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0;
sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0;
sendCount_xY = sendCount_yZ = sendCount_Xz = sendCount_XY = sendCount_YZ = sendCount_XZ = 0;
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
//......................................................................................
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
CommunicateSendRecvCounts( comm, sendtag, recvtag,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
rank_xy, rank_XY, rank_xY, rank_Xy,
rank_xz, rank_XZ, rank_xZ, rank_Xz,
rank_yz, rank_YZ, rank_yZ, rank_Yz,
sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z,
sendCount_xy, sendCount_XY, sendCount_xY, sendCount_Xy,
sendCount_xz, sendCount_XZ, sendCount_xZ, sendCount_Xz,
sendCount_yz, sendCount_YZ, sendCount_yZ, sendCount_Yz,
recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z,
recvCount_xy, recvCount_XY, recvCount_xY, recvCount_Xy,
recvCount_xz, recvCount_XZ, recvCount_xZ, recvCount_Xz,
recvCount_yz, recvCount_YZ, recvCount_yZ, recvCount_Yz );
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ;
//......................................................................................
// recv buffers
recvList_x = new int [recvCount_x];
recvList_y = new int [recvCount_y];
recvList_z = new int [recvCount_z];
recvList_X = new int [recvCount_X];
recvList_Y = new int [recvCount_Y];
recvList_Z = new int [recvCount_Z];
recvList_xy = new int [recvCount_xy];
recvList_yz = new int [recvCount_yz];
recvList_xz = new int [recvCount_xz];
recvList_Xy = new int [recvCount_Xy];
recvList_Yz = new int [recvCount_Yz];
recvList_xZ = new int [recvCount_xZ];
recvList_xY = new int [recvCount_xY];
recvList_yZ = new int [recvCount_yZ];
recvList_Xz = new int [recvCount_Xz];
recvList_XY = new int [recvCount_XY];
recvList_YZ = new int [recvCount_YZ];
recvList_XZ = new int [recvCount_XZ];
//......................................................................................
//......................................................................................
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
CommunicateRecvLists( comm, sendtag, recvtag,
sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z,
sendList_xy, sendList_XY, sendList_xY, sendList_Xy,
sendList_xz, sendList_XZ, sendList_xZ, sendList_Xz,
sendList_yz, sendList_YZ, sendList_yZ, sendList_Yz,
sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z,
sendCount_xy, sendCount_XY, sendCount_xY, sendCount_Xy,
sendCount_xz, sendCount_XZ, sendCount_xZ, sendCount_Xz,
sendCount_yz, sendCount_YZ, sendCount_yZ, sendCount_Yz,
recvList_x, recvList_y, recvList_z, recvList_X, recvList_Y, recvList_Z,
recvList_xy, recvList_XY, recvList_xY, recvList_Xy,
recvList_xz, recvList_XZ, recvList_xZ, recvList_Xz,
recvList_yz, recvList_YZ, recvList_yZ, recvList_Yz,
recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z,
recvCount_xy, recvCount_XY, recvCount_xY, recvCount_Xy,
recvCount_xz, recvCount_XZ, recvCount_xZ, recvCount_Xz,
recvCount_yz, recvCount_YZ, recvCount_yZ, recvCount_Yz,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, rank_xy, rank_XY, rank_xY,
rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz );
MPI_Barrier(comm);
if (rank==0) printf ("RecvLists finished\n");
// Finished with no errors
return 0;
}
int main(int argc, char **argv)
{
// Initialize MPI
int nprocs;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
// Run the test with different domains
int N_errors = 0;
N_errors += test_communication( MPI_COMM_WORLD, nprocs, 1, 1 );
N_errors += test_communication( MPI_COMM_WORLD, 1, nprocs, 1 );
N_errors += test_communication( MPI_COMM_WORLD, 1, 1, nprocs );
if ( nprocs==4 ) {
N_errors += test_communication( MPI_COMM_WORLD, 2, 2, 1 );
N_errors += test_communication( MPI_COMM_WORLD, 2, 1, 2 );
N_errors += test_communication( MPI_COMM_WORLD, 1, 2, 2 );
}
// Finished
MPI_Barrier(MPI_COMM_WORLD);
int N_errors_global=0;
MPI_Allreduce( &N_errors, &N_errors_global, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return N_errors_global;
}