OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

partial merge

Browse Source
This commit is contained in:
James McClure
2021-01-04 19:33:27 -05:00
parent a9bfe9d79d 2fc85dead8
commit 20800ed7eb
173 changed files with 27358 additions and 7516 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
cuda/Color.cu
View File

@@ -1240,6 +1240,15 @@ __global__ void dvc_ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny
}
__global__ void dvc_ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){
double value;
int n = blockIdx.x*blockDim.x + threadIdx.x;
if (n < Nx*Ny){
value = Phi[Source*Nx*Ny+n];
Phi[Dest*Nx*Ny+n] = value;
}
}
__global__ void dvc_ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
double *Velocity, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
@@ -4134,5 +4143,9 @@ extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den,
}
}
extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){
int GRID = Nx*Ny / 512 + 1;
dvc_ScaLBL_CopySlice_z<<<GRID,512>>>(Phi,Nx,Ny,Nz,Source,Dest);
}

63
cuda/D3Q19.cu
View File

@@ -267,7 +267,6 @@ __global__ void dvc_ScaLBL_D3Q19_Init(double *dist, int Np)
}
}
//*************************************************************************
__global__ void dvc_ScaLBL_D3Q19_Swap_Compact(int *neighborList, double *disteven, double *distodd, int Np, int q){
int n,nn;
@@ -1728,6 +1727,43 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist,
//...................................................
}
}
__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
int idx, n;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
double f5 = 0.111111111111111111111111 - dist[6*Np+n];
double f11 = 0.05555555555555555555556 - dist[12*Np+n];
double f14 = 0.05555555555555555555556 - dist[13*Np+n];
double f15 = 0.05555555555555555555556 - dist[16*Np+n];
double f18 = 0.05555555555555555555556 - dist[17*Np+n];
dist[6*Np+n] = f5;
dist[12*Np+n] = f11;
dist[13*Np+n] = f14;
dist[16*Np+n] = f15;
dist[17*Np+n] = f18;
}
}
__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
int idx, n;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
double f6 = 0.111111111111111111111111 - dist[5*Np+n];
double f12 = 0.05555555555555555555556 - dist[11*Np+n];
double f13 = 0.05555555555555555555556 - dist[14*Np+n] ;
double f16 = 0.05555555555555555555556 - dist[15*Np+n];
double f17 = 0.05555555555555555555556 - dist[18*Np+n];
dist[5*Np+n] = f6;
dist[11*Np+n] = f12;
dist[14*Np+n] = f13;
dist[15*Np+n] = f16;
dist[18*Np+n] = f17;
}
}
__global__ void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np)
{
@@ -2324,10 +2360,11 @@ extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np){
dvc_ScaLBL_D3Q19_Init<<<NBLOCKS,NTHREADS >>>(dist, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_AA_Init: %s \n",cudaGetErrorString(err));
printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){
dvc_ScaLBL_D3Q19_Swap<<<NBLOCKS,NTHREADS >>>(ID, disteven, distodd, Nx, Ny, Nz);
cudaError_t err = cudaGetLastError();
@@ -2614,11 +2651,23 @@ extern "C" double deviceReduce(double *in, double* out, int N) {
return sum;
}
//
//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(int *list, double *dist, double dout, int count, int Np){
// int GRID = count / 512 + 1;
// dvc_ScaLBL_D3Q19_Pressure_BC_Z<<<GRID,512>>>(disteven, distodd, dout, Nx, Ny, Nz, outlet);
//}
extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q19_Reflection_BC_z<<<GRID,512>>>(list, dist, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q19_Reflection_BC_Z<<<GRID,512>>>(list, dist, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
double Fy, double Fz){

38
cuda/D3Q7.cu
View File

@@ -1,4 +1,5 @@
// GPU Functions for D3Q7 Lattice Boltzmann Methods
#include <stdio.h>
#define NBLOCKS 560
#define NTHREADS 128
@@ -79,6 +80,25 @@ __global__ void dvc_ScaLBL_D3Q7_Unpack(int q, int *list, int start, int count,
}
}
__global__ void dvc_ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){
int idx, n;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
double f5 = 0.222222222222222222222222 - dist[6*Np+n];
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){
int idx, n;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
double f6 = 0.222222222222222222222222 - dist[5*Np+n];
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz)
{
int n,N;
@@ -207,6 +227,24 @@ __global__ void dvc_ScaLBL_D3Q7_Density(char *ID, double *disteven, double *dis
}
}
extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_Reflection_BC_z<<<GRID,512>>>(list, dist, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_Reflection_BC_Z<<<GRID,512>>>(list, dist, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_Unpack(int q, int *list, int start, int count, double *recvbuf, double *dist, int N){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_Unpack <<<GRID,512 >>>(q, list, start, count, recvbuf, dist, N);

348
cuda/exe/lb1_MRT_mpi.cpp
View File

@@ -6,7 +6,7 @@
#include <iostream>
#include <fstream>
#include <string.h>
#include "common/MPI.h"
#include <mpi.h>
#include <stdlib.h>
using namespace std;
@@ -64,11 +64,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@@ -82,6 +86,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//**********************************
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -131,23 +136,24 @@ int main(int argc, char **argv)
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
double rlx_setA = 1.f/tau;
@@ -170,7 +176,7 @@ int main(int argc, char **argv)
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@@ -451,7 +457,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@@ -588,7 +594,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@@ -599,46 +605,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
comm.send(&sendCount_x,1,rank_X,sendtag);
comm.recv(&recvCount_X,1,rank_x,recvtag);
comm.send(&sendCount_X,1,rank_x,sendtag);
comm.recv(&recvCount_x,1,rank_X,recvtag);
comm.send(&sendCount_y,1,rank_Y,sendtag);
comm.recv(&recvCount_Y,1,rank_y,recvtag);
comm.send(&sendCount_Y,1,rank_y,sendtag);
comm.recv(&recvCount_y,1,rank_Y,recvtag);
comm.send(&sendCount_z,1,rank_Z,sendtag);
comm.recv(&recvCount_Z,1,rank_z,recvtag);
comm.send(&sendCount_Z,1,rank_z,sendtag);
comm.recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xy,1,rank_XY,sendtag);
comm.recv(&recvCount_XY,1,rank_xy,recvtag);
comm.send(&sendCount_XY,1,rank_xy,sendtag);
comm.recv(&recvCount_xy,1,rank_XY,recvtag);
comm.send(&sendCount_Xy,1,rank_xY,sendtag);
comm.recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.send(&sendCount_xY,1,rank_Xy,sendtag);
comm.recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xz,1,rank_XZ,sendtag);
comm.recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.send(&sendCount_XZ,1,rank_xz,sendtag);
comm.recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_yz,1,rank_YZ,sendtag);
comm.recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.send(&sendCount_YZ,1,rank_yz,sendtag);
comm.recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
@@ -669,48 +675,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
@@ -909,24 +915,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@@ -959,7 +983,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
//......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@@ -999,8 +1023,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@@ -1112,48 +1136,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@@ -1236,7 +1260,7 @@ int main(int argc, char **argv)
//*****************************************************************************
//*****************************************************************************
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
timestep++;
//...................................................................
@@ -1245,8 +1269,8 @@ int main(int argc, char **argv)
// cudaThreadSynchronize();
dvc_Barrier();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@@ -1280,7 +1304,7 @@ int main(int argc, char **argv)
// dvc_CopyToDevice(velocity, vel, 3*dist_mem_size, dvc_CopyToDeviceDeviceToHost);
//..............................................................................
// cudaThreadSynchronize();
// comm.barrier();
// MPI_Barrier(comm);
//............................................................
//....Write the z-velocity to test poiseuille flow............
// double vz,vz_avg;
@@ -1309,7 +1333,7 @@ int main(int argc, char **argv)
// free (velocity); free(id);
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

352
cuda/exe/lb1_MRT_mpi.cu
View File

@@ -1,10 +1,8 @@
#include "common/MPI.h"
#include <stdio.h>
#include <iostream>
#include <fstream>
#include <cuda.h>
#include <mpi.h>
inline void PackID(int *list, int count, char *sendbuf, char *ID){
// Fill in the phase ID values from neighboring processors
@@ -555,11 +553,15 @@ void Write_Out(double *array, int Nx, int Ny, int Nz){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@@ -573,6 +575,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//**********************************
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -622,21 +625,24 @@ int main(int argc, char **argv)
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&iterMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&iterMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
double rlx_setA = 1.f/tau;
@@ -659,7 +665,7 @@ int main(int argc, char **argv)
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@@ -940,7 +946,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@@ -1077,7 +1083,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@@ -1088,46 +1094,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
comm.send(&sendCount_x,1,rank_X,sendtag);
comm.recv(&recvCount_X,1,rank_x,recvtag);
comm.send(&sendCount_X,1,rank_x,sendtag);
comm.recv(&recvCount_x,1,rank_X,recvtag);
comm.send(&sendCount_y,1,rank_Y,sendtag);
comm.recv(&recvCount_Y,1,rank_y,recvtag);
comm.send(&sendCount_Y,1,rank_y,sendtag);
comm.recv(&recvCount_y,1,rank_Y,recvtag);
comm.send(&sendCount_z,1,rank_Z,sendtag);
comm.recv(&recvCount_Z,1,rank_z,recvtag);
comm.send(&sendCount_Z,1,rank_z,sendtag);
comm.recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xy,1,rank_XY,sendtag);
comm.recv(&recvCount_XY,1,rank_xy,recvtag);
comm.send(&sendCount_XY,1,rank_xy,sendtag);
comm.recv(&recvCount_xy,1,rank_XY,recvtag);
comm.send(&sendCount_Xy,1,rank_xY,sendtag);
comm.recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.send(&sendCount_xY,1,rank_Xy,sendtag);
comm.recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_xz,1,rank_XZ,sendtag);
comm.recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.send(&sendCount_XZ,1,rank_xz,sendtag);
comm.recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.send(&sendCount_yz,1,rank_YZ,sendtag);
comm.recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.send(&sendCount_YZ,1,rank_yz,sendtag);
comm.recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
//**********************************************************************************
//recvCount_x = sendCount_x;
//recvCount_X = sendCount_X;
@@ -1151,7 +1157,7 @@ int main(int argc, char **argv)
//......................................................................................
// Use MPI to fill in the appropriate values
// int tag = 5;
// Mcomm.sendrecv(sendCount_x,1,rank_x,tag,sendCount_X,1);
// MPI_Sendrecv(sendCount_x,1,MPI_INT,rank_x,tag,sendCount_X,1,MPI_INT,comm,req);
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@@ -1181,48 +1187,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
@@ -1421,24 +1427,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@@ -1471,7 +1495,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
//......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@@ -1511,8 +1535,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@@ -1609,48 +1633,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@@ -1734,7 +1758,7 @@ int main(int argc, char **argv)
//*****************************************************************************
//*****************************************************************************
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
iter++;
//...................................................................
@@ -1742,8 +1766,8 @@ int main(int argc, char **argv)
//************************************************************************/
cudaThreadSynchronize();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*iter)/cputime/1000000 << " MLUPS" << endl;
@@ -1778,7 +1802,7 @@ int main(int argc, char **argv)
cudaMemcpy(velocity, vel, 3*dist_mem_size, cudaMemcpyDeviceToHost);
//..............................................................................
cudaThreadSynchronize();
comm.barrier();
MPI_Barrier(comm);
//............................................................
//....Write the z-velocity to test poiseuille flow............
double vz,vz_avg;
@@ -1807,7 +1831,7 @@ int main(int argc, char **argv)
free (velocity); free(id);
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

65
cuda/exe/lb2_Color.cu
View File

@@ -1,4 +1,6 @@
#include "common/MPI.h"
#ifdef useMPI
#include <mpi.h>
#endif
#include <stdio.h>
#include <stdlib.h>
@@ -60,10 +62,18 @@ int main(int argc, char *argv[])
{
//********** Initialize MPI ****************
int numprocs,rank;
#ifdef useMPI
MPI_Status stat;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int numprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_size(comm,&numprocs);
MPI_Comm_rank(comm,&rank);
#else
MPI_Comm comm = MPI_COMM_WORLD;
numprocs = 1;
rank = 0;
#endif
//******************************************
if (rank == 0){
@@ -113,31 +123,32 @@ int main(int argc, char *argv[])
input >> tol; // error tolerance
//.............................................................
}
#ifdef useMPI
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
#endif
double rlxA = 1.f/tau;
double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA);
@@ -232,7 +243,11 @@ int main(int argc, char *argv[])
if (k==4) k=Nz-5;
}
}
comm.bcast(&id[0],N,0);
#ifdef useMPI //............................................................
MPI_Barrier(comm);
MPI_Bcast(&id[0],N,MPI_CHAR,0,comm);
MPI_Barrier(comm);
#endif
if (rank == 0) printf("Domain set.\n");
//...........................................................................

541
cuda/exe/lb2_Color_mpi.cpp
View File

@@ -2,7 +2,7 @@
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include "common/MPI.h"
#include <mpi.h>
using namespace std;
@@ -98,11 +98,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@@ -116,6 +120,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@@ -172,30 +177,31 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
@@ -225,7 +231,7 @@ int main(int argc, char **argv)
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@@ -507,7 +513,7 @@ int main(int argc, char **argv)
PM.close();
// printf("File porosity = %f\n", double(sum)/N);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//...........................................................................
// Write the communcation structure into a file for debugging
@@ -644,7 +650,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@@ -655,46 +661,46 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
comm.Send(&sendCount_x,1,rank_X,sendtag);
comm.Recv(&recvCount_X,1,rank_x,recvtag);
comm.Send(&sendCount_X,1,rank_x,sendtag);
comm.Recv(&recvCount_x,1,rank_X,recvtag);
comm.Send(&sendCount_y,1,rank_Y,sendtag);
comm.Recv(&recvCount_Y,1,rank_y,recvtag);
comm.Send(&sendCount_Y,1,rank_y,sendtag);
comm.Recv(&recvCount_y,1,rank_Y,recvtag);
comm.Send(&sendCount_z,1,rank_Z,sendtag);
comm.Recv(&recvCount_Z,1,rank_z,recvtag);
comm.Send(&sendCount_Z,1,rank_z,sendtag);
comm.Recv(&recvCount_z,1,rank_Z,recvtag);
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xy,1,rank_XY,sendtag);
comm.Recv(&recvCount_XY,1,rank_xy,recvtag);
comm.Send(&sendCount_XY,1,rank_xy,sendtag);
comm.Recv(&recvCount_xy,1,rank_XY,recvtag);
comm.Send(&sendCount_Xy,1,rank_xY,sendtag);
comm.Recv(&recvCount_xY,1,rank_Xy,recvtag);
comm.Send(&sendCount_xY,1,rank_Xy,sendtag);
comm.Recv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_xz,1,rank_XZ,sendtag);
comm.Recv(&recvCount_XZ,1,rank_xz,recvtag);
comm.Send(&sendCount_XZ,1,rank_xz,sendtag);
comm.Recv(&recvCount_xz,1,rank_XZ,recvtag);
comm.Send(&sendCount_Xz,1,rank_xZ,sendtag);
comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag);
comm.Send(&sendCount_xZ,1,rank_Xz,sendtag);
comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
comm.Send(&sendCount_yz,1,rank_YZ,sendtag);
comm.Recv(&recvCount_YZ,1,rank_yz,recvtag);
comm.Send(&sendCount_YZ,1,rank_yz,sendtag);
comm.Recv(&recvCount_yz,1,rank_YZ,recvtag);
comm.Send(&sendCount_Yz,1,rank_yZ,sendtag);
comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag);
comm.Send(&sendCount_yZ,1,rank_Yz,sendtag);
comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.barrier();
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
//**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
@@ -725,48 +731,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@@ -987,24 +993,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@@ -1037,7 +1061,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
*/ //......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@@ -1102,49 +1126,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@@ -1184,8 +1207,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
// Old cuda timer is below
// cudaEvent_t start, stop;
// float time;
@@ -1286,42 +1309,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//*************************************************************************
@@ -1340,8 +1363,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@@ -1423,23 +1446,23 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the D3Q7 distributions
req1[0] = comm.Isend(recvbuf_x, 2*recvCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(sendbuf_X, 2*sendCount_X,rank_x,recvtag);
req1[1] = comm.Isend(recvbuf_X, 2*recvCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(sendbuf_x, 2*sendCount_x,rank_X,recvtag);
req1[2] = comm.Isend(recvbuf_y, 2*recvCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y, 2*sendCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(recvbuf_Y, 2*recvCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(sendbuf_y, 2*sendCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(recvbuf_z, 2*recvCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z, 2*sendCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(recvbuf_Z, 2*recvCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(sendbuf_z, 2*sendCount_z,rank_Z,recvtag);
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
comm.waitAll(6,req1);
comm.waitAll(6,req2);
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
//...................................................................................
//...................................................................................
dvc_UnpackDenD3Q7(faceGrid,packThreads,dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
@@ -1484,48 +1507,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@@ -1554,7 +1577,7 @@ int main(int argc, char **argv)
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
timestep++;
@@ -1564,8 +1587,8 @@ int main(int argc, char **argv)
// cudaThreadSynchronize();
dvc_Barrier();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
@@ -1593,7 +1616,7 @@ int main(int argc, char **argv)
//************************************************************************/
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}

621
cuda/exe/lb2_Color_pBC_wia_mpi.cpp
View File

@@ -2,7 +2,7 @@
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include "common/MPI.h"
#include <mpi.h>
#include "pmmc.h"
#include "Domain.h"
@@ -101,11 +101,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
int main(int argc, char **argv)
{
//*****************************************
// ***** MPI STUFF ****************
//*****************************************
// Initialize MPI
int rank,nprocs;
MPI_Init(&argc,&argv);
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
@@ -119,6 +123,7 @@ int main(int argc, char **argv)
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@@ -198,35 +203,35 @@ int main(int argc, char **argv)
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
comm.barrier();
MPI_Barrier(comm);
//.................................................
comm.bcast(&tau,1,0);
comm.bcast(&alpha,1,0);
comm.bcast(&beta,1,0);
comm.bcast(&das,1,0);
comm.bcast(&dbs,1,0);
comm.bcast(&pBC,1,0);
comm.bcast(&din,1,0);
comm.bcast(&dout,1,0);
comm.bcast(&Fx,1,0);
comm.bcast(&Fy,1,0);
comm.bcast(&Fz,1,0);
comm.bcast(&timestepMax,1,0);
comm.bcast(&interval,1,0);
comm.bcast(&tol,1,0);
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
comm.bcast(&Nz,1,0);
comm.bcast(&nBlocks,1,0);
comm.bcast(&nthreads,1,0);
comm.bcast(&nprocx,1,0);
comm.bcast(&nprocy,1,0);
comm.bcast(&nprocz,1,0);
comm.bcast(&nspheres,1,0);
comm.bcast(&Lx,1,0);
comm.bcast(&Ly,1,0);
comm.bcast(&Lz,1,0);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
comm.barrier();
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
double Ps = -(das-dbs)/(das+dbs);
@@ -258,7 +263,7 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
comm.barrier();
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@@ -556,14 +561,14 @@ int main(int argc, char **argv)
//.......................................................................
if (rank == 0) printf("Reading the sphere packing \n");
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
comm.barrier();
MPI_Barrier(comm);
// Broadcast the sphere packing to all processes
comm.bcast(cx,nspheres,0);
comm.bcast(cy,nspheres,0);
comm.bcast(cz,nspheres,0);
comm.bcast(rad,nspheres,0);
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
//...........................................................................
comm.barrier();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//.......................................................................
// sprintf(LocalRankString,"%05d",rank);
@@ -713,7 +718,7 @@ int main(int argc, char **argv)
}
}
}
comm.barrier();
MPI_Barrier(comm);
if (rank==0) printf ("SendLists are ready on host\n");
//......................................................................................
// Use MPI to fill in the recvCounts form the associated processes
@@ -724,49 +729,89 @@ int main(int argc, char **argv)
//**********************************************************************************
// Fill in the recieve counts using MPI
sendtag = recvtag = 3;
req1[0] = comm.Isend(&sendCount_x,1,rank_X,sendtag);
req2[0] = comm.Irecv(&recvCount_X,1,rank_x,recvtag);
req1[1] = comm.Isend(&sendCount_X,1,rank_x,sendtag);
req2[1] = comm.Irecv(&recvCount_x,1,rank_X,recvtag);
req1[2] = comm.Isend(&sendCount_y,1,rank_Y,sendtag);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_y,recvtag);
req1[3] = comm.Isend(&sendCount_Y,1,rank_y,sendtag);
req2[3] = comm.Irecv(&recvCount_y,1,rank_Y,recvtag);
req1[4] = comm.Isend(&sendCount_z,1,rank_Z,sendtag);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_z,recvtag);
req1[5] = comm.Isend(&sendCount_Z,1,rank_z,sendtag);
req2[5] = comm.Irecv(&recvCount_z,1,rank_Z,recvtag);
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_XY,sendtag);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_xy,recvtag);
req1[7] = comm.Isend(&sendCount_XY,1,rank_xy,sendtag);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_XY,recvtag);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_xY,sendtag);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_Xy,recvtag);
req1[9] = comm.Isend(&sendCount_xY,1,rank_Xy,sendtag);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_xY,recvtag);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_XZ,sendtag);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_xz,recvtag);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_xz,sendtag);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_XZ,recvtag);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_xZ,sendtag);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_Xz,recvtag);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_Xz,sendtag);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_xZ,recvtag);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(&sendCount_yz,1,rank_YZ,sendtag);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_yz,recvtag);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_yz,sendtag);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_YZ,recvtag);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_yZ,sendtag);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_Yz,recvtag);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_Yz,sendtag);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
//**********************************************************************************
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Barrier(comm);
*/ //**********************************************************************************
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
@@ -796,48 +841,48 @@ int main(int argc, char **argv)
// Use MPI to fill in the appropriate values for recvList
// Fill in the recieve lists using MPI
sendtag = recvtag = 4;
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
comm.waitAll(18,req1);
comm.waitAll(18,req2);
comm.barrier();
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(comm);
//......................................................................................
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
@@ -1058,24 +1103,42 @@ int main(int argc, char **argv)
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
//......................................................................................
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
@@ -1108,7 +1171,7 @@ int main(int argc, char **argv)
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
*/ //......................................................................................
if (rank==0) printf ("Devices are ready to communicate. \n");
comm.barrier();
MPI_Barrier(comm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
@@ -1259,48 +1322,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@@ -1340,8 +1403,8 @@ int main(int argc, char **argv)
//.......create and start timer............
double starttime,stoptime,cputime;
comm.barrier();
starttime = Utilities::MPI::time();
MPI_Barrier(comm);
starttime = MPI_Wtime();
//.........................................
sendtag = recvtag = 5;
@@ -1431,42 +1494,42 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the distributions
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//*************************************************************************
@@ -1484,8 +1547,8 @@ int main(int argc, char **argv)
//...................................................................................
// Wait for completion of D3Q19 communication
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
// Unpack the distributions on the device
//...................................................................................
@@ -1567,23 +1630,23 @@ int main(int argc, char **argv)
//...................................................................................
// Send all the D3Q7 distributions
req1[0] = comm.Isend(recvbuf_x,2*recvCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(sendbuf_X,2*sendCount_X,rank_x,recvtag);
req1[1] = comm.Isend(recvbuf_X,2*recvCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(sendbuf_x,2*sendCount_x,rank_X,recvtag);
req1[2] = comm.Isend(recvbuf_y,2*recvCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(sendbuf_Y,2*sendCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(recvbuf_Y,2*recvCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(sendbuf_y,2*sendCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(recvbuf_z,2*recvCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(sendbuf_Z,2*sendCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(recvbuf_Z,2*recvCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(sendbuf_z,2*sendCount_z,rank_Z,recvtag);
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
//...................................................................................
//...................................................................................
// Wait for completion of D3Q7 communication
comm.waitAll(6,req1);
comm.waitAll(6,req2);
MPI_Waitall(6,req1,stat1);
MPI_Waitall(6,req2,stat2);
//...................................................................................
//...................................................................................
dvc_UnpackDenD3Q7(faceGrid,packThreads,dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
@@ -1622,48 +1685,48 @@ int main(int argc, char **argv)
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
//...................................................................................
//...................................................................................
// Wait for completion of Indicator Field communication
//...................................................................................
comm.waitAll(18,req1);
comm.waitAll(18,req2);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
//...................................................................................
//...................................................................................
/* dvc_UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N);
@@ -1692,7 +1755,7 @@ int main(int argc, char **argv)
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
dvc_UnpackValues(faceGrid, packThreads,dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
//...................................................................................
comm.barrier();
MPI_Barrier(comm);
// Iteration completed!
timestep++;
//...................................................................
@@ -1703,7 +1766,7 @@ int main(int argc, char **argv)
//...........................................................................
dvc_Barrier();
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double));
comm.barrier();
MPI_Barrier(comm);
//...........................................................................
// Compute areas using porous medium marching cubes algorithm
// McClure, Adalsteinsson, et al. (2007)
@@ -1872,15 +1935,15 @@ int main(int argc, char **argv)
//*******************************************************************
}
//...........................................................................
comm.barrier();
nwp_volume_global = comm.sumReduce( nwp_volume );
awn_global = comm.sumReduce( awn );
ans_global = comm.sumReduce( ans );
aws_global = comm.sumReduce( aws );
lwns_global = comm.sumReduce( lwns );
As_global = comm.sumReduce( As );
MPI_Barrier(comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
comm.barrier();
MPI_Barrier(comm);
//.........................................................................
// Compute the change in the total surface energy based on the defined interval
// See McClure, Prins and Miller (2013)
@@ -1909,8 +1972,8 @@ int main(int argc, char **argv)
}
//************************************************************************/
dvc_Barrier();
comm.barrier();
stoptime = Utilities::MPI::time();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
@@ -1946,7 +2009,7 @@ int main(int argc, char **argv)
*/ //************************************************************************/
// ****************************************************
comm.barrier();
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
}