merge with morphLBM to incorporate the newest backend updates
This commit is contained in:
Rex Zhe Li
@@ -1,5 +1,4 @@
|
||||
extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
|
||||
int n;
|
||||
// conserved momemnts
|
||||
double rho,ux,uy,uz,uu;
|
||||
// non-conserved moments
|
||||
@@ -111,14 +110,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
|
||||
int n;
|
||||
// conserved momemnts
|
||||
double rho,ux,uy,uz,uu;
|
||||
// non-conserved moments
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18;
|
||||
|
||||
int nread;
|
||||
for (int n=start; n<finish; n++){
|
||||
|
||||
// q=0
|
||||
@@ -275,4 +272,4 @@ extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int star
|
||||
rlx*0.02777777777777778*(rho - 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) - 0.08333333333*(Fy-Fz);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -920,21 +920,17 @@ extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A
|
||||
double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC)
|
||||
{
|
||||
char id;
|
||||
|
||||
int idx,n,q,Cqx,Cqy,Cqz;
|
||||
// int sendLoc;
|
||||
|
||||
double f0,f1,f2,f3,f4,f5,f6;
|
||||
double na,nb,nab; // density values
|
||||
double ux,uy,uz; // flow velocity
|
||||
double nx,ny,nz,C; // color gradient components
|
||||
double a1,a2,b1,b2;
|
||||
double sp,delta;
|
||||
double delta;
|
||||
//double feq[6]; // equilibrium distributions
|
||||
// Set of Discrete velocities for the D3Q19 Model
|
||||
//int D3Q7[3][3]={{1,0,0},{0,1,0},{0,0,1}};
|
||||
|
||||
for (n=0; n<N; n++){
|
||||
for (int n=0; n<N; n++){
|
||||
id = ID[n];
|
||||
if (id != 0 ){
|
||||
|
||||
@@ -1224,25 +1220,20 @@ extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *P
|
||||
|
||||
extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N)
|
||||
{
|
||||
int n;
|
||||
double Na,Nb;
|
||||
//...................................................................
|
||||
// Update Phi
|
||||
for (n=0; n<N; n++){
|
||||
for (int n=0; n<N; n++){
|
||||
|
||||
if (ID[n] > 0 ){
|
||||
// Get the density value (Streaming already performed)
|
||||
Na = Den[n];
|
||||
Nb = Den[N+n];
|
||||
double Na = Den[n];
|
||||
double Nb = Den[N+n];
|
||||
Phi[n] = (Na-Nb)/(Na+Nb);
|
||||
}
|
||||
}
|
||||
//...................................................................
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){
|
||||
int n;
|
||||
for (n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){
|
||||
for (int n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){
|
||||
Phi[n] = value;
|
||||
}
|
||||
}
|
||||
@@ -1255,7 +1246,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, do
|
||||
double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
|
||||
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){
|
||||
|
||||
int ijk,nn,n;
|
||||
int ijk,nn;
|
||||
double fq;
|
||||
// conserved momemnts
|
||||
double rho,jx,jy,jz;
|
||||
@@ -1838,7 +1829,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
|
||||
double *Phi, double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
|
||||
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){
|
||||
|
||||
int n,nn,ijk,nread;
|
||||
int nn,ijk,nread;
|
||||
int nr1,nr2,nr3,nr4,nr5,nr6;
|
||||
int nr7,nr8,nr9,nr10;
|
||||
int nr11,nr12,nr13,nr14;
|
||||
@@ -2492,7 +2483,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
|
||||
extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double *Aq, double *Bq,
|
||||
double *Den, double *Phi, int start, int finish, int Np){
|
||||
|
||||
int idx,n,nread;
|
||||
int idx, nread;
|
||||
double fq,nA,nB;
|
||||
|
||||
for (int n=start; n<finish; n++){
|
||||
@@ -2578,11 +2569,11 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi,
|
||||
int start, int finish, int Np){
|
||||
int idx,n,nread;
|
||||
double fq,nA,nB;
|
||||
int start, int finish, int Np)
|
||||
{
|
||||
for (int n=start; n<finish; n++){
|
||||
|
||||
double fq,nA,nB;
|
||||
|
||||
// compute number density for component A
|
||||
// q=0
|
||||
fq = Aq[n];
|
||||
@@ -2646,27 +2637,25 @@ extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq,
|
||||
Den[Np+n] = nB;
|
||||
|
||||
// save the phase indicator field
|
||||
idx = Map[n];
|
||||
int idx = Map[n];
|
||||
Phi[idx] = (nA-nB)/(nA+nB);
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *phi, double *ColorGrad, int start, int finish, int Np, int Nx, int Ny, int Nz){
|
||||
int idx,n,N,i,j,k,nn;
|
||||
// distributions
|
||||
double f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double nx,ny,nz;
|
||||
|
||||
for (idx=0; idx<Np; idx++){
|
||||
for (int idx=0; idx<Np; idx++){
|
||||
|
||||
// Get the 1D index based on regular data layout
|
||||
n = Map[idx];
|
||||
int n = Map[idx];
|
||||
|
||||
//.......Back out the 3D indices for node n..............
|
||||
k = n/(Nx*Ny);
|
||||
j = (n-Nx*Ny*k)/Nx;
|
||||
i = n-Nx*Ny*k-Nx*j;
|
||||
int k = n/(Nx*Ny);
|
||||
int j = (n-Nx*Ny*k)/Nx;
|
||||
int i = n-Nx*Ny*k-Nx*j;
|
||||
//........................................................................
|
||||
//........Get 1-D index for this thread....................
|
||||
// n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
|
||||
@@ -2675,7 +2664,7 @@ extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *phi, double *ColorGrad,
|
||||
//........................................................................
|
||||
//.................Read Phase Indicator Values............................
|
||||
//........................................................................
|
||||
nn = n-1; // neighbor index (get convention)
|
||||
int nn = n-1; // neighbor index (get convention)
|
||||
if (i-1<0) nn += Nx; // periodic BC along the x-boundary
|
||||
f1 = phi[nn]; // get neighbor for phi - 1
|
||||
//........................................................................
|
||||
|
||||
654
cpu/D3Q19.cpp
654
cpu/D3Q19.cpp
@@ -270,8 +270,6 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
|
||||
// odd distributions in disteven and even distributions in distodd.
|
||||
int n,N;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double din = 0.f;
|
||||
N = Nx*Ny*Nz;
|
||||
|
||||
@@ -283,26 +281,26 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
|
||||
// Read distributions from "opposite" memory convention
|
||||
//........................................................................
|
||||
//........................................................................
|
||||
f1 = distodd[n];
|
||||
f3 = distodd[N+n];
|
||||
f5 = distodd[2*N+n];
|
||||
f7 = distodd[3*N+n];
|
||||
f9 = distodd[4*N+n];
|
||||
f11 = distodd[5*N+n];
|
||||
f13 = distodd[6*N+n];
|
||||
f15 = distodd[7*N+n];
|
||||
f17 = distodd[8*N+n];
|
||||
double f1 = distodd[n];
|
||||
double f3 = distodd[N+n];
|
||||
//double f5 = distodd[2*N+n];
|
||||
double f7 = distodd[3*N+n];
|
||||
double f9 = distodd[4*N+n];
|
||||
//double f11 = distodd[5*N+n];
|
||||
double f13 = distodd[6*N+n];
|
||||
//double f15 = distodd[7*N+n];
|
||||
double f17 = distodd[8*N+n];
|
||||
//........................................................................
|
||||
f0 = disteven[n];
|
||||
f2 = disteven[N+n];
|
||||
f4 = disteven[2*N+n];
|
||||
f6 = disteven[3*N+n];
|
||||
f8 = disteven[4*N+n];
|
||||
f10 = disteven[5*N+n];
|
||||
f12 = disteven[6*N+n];
|
||||
f14 = disteven[7*N+n];
|
||||
f16 = disteven[8*N+n];
|
||||
f18 = disteven[9*N+n];
|
||||
double f0 = disteven[n];
|
||||
double f2 = disteven[N+n];
|
||||
double f4 = disteven[2*N+n];
|
||||
double f6 = disteven[3*N+n];
|
||||
double f8 = disteven[4*N+n];
|
||||
double f10 = disteven[5*N+n];
|
||||
double f12 = disteven[6*N+n];
|
||||
//double f14 = disteven[7*N+n];
|
||||
double f16 = disteven[8*N+n];
|
||||
//double f18 = disteven[9*N+n];
|
||||
//...................................................
|
||||
|
||||
// Determine the outlet flow velocity
|
||||
@@ -315,59 +313,58 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
|
||||
}
|
||||
|
||||
extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *d_neighborList, int *list, double *dist, double flux,
|
||||
double area, int count, int Np){
|
||||
double area, int count, int Np)
|
||||
{
|
||||
int idx, n;
|
||||
int nread;
|
||||
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double factor = 1.f/(area);
|
||||
double sum = 0.f;
|
||||
|
||||
for (idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
|
||||
f0 = dist[n];
|
||||
double f0 = dist[n];
|
||||
|
||||
nread = d_neighborList[n];
|
||||
f1 = dist[nread];
|
||||
double f1 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+2*Np];
|
||||
f3 = dist[nread];
|
||||
double f3 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+6*Np];
|
||||
f7 = dist[nread];
|
||||
double f7 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+8*Np];
|
||||
f9 = dist[nread];
|
||||
double f9 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+12*Np];
|
||||
f13 = dist[nread];
|
||||
double f13 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+16*Np];
|
||||
f17 = dist[nread];
|
||||
double f17 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+Np];
|
||||
f2 = dist[nread];
|
||||
double f2 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+3*Np];
|
||||
f4 = dist[nread];
|
||||
double f4 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+5*Np];
|
||||
f6 = dist[nread];
|
||||
double f6 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+7*Np];
|
||||
f8 = dist[nread];
|
||||
double f8 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+9*Np];
|
||||
f10 = dist[nread];
|
||||
double f10 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+11*Np];
|
||||
f12 = dist[nread];
|
||||
double f12 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+15*Np];
|
||||
f16 = dist[nread];
|
||||
double f16 = dist[nread];
|
||||
|
||||
sum += factor*(f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17));
|
||||
}
|
||||
@@ -376,30 +373,29 @@ extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *d_neighborList, int *list, d
|
||||
}
|
||||
|
||||
extern "C" double ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double flux, double area,
|
||||
int count, int Np){
|
||||
int count, int Np)
|
||||
{
|
||||
int idx, n;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double factor = 1.f/(area);
|
||||
double sum = 0.f;
|
||||
|
||||
for (idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
f0 = dist[n];
|
||||
f1 = dist[2*Np+n];
|
||||
f2 = dist[1*Np+n];
|
||||
f3 = dist[4*Np+n];
|
||||
f4 = dist[3*Np+n];
|
||||
f6 = dist[5*Np+n];
|
||||
f7 = dist[8*Np+n];
|
||||
f8 = dist[7*Np+n];
|
||||
f9 = dist[10*Np+n];
|
||||
f10 = dist[9*Np+n];
|
||||
f12 = dist[11*Np+n];
|
||||
f13 = dist[14*Np+n];
|
||||
f16 = dist[15*Np+n];
|
||||
f17 = dist[18*Np+n];
|
||||
double f0 = dist[n];
|
||||
double f1 = dist[2*Np+n];
|
||||
double f2 = dist[1*Np+n];
|
||||
double f3 = dist[4*Np+n];
|
||||
double f4 = dist[3*Np+n];
|
||||
double f6 = dist[5*Np+n];
|
||||
double f7 = dist[8*Np+n];
|
||||
double f8 = dist[7*Np+n];
|
||||
double f9 = dist[10*Np+n];
|
||||
double f10 = dist[9*Np+n];
|
||||
double f12 = dist[11*Np+n];
|
||||
double f13 = dist[14*Np+n];
|
||||
double f16 = dist[15*Np+n];
|
||||
double f17 = dist[18*Np+n];
|
||||
sum += factor*(f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17));
|
||||
}
|
||||
return sum;
|
||||
@@ -412,8 +408,6 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub
|
||||
// odd distributions in disteven and even distributions in distodd.
|
||||
int n,N;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double dout = 0.f;
|
||||
|
||||
N = Nx*Ny*Nz;
|
||||
@@ -426,26 +420,26 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub
|
||||
//........................................................................
|
||||
// Read distributions from "opposite" memory convention
|
||||
//........................................................................
|
||||
f1 = distodd[n];
|
||||
f3 = distodd[N+n];
|
||||
f5 = distodd[2*N+n];
|
||||
f7 = distodd[3*N+n];
|
||||
f9 = distodd[4*N+n];
|
||||
f11 = distodd[5*N+n];
|
||||
f13 = distodd[6*N+n];
|
||||
f15 = distodd[7*N+n];
|
||||
f17 = distodd[8*N+n];
|
||||
double f1 = distodd[n];
|
||||
double f3 = distodd[N+n];
|
||||
double f5 = distodd[2*N+n];
|
||||
double f7 = distodd[3*N+n];
|
||||
double f9 = distodd[4*N+n];
|
||||
double f11 = distodd[5*N+n];
|
||||
//double f13 = distodd[6*N+n];
|
||||
double f15 = distodd[7*N+n];
|
||||
//double f17 = distodd[8*N+n];
|
||||
//........................................................................
|
||||
f0 = disteven[n];
|
||||
f2 = disteven[N+n];
|
||||
f4 = disteven[2*N+n];
|
||||
f6 = disteven[3*N+n];
|
||||
f8 = disteven[4*N+n];
|
||||
f10 = disteven[5*N+n];
|
||||
f12 = disteven[6*N+n];
|
||||
f14 = disteven[7*N+n];
|
||||
f16 = disteven[8*N+n];
|
||||
f18 = disteven[9*N+n];
|
||||
double f0 = disteven[n];
|
||||
double f2 = disteven[N+n];
|
||||
double f4 = disteven[2*N+n];
|
||||
//double f6 = disteven[3*N+n];
|
||||
double f8 = disteven[4*N+n];
|
||||
double f10 = disteven[5*N+n];
|
||||
//double f12 = disteven[6*N+n];
|
||||
double f14 = disteven[7*N+n];
|
||||
//double f16 = disteven[8*N+n];
|
||||
double f18 = disteven[9*N+n];
|
||||
|
||||
sum += (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f5+f11+f14+f15+f18));
|
||||
|
||||
@@ -456,28 +450,25 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, double din, int count, int Np)
|
||||
{
|
||||
int idx, n;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double ux,uy,uz,Cyz,Cxz;
|
||||
ux = uy = 0.0;
|
||||
for (int idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
f0 = dist[n];
|
||||
f1 = dist[2*Np+n];
|
||||
f2 = dist[1*Np+n];
|
||||
f3 = dist[4*Np+n];
|
||||
f4 = dist[3*Np+n];
|
||||
f6 = dist[5*Np+n];
|
||||
f7 = dist[8*Np+n];
|
||||
f8 = dist[7*Np+n];
|
||||
f9 = dist[10*Np+n];
|
||||
f10 = dist[9*Np+n];
|
||||
f12 = dist[11*Np+n];
|
||||
f13 = dist[14*Np+n];
|
||||
f16 = dist[15*Np+n];
|
||||
f17 = dist[18*Np+n];
|
||||
int n = list[idx];
|
||||
double f0 = dist[n];
|
||||
double f1 = dist[2*Np+n];
|
||||
double f2 = dist[1*Np+n];
|
||||
double f3 = dist[4*Np+n];
|
||||
double f4 = dist[3*Np+n];
|
||||
double f6 = dist[5*Np+n];
|
||||
double f7 = dist[8*Np+n];
|
||||
double f8 = dist[7*Np+n];
|
||||
double f9 = dist[10*Np+n];
|
||||
double f10 = dist[9*Np+n];
|
||||
double f12 = dist[11*Np+n];
|
||||
double f13 = dist[14*Np+n];
|
||||
double f16 = dist[15*Np+n];
|
||||
double f17 = dist[18*Np+n];
|
||||
//...................................................
|
||||
// Determine the inlet flow velocity
|
||||
//ux = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14);
|
||||
@@ -487,11 +478,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, doubl
|
||||
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
|
||||
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
|
||||
|
||||
f5 = f6 + 0.33333333333333338*uz;
|
||||
f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
|
||||
f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
|
||||
f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
|
||||
f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
|
||||
double f5 = f6 + 0.33333333333333338*uz;
|
||||
double f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
|
||||
double f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
|
||||
double f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
|
||||
double f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
|
||||
|
||||
dist[6*Np+n] = f5;
|
||||
dist[12*Np+n] = f11;
|
||||
@@ -503,31 +494,28 @@ extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, doubl
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, double dout, int count, int Np)
|
||||
{
|
||||
int idx, n;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double ux,uy,uz,Cyz,Cxz;
|
||||
ux = uy = 0.0;
|
||||
for (int idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
int n = list[idx];
|
||||
//........................................................................
|
||||
// Read distributions
|
||||
//........................................................................
|
||||
f0 = dist[n];
|
||||
f1 = dist[2*Np+n];
|
||||
f2 = dist[1*Np+n];
|
||||
f3 = dist[4*Np+n];
|
||||
f4 = dist[3*Np+n];
|
||||
f5 = dist[6*Np+n];
|
||||
f7 = dist[8*Np+n];
|
||||
f8 = dist[7*Np+n];
|
||||
f9 = dist[10*Np+n];
|
||||
f10 = dist[9*Np+n];
|
||||
f11 = dist[12*Np+n];
|
||||
f14 = dist[13*Np+n];
|
||||
f15 = dist[16*Np+n];
|
||||
f18 = dist[17*Np+n];
|
||||
double f0 = dist[n];
|
||||
double f1 = dist[2*Np+n];
|
||||
double f2 = dist[1*Np+n];
|
||||
double f3 = dist[4*Np+n];
|
||||
double f4 = dist[3*Np+n];
|
||||
double f5 = dist[6*Np+n];
|
||||
double f7 = dist[8*Np+n];
|
||||
double f8 = dist[7*Np+n];
|
||||
double f9 = dist[10*Np+n];
|
||||
double f10 = dist[9*Np+n];
|
||||
double f11 = dist[12*Np+n];
|
||||
double f14 = dist[13*Np+n];
|
||||
double f15 = dist[16*Np+n];
|
||||
double f18 = dist[17*Np+n];
|
||||
|
||||
// Determine the outlet flow velocity
|
||||
//ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
|
||||
@@ -537,11 +525,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, doubl
|
||||
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
|
||||
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
|
||||
|
||||
f6 = f5 - 0.33333333333333338*uz;
|
||||
f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
|
||||
f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
|
||||
f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
|
||||
f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
|
||||
double f6 = f5 - 0.33333333333333338*uz;
|
||||
double f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
|
||||
double f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
|
||||
double f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
|
||||
double f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
|
||||
|
||||
dist[5*Np+n] = f6;
|
||||
dist[11*Np+n] = f12;
|
||||
@@ -554,57 +542,54 @@ extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, doubl
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np)
|
||||
{
|
||||
int idx, n;
|
||||
int nread;
|
||||
int nr5,nr11,nr14,nr15,nr18;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double ux,uy,uz,Cyz,Cxz;
|
||||
ux = uy = 0.0;
|
||||
|
||||
for (int idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
f0 = dist[n];
|
||||
int n = list[idx];
|
||||
double f0 = dist[n];
|
||||
|
||||
nread = d_neighborList[n];
|
||||
f1 = dist[nread];
|
||||
double f1 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+2*Np];
|
||||
f3 = dist[nread];
|
||||
double f3 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+6*Np];
|
||||
f7 = dist[nread];
|
||||
double f7 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+8*Np];
|
||||
f9 = dist[nread];
|
||||
double f9 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+12*Np];
|
||||
f13 = dist[nread];
|
||||
double f13 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+16*Np];
|
||||
f17 = dist[nread];
|
||||
double f17 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+Np];
|
||||
f2 = dist[nread];
|
||||
double f2 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+3*Np];
|
||||
f4 = dist[nread];
|
||||
double f4 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+5*Np];
|
||||
f6 = dist[nread];
|
||||
double f6 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+7*Np];
|
||||
f8 = dist[nread];
|
||||
double f8 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+9*Np];
|
||||
f10 = dist[nread];
|
||||
double f10 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+11*Np];
|
||||
f12 = dist[nread];
|
||||
double f12 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+15*Np];
|
||||
f16 = dist[nread];
|
||||
double f16 = dist[nread];
|
||||
|
||||
// Unknown distributions
|
||||
nr5 = d_neighborList[n+4*Np];
|
||||
@@ -622,11 +607,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list,
|
||||
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
|
||||
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
|
||||
|
||||
f5 = f6 + 0.33333333333333338*uz;
|
||||
f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
|
||||
f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
|
||||
f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
|
||||
f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
|
||||
double f5 = f6 + 0.33333333333333338*uz;
|
||||
double f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
|
||||
double f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
|
||||
double f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
|
||||
double f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
|
||||
|
||||
dist[nr5] = f5;
|
||||
dist[nr11] = f11;
|
||||
@@ -638,60 +623,58 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list,
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list, double *dist, double dout, int count, int Np)
|
||||
{
|
||||
int idx,n,nread;
|
||||
int nread;
|
||||
int nr6,nr12,nr13,nr16,nr17;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double ux,uy,uz,Cyz,Cxz;
|
||||
ux = uy = 0.0;
|
||||
|
||||
for (int idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
int n = list[idx];
|
||||
//........................................................................
|
||||
// Read distributions
|
||||
//........................................................................
|
||||
f0 = dist[n];
|
||||
double f0 = dist[n];
|
||||
|
||||
nread = d_neighborList[n];
|
||||
f1 = dist[nread];
|
||||
double f1 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+2*Np];
|
||||
f3 = dist[nread];
|
||||
double f3 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+4*Np];
|
||||
f5 = dist[nread];
|
||||
double f5 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+6*Np];
|
||||
f7 = dist[nread];
|
||||
double f7 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+8*Np];
|
||||
f9 = dist[nread];
|
||||
double f9 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+10*Np];
|
||||
f11 = dist[nread];
|
||||
double f11 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+14*Np];
|
||||
f15 = dist[nread];
|
||||
double f15 = dist[nread];
|
||||
|
||||
|
||||
nread = d_neighborList[n+Np];
|
||||
f2 = dist[nread];
|
||||
double f2 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+3*Np];
|
||||
f4 = dist[nread];
|
||||
double f4 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+7*Np];
|
||||
f8 = dist[nread];
|
||||
double f8 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+9*Np];
|
||||
f10 = dist[nread];
|
||||
double f10 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+13*Np];
|
||||
f14 = dist[nread];
|
||||
double f14 = dist[nread];
|
||||
|
||||
nread = d_neighborList[n+17*Np];
|
||||
f18 = dist[nread];
|
||||
double f18 = dist[nread];
|
||||
|
||||
// unknown distributions
|
||||
nr6 = d_neighborList[n+5*Np];
|
||||
@@ -708,11 +691,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list,
|
||||
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
|
||||
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
|
||||
|
||||
f6 = f5 - 0.33333333333333338*uz;
|
||||
f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
|
||||
f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
|
||||
f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
|
||||
f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
|
||||
double f6 = f5 - 0.33333333333333338*uz;
|
||||
double f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
|
||||
double f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
|
||||
double f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
|
||||
double f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
|
||||
|
||||
//........Store in "opposite" memory location..........
|
||||
dist[nr6] = f6;
|
||||
@@ -724,145 +707,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list,
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_Pressure_BC_z(int *list, double *dist, double din, int count, int Np)
|
||||
{
|
||||
int n;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double ux,uy,uz;
|
||||
double Cxz,Cyz;
|
||||
|
||||
for (int idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
//........................................................................
|
||||
// Read distributions from "opposite" memory convention
|
||||
//........................................................................
|
||||
//........................................................................
|
||||
f0 = dist[n];
|
||||
f1 = dist[Np+n];
|
||||
f2 = dist[2*Np+n];
|
||||
f3 = dist[3*Np+n];
|
||||
f4 = dist[4*Np+n];
|
||||
f6 = dist[6*Np+n];
|
||||
f7 = dist[7*Np+n];
|
||||
f8 = dist[8*Np+n];
|
||||
f9 = dist[9*Np+n];
|
||||
f10 = dist[10*Np+n];
|
||||
f12 = dist[12*Np+n];
|
||||
f13 = dist[13*Np+n];
|
||||
f16 = dist[16*Np+n];
|
||||
f17 = dist[17*Np+n];
|
||||
//...................................................
|
||||
//........Determine the inlet flow velocity.........
|
||||
// uz = -1 + (f0+f3+f4+f1+f2+f7+f8+f10+f9
|
||||
// + 2*(f5+f15+f18+f11+f14))/din;
|
||||
//........Set the unknown distributions..............
|
||||
// f6 = f5 - 0.3333333333333333*din*uz;
|
||||
// f16 = f15 - 0.1666666666666667*din*uz;
|
||||
// f17 = f16 - f3 + f4-f15+f18-f7+f8-f10+f9;
|
||||
// f12= 0.5*(-din*uz+f5+f15+f18+f11+f14-f6-f16-
|
||||
// f17+f1-f2-f14+f11+f7-f8-f10+f9);
|
||||
// f13= -din*uz+f5+f15+f18+f11+f14-f6-f16-f17-f12;
|
||||
// Determine the inlet flow velocity
|
||||
ux = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14);
|
||||
uy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18);
|
||||
uz = din - (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17));
|
||||
|
||||
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
|
||||
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
|
||||
|
||||
f5 = f6 + 0.33333333333333338*uz;
|
||||
f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz;
|
||||
f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz;
|
||||
f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz;
|
||||
f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz;
|
||||
//........Store in "opposite" memory location..........
|
||||
dist[5*Np+n] = f5;
|
||||
dist[11*Np+n] = f11;
|
||||
dist[14*Np+n] = f14;
|
||||
dist[15*Np+n] = f15;
|
||||
dist[18*Np+n] = f18;
|
||||
|
||||
/*
|
||||
printf("Site=%i\n",n);
|
||||
printf("ux=%f, uy=%f, uz=%f\n",ux,uy,uz);
|
||||
printf("Cxz=%f, Cyz=%f\n",Cxz,Cyz);
|
||||
n = N;
|
||||
*/
|
||||
//...................................................
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(int *list, double *dist, double dout, int count, int Np)
|
||||
{
|
||||
int n;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
double ux,uy,uz;
|
||||
double Cxz,Cyz;
|
||||
|
||||
for (int idx=0; idx<count; idx++){
|
||||
n = list[idx];
|
||||
|
||||
//........................................................................
|
||||
// Read distributions
|
||||
//........................................................................
|
||||
f0 = dist[n];
|
||||
f1 = dist[Np+n];
|
||||
f2 = dist[2*Np+n];
|
||||
f3 = dist[3*Np+n];
|
||||
f4 = dist[4*Np+n];
|
||||
f5 = dist[5*Np+n];
|
||||
f7 = dist[7*Np+n];
|
||||
f8 = dist[8*Np+n];
|
||||
f9 = dist[9*Np+n];
|
||||
f10 = dist[10*Np+n];
|
||||
f11 = dist[11*Np+n];
|
||||
f14 = dist[14*Np+n];
|
||||
f15 = dist[15*Np+n];
|
||||
f18 = dist[18*Np+n];
|
||||
//........Determine the outlet flow velocity.........
|
||||
// uz = 1 - (f0+f3+f4+f1+f2+f7+f8+f10+f9+
|
||||
// 2*(f6+f16+f17+f12+f13))/dout;
|
||||
//...................................................
|
||||
//........Set the Unknown Distributions..............
|
||||
// f5 = f6 + 0.33333333333333338*dout*uz;
|
||||
// f15 = f16 + 0.16666666666666678*dout*uz;
|
||||
// f18 = f15+f3-f4-f16+f17+f7-f8+f10-f9;
|
||||
// f11= 0.5*(dout*uz+f6+ f16+f17+f12+f13-f5
|
||||
// -f15-f18-f1+f2-f13+f12-f7+f8+f10-f9);
|
||||
// f14= dout*uz+f6+ f16+f17+f12+f13-f5-f15-f18-f11;
|
||||
// Determine the outlet flow velocity
|
||||
//ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
|
||||
//uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
|
||||
//uz = -1.0 + (f0+f4+f3+f2+f1+f8+f7+f9+f10 + 2*(f6+f16+f17+f12+f13))/dout;
|
||||
|
||||
// Determine the inlet flow velocity
|
||||
ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
|
||||
uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
|
||||
uz = -dout + (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f5+f11+f14+f15+f18));
|
||||
|
||||
Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux;
|
||||
Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy;
|
||||
|
||||
f6 = f5 - 0.33333333333333338*uz;
|
||||
f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz;
|
||||
f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz;
|
||||
f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz;
|
||||
f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz;
|
||||
|
||||
//........Store in "opposite" memory location..........
|
||||
dist[6*Np+n] = f6;
|
||||
dist[12*Np+n] = f12;
|
||||
dist[13*Np+n] = f13;
|
||||
dist[16*Np+n] = f16;
|
||||
dist[17*Np+n] = f17;
|
||||
//...................................................
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz,
|
||||
int Nx, int Ny, int Nz)
|
||||
{
|
||||
@@ -1022,35 +866,30 @@ extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np)
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *Pressure, int N)
|
||||
{
|
||||
int n;
|
||||
// distributions
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
|
||||
for (n=0; n<N; n++){
|
||||
for (int n=0; n<N; n++){
|
||||
//........................................................................
|
||||
// Registers to store the distributions
|
||||
//........................................................................
|
||||
f0 = dist[n];
|
||||
f2 = dist[2*N+n];
|
||||
f4 = dist[4*N+n];
|
||||
f6 = dist[6*N+n];
|
||||
f8 = dist[8*N+n];
|
||||
f10 = dist[10*N+n];
|
||||
f12 = dist[12*N+n];
|
||||
f14 = dist[14*N+n];
|
||||
f16 = dist[16*N+n];
|
||||
f18 = dist[18*N+n];
|
||||
double f0 = dist[n];
|
||||
double f2 = dist[2*N+n];
|
||||
double f4 = dist[4*N+n];
|
||||
double f6 = dist[6*N+n];
|
||||
double f8 = dist[8*N+n];
|
||||
double f10 = dist[10*N+n];
|
||||
double f12 = dist[12*N+n];
|
||||
double f14 = dist[14*N+n];
|
||||
double f16 = dist[16*N+n];
|
||||
double f18 = dist[18*N+n];
|
||||
//........................................................................
|
||||
f1 = dist[N+n];
|
||||
f3 = dist[3*N+n];
|
||||
f5 = dist[5*N+n];
|
||||
f7 = dist[7*N+n];
|
||||
f9 = dist[9*N+n];
|
||||
f11 = dist[11*N+n];
|
||||
f13 = dist[13*N+n];
|
||||
f15 = dist[15*N+n];
|
||||
f17 = dist[17*N+n];
|
||||
double f1 = dist[N+n];
|
||||
double f3 = dist[3*N+n];
|
||||
double f5 = dist[5*N+n];
|
||||
double f7 = dist[7*N+n];
|
||||
double f9 = dist[9*N+n];
|
||||
double f11 = dist[11*N+n];
|
||||
double f13 = dist[13*N+n];
|
||||
double f15 = dist[15*N+n];
|
||||
double f17 = dist[17*N+n];
|
||||
//.................Compute the velocity...................................
|
||||
Pressure[n] = 0.3333333333333333*(f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+
|
||||
f9+f12+f11+f14+f13+f16+f15+f18+f17);
|
||||
@@ -1058,31 +897,29 @@ extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *Pressure, int N)
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
|
||||
double Fy, double Fz){
|
||||
int n;
|
||||
double fq,fp;
|
||||
double Fy, double Fz)
|
||||
{
|
||||
// conserved momemnts
|
||||
double rho,jx,jy,jz;
|
||||
// non-conserved moments
|
||||
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
|
||||
|
||||
const double mrt_V1=0.05263157894736842;
|
||||
const double mrt_V2=0.012531328320802;
|
||||
const double mrt_V3=0.04761904761904762;
|
||||
const double mrt_V4=0.004594820384294068;
|
||||
const double mrt_V5=0.01587301587301587;
|
||||
const double mrt_V6=0.0555555555555555555555555;
|
||||
const double mrt_V7=0.02777777777777778;
|
||||
const double mrt_V8=0.08333333333333333;
|
||||
const double mrt_V9=0.003341687552213868;
|
||||
const double mrt_V10=0.003968253968253968;
|
||||
const double mrt_V11=0.01388888888888889;
|
||||
const double mrt_V12=0.04166666666666666;
|
||||
|
||||
constexpr double mrt_V1=0.05263157894736842;
|
||||
constexpr double mrt_V2=0.012531328320802;
|
||||
constexpr double mrt_V3=0.04761904761904762;
|
||||
constexpr double mrt_V4=0.004594820384294068;
|
||||
constexpr double mrt_V5=0.01587301587301587;
|
||||
constexpr double mrt_V6=0.0555555555555555555555555;
|
||||
constexpr double mrt_V7=0.02777777777777778;
|
||||
constexpr double mrt_V8=0.08333333333333333;
|
||||
constexpr double mrt_V9=0.003341687552213868;
|
||||
constexpr double mrt_V10=0.003968253968253968;
|
||||
constexpr double mrt_V11=0.01388888888888889;
|
||||
constexpr double mrt_V12=0.04166666666666666;
|
||||
|
||||
for (int n=start; n<finish; n++){
|
||||
// q=0
|
||||
fq = dist[n];
|
||||
double fq = dist[n];
|
||||
rho = fq;
|
||||
m1 = -30.0*fq;
|
||||
m2 = 12.0*fq;
|
||||
@@ -1489,31 +1326,30 @@ extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
|
||||
double Fy, double Fz){
|
||||
int n;
|
||||
double fq,fp;
|
||||
double Fy, double Fz)
|
||||
{
|
||||
// conserved momemnts
|
||||
double rho,jx,jy,jz;
|
||||
// non-conserved moments
|
||||
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
|
||||
const double mrt_V1=0.05263157894736842;
|
||||
const double mrt_V2=0.012531328320802;
|
||||
const double mrt_V3=0.04761904761904762;
|
||||
const double mrt_V4=0.004594820384294068;
|
||||
const double mrt_V5=0.01587301587301587;
|
||||
const double mrt_V6=0.0555555555555555555555555;
|
||||
const double mrt_V7=0.02777777777777778;
|
||||
const double mrt_V8=0.08333333333333333;
|
||||
const double mrt_V9=0.003341687552213868;
|
||||
const double mrt_V10=0.003968253968253968;
|
||||
const double mrt_V11=0.01388888888888889;
|
||||
const double mrt_V12=0.04166666666666666;
|
||||
constexpr double mrt_V1=0.05263157894736842;
|
||||
constexpr double mrt_V2=0.012531328320802;
|
||||
constexpr double mrt_V3=0.04761904761904762;
|
||||
constexpr double mrt_V4=0.004594820384294068;
|
||||
constexpr double mrt_V5=0.01587301587301587;
|
||||
constexpr double mrt_V6=0.0555555555555555555555555;
|
||||
constexpr double mrt_V7=0.02777777777777778;
|
||||
constexpr double mrt_V8=0.08333333333333333;
|
||||
constexpr double mrt_V9=0.003341687552213868;
|
||||
constexpr double mrt_V10=0.003968253968253968;
|
||||
constexpr double mrt_V11=0.01388888888888889;
|
||||
constexpr double mrt_V12=0.04166666666666666;
|
||||
|
||||
|
||||
int nread;
|
||||
for (int n=start; n<finish; n++){
|
||||
// q=0
|
||||
fq = dist[n];
|
||||
double fq = dist[n];
|
||||
rho = fq;
|
||||
m1 = -30.0*fq;
|
||||
m2 = 12.0*fq;
|
||||
@@ -1964,11 +1800,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int star
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *dist, int Np) {
|
||||
|
||||
int n;
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *dist, int Np)
|
||||
{
|
||||
|
||||
for (int n=0; n<Np; n++){
|
||||
|
||||
@@ -1977,27 +1810,26 @@ extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *dist, int Np) {
|
||||
// (read from opposite array due to previous swap operation)
|
||||
//........................................................................
|
||||
// even
|
||||
f2 = dist[10*Np+n];
|
||||
f4 = dist[11*Np+n];
|
||||
f6 = dist[12*Np+n];
|
||||
f8 = dist[13*Np+n];
|
||||
f10 = dist[14*Np+n];
|
||||
f12 = dist[15*Np+n];
|
||||
f14 = dist[16*Np+n];
|
||||
f16 = dist[17*Np+n];
|
||||
f18 = dist[18*Np+n];
|
||||
double f2 = dist[10*Np+n];
|
||||
double f4 = dist[11*Np+n];
|
||||
double f6 = dist[12*Np+n];
|
||||
double f8 = dist[13*Np+n];
|
||||
double f10 = dist[14*Np+n];
|
||||
double f12 = dist[15*Np+n];
|
||||
double f14 = dist[16*Np+n];
|
||||
double f16 = dist[17*Np+n];
|
||||
double f18 = dist[18*Np+n];
|
||||
|
||||
f0 = dist[n];
|
||||
// odd
|
||||
f1 = dist[Np+n];
|
||||
f3 = dist[2*Np+n];
|
||||
f5 = dist[3*Np+n];
|
||||
f7 = dist[4*Np+n];
|
||||
f9 = dist[5*Np+n];
|
||||
f11 = dist[6*Np+n];
|
||||
f13 = dist[7*Np+n];
|
||||
f15 = dist[8*Np+n];
|
||||
f17 = dist[9*Np+n];
|
||||
double f1 = dist[Np+n];
|
||||
double f3 = dist[2*Np+n];
|
||||
double f5 = dist[3*Np+n];
|
||||
double f7 = dist[4*Np+n];
|
||||
double f9 = dist[5*Np+n];
|
||||
double f11 = dist[6*Np+n];
|
||||
double f13 = dist[7*Np+n];
|
||||
double f15 = dist[8*Np+n];
|
||||
double f17 = dist[9*Np+n];
|
||||
|
||||
//........................................................................
|
||||
// WRITE THE DISTRIBUTIONS
|
||||
@@ -2027,71 +1859,69 @@ extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *dist, int Np) {
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q19_AAodd_Compact(char * ID, int *neighborList, double *dist, int Np) {
|
||||
int n;
|
||||
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
|
||||
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
|
||||
extern "C" void ScaLBL_D3Q19_AAodd_Compact(char * ID, int *neighborList, double *dist, int Np)
|
||||
{
|
||||
int nread;
|
||||
|
||||
for (int n=0; n<Np; n++){
|
||||
//........Get 1-D index for this thread....................
|
||||
|
||||
f0 = dist[n];
|
||||
//double f0 = dist[n];
|
||||
|
||||
nread = neighborList[n]; // + 0*Np
|
||||
f2 = dist[nread];
|
||||
double f2 = dist[nread];
|
||||
|
||||
nread = neighborList[n+2*Np];
|
||||
f4 = dist[nread];
|
||||
double f4 = dist[nread];
|
||||
|
||||
nread = neighborList[n+4*Np];
|
||||
f6 = dist[nread];
|
||||
double f6 = dist[nread];
|
||||
|
||||
nread = neighborList[n+6*Np];
|
||||
f8 = dist[nread];
|
||||
double f8 = dist[nread];
|
||||
|
||||
nread = neighborList[n+8*Np];
|
||||
f10 = dist[nread];
|
||||
double f10 = dist[nread];
|
||||
|
||||
nread = neighborList[n+10*Np];
|
||||
f12 = dist[nread];
|
||||
double f12 = dist[nread];
|
||||
|
||||
nread = neighborList[n+12*Np];
|
||||
f14 = dist[nread];
|
||||
double f14 = dist[nread];
|
||||
|
||||
nread = neighborList[n+14*Np];
|
||||
f16 = dist[nread];
|
||||
double f16 = dist[nread];
|
||||
|
||||
nread = neighborList[n+16*Np];
|
||||
f18 = dist[nread];
|
||||
double f18 = dist[nread];
|
||||
|
||||
|
||||
nread = neighborList[n+Np];
|
||||
f1 = dist[nread];
|
||||
double f1 = dist[nread];
|
||||
|
||||
nread = neighborList[n+3*Np];
|
||||
f3 = dist[nread];
|
||||
double f3 = dist[nread];
|
||||
|
||||
nread = neighborList[n+5*Np];
|
||||
f5 = dist[nread];
|
||||
double f5 = dist[nread];
|
||||
|
||||
nread = neighborList[n+7*Np];
|
||||
f7 = dist[nread];
|
||||
double f7 = dist[nread];
|
||||
|
||||
nread = neighborList[n+9*Np];
|
||||
f9 = dist[nread];
|
||||
double f9 = dist[nread];
|
||||
|
||||
nread = neighborList[n+11*Np];
|
||||
f11 = dist[nread];
|
||||
double f11 = dist[nread];
|
||||
|
||||
nread = neighborList[n+13*Np];
|
||||
f13 = dist[nread];
|
||||
double f13 = dist[nread];
|
||||
|
||||
nread = neighborList[n+15*Np];
|
||||
f15 = dist[nread];
|
||||
double f15 = dist[nread];
|
||||
|
||||
nread = neighborList[n+17*Np];
|
||||
f17 = dist[nread];
|
||||
double f17 = dist[nread];
|
||||
|
||||
|
||||
nread = neighborList[n];
|
||||
|
||||
33
cpu/dfh.cpp
33
cpu/dfh.cpp
@@ -23,11 +23,10 @@ extern "C" void ScaLBL_Gradient_Unpack(double weight, double Cqx, double Cqy, do
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np){
|
||||
int idx,n;
|
||||
double phi,nA,nB;
|
||||
|
||||
for (idx=start; idx<finish; idx++){
|
||||
extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np)
|
||||
{
|
||||
for (int idx=start; idx<finish; idx++){
|
||||
double phi,nA,nB;
|
||||
phi = Phi[idx];
|
||||
if (phi > 0.f){
|
||||
nA = 1.0; nB = 0.f;
|
||||
@@ -60,15 +59,13 @@ extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq
|
||||
// LBM based on density functional hydrodynamics
|
||||
extern "C" void ScaLBL_D3Q19_AAeven_DFH(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
|
||||
double *Gradient, double *SolidForce, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
|
||||
double Fx, double Fy, double Fz, int start, int finish, int Np){
|
||||
|
||||
int ijk,nn,n;
|
||||
double Fx, double Fy, double Fz, int start, int finish, int Np)
|
||||
{
|
||||
double fq;
|
||||
// conserved momemnts
|
||||
double rho,jx,jy,jz;
|
||||
// non-conserved moments
|
||||
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
|
||||
double m3,m5,m7;
|
||||
double nA,nB; // number density
|
||||
double a1,b1,a2,b2,nAB,delta;
|
||||
double C,nx,ny,nz; //color gradient magnitude and direction
|
||||
@@ -586,7 +583,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
|
||||
double *Phi, double *Gradient, double *SolidForce, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
|
||||
double Fx, double Fy, double Fz, int start, int finish, int Np){
|
||||
|
||||
int n,nn,ijk,nread;
|
||||
int nread;
|
||||
int nr1,nr2,nr3,nr4,nr5,nr6;
|
||||
int nr7,nr8,nr9,nr10;
|
||||
int nr11,nr12,nr13,nr14;
|
||||
@@ -596,7 +593,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
|
||||
double rho,jx,jy,jz;
|
||||
// non-conserved moments
|
||||
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
|
||||
double m3,m5,m7;
|
||||
double nA,nB; // number density
|
||||
double a1,b1,a2,b2,nAB,delta;
|
||||
double C,nx,ny,nz; //color gradient magnitude and direction
|
||||
@@ -1182,12 +1178,12 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *neighborList, double *Aq, double *Bq,
|
||||
double *Den, double *Phi, int start, int finish, int Np){
|
||||
|
||||
int idx,n,nread;
|
||||
double fq,nA,nB;
|
||||
double *Den, double *Phi, int start, int finish, int Np)
|
||||
{
|
||||
|
||||
for (int n=start; n<finish; n++){
|
||||
int nread;
|
||||
double fq,nA,nB;
|
||||
|
||||
//..........Compute the number density for component A............
|
||||
// q=0
|
||||
@@ -1270,11 +1266,10 @@ extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *neighborList, double *Aq, double *Bq,
|
||||
}
|
||||
|
||||
extern "C" void ScaLBL_D3Q7_AAeven_DFH(double *Aq, double *Bq, double *Den, double *Phi,
|
||||
int start, int finish, int Np){
|
||||
int idx,n,nread;
|
||||
double fq,nA,nB;
|
||||
int start, int finish, int Np)
|
||||
{
|
||||
for (int n=start; n<finish; n++){
|
||||
|
||||
double fq,nA,nB;
|
||||
// compute number density for component A
|
||||
// q=0
|
||||
fq = Aq[n];
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
#include "D3Q19.h"
|
||||
#include "D3Q7.h"
|
||||
#include "Color.h"
|
||||
#include "common/MPI_Helpers.h"
|
||||
#include "common/MPI.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@@ -36,15 +36,11 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
|
||||
//***************************************************************************************
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
//*****************************************
|
||||
// ***** MPI STUFF ****************
|
||||
//*****************************************
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
Utilities::MPI comm( MPI_COMM_WORLD );
|
||||
int rank = comm.getRank();
|
||||
int nprocs = comm.getSize();
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -58,7 +54,6 @@ int main(int argc, char **argv)
|
||||
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
|
||||
//**********************************
|
||||
MPI_Request req1[18],req2[18];
|
||||
MPI_Status stat1[18],stat2[18];
|
||||
|
||||
if (rank == 0){
|
||||
printf("********************************************************\n");
|
||||
@@ -115,31 +110,30 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
//.................................................
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
comm.bcast(&Nz,1,0);
|
||||
comm.bcast(&nBlocks,1,0);
|
||||
comm.bcast(&nthreads,1,0);
|
||||
comm.bcast(&Fx,1,0);
|
||||
comm.bcast(&Fy,1,0);
|
||||
comm.bcast(&Fz,1,0);
|
||||
comm.bcast(&tau,1,0);
|
||||
comm.bcast(&alpha,1,0);
|
||||
comm.bcast(&beta,1,0);
|
||||
comm.bcast(&das,1,0);
|
||||
comm.bcast(&dbs,1,0);
|
||||
comm.bcast(&pBC,1,0);
|
||||
comm.bcast(&din,1,0);
|
||||
comm.bcast(&dout,1,0);
|
||||
comm.bcast(×tepMax,1,0);
|
||||
comm.bcast(&interval,1,0);
|
||||
comm.bcast(&tol,1,0);
|
||||
comm.bcast(&nprocx,1,0);
|
||||
comm.bcast(&nprocy,1,0);
|
||||
comm.bcast(&nprocz,1,0);
|
||||
//.................................................
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
// **************************************************************
|
||||
// **************************************************************
|
||||
|
||||
@@ -169,7 +163,7 @@ int main(int argc, char **argv)
|
||||
|
||||
}
|
||||
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -451,7 +445,7 @@ int main(int argc, char **argv)
|
||||
PM.close();
|
||||
// printf("File porosity = %f\n", double(sum)/N);
|
||||
//...........................................................................
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//...........................................................................
|
||||
// Write the communcation structure into a file for debugging
|
||||
@@ -588,7 +582,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -599,46 +593,46 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
comm.Send(&sendCount_x,1,rank_X,sendtag);
|
||||
comm.Recv(&recvCount_X,1,rank_x,recvtag);
|
||||
comm.Send(&sendCount_X,1,rank_x,sendtag);
|
||||
comm.Recv(&recvCount_x,1,rank_X,recvtag);
|
||||
comm.Send(&sendCount_y,1,rank_Y,sendtag);
|
||||
comm.Recv(&recvCount_Y,1,rank_y,recvtag);
|
||||
comm.Send(&sendCount_Y,1,rank_y,sendtag);
|
||||
comm.Recv(&recvCount_y,1,rank_Y,recvtag);
|
||||
comm.Send(&sendCount_z,1,rank_Z,sendtag);
|
||||
comm.Recv(&recvCount_Z,1,rank_z,recvtag);
|
||||
comm.Send(&sendCount_Z,1,rank_z,sendtag);
|
||||
comm.Recv(&recvCount_z,1,rank_Z,recvtag);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
comm.Send(&sendCount_xy,1,rank_XY,sendtag);
|
||||
comm.Recv(&recvCount_XY,1,rank_xy,recvtag);
|
||||
comm.Send(&sendCount_XY,1,rank_xy,sendtag);
|
||||
comm.Recv(&recvCount_xy,1,rank_XY,recvtag);
|
||||
comm.Send(&sendCount_Xy,1,rank_xY,sendtag);
|
||||
comm.Recv(&recvCount_xY,1,rank_Xy,recvtag);
|
||||
comm.Send(&sendCount_xY,1,rank_Xy,sendtag);
|
||||
comm.Recv(&recvCount_Xy,1,rank_xY,recvtag);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
comm.Send(&sendCount_xz,1,rank_XZ,sendtag);
|
||||
comm.Recv(&recvCount_XZ,1,rank_xz,recvtag);
|
||||
comm.Send(&sendCount_XZ,1,rank_xz,sendtag);
|
||||
comm.Recv(&recvCount_xz,1,rank_XZ,recvtag);
|
||||
comm.Send(&sendCount_Xz,1,rank_xZ,sendtag);
|
||||
comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag);
|
||||
comm.Send(&sendCount_xZ,1,rank_Xz,sendtag);
|
||||
comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
comm.Send(&sendCount_yz,1,rank_YZ,sendtag);
|
||||
comm.Recv(&recvCount_YZ,1,rank_yz,recvtag);
|
||||
comm.Send(&sendCount_YZ,1,rank_yz,sendtag);
|
||||
comm.Recv(&recvCount_yz,1,rank_YZ,recvtag);
|
||||
comm.Send(&sendCount_Yz,1,rank_yZ,sendtag);
|
||||
comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag);
|
||||
comm.Send(&sendCount_yZ,1,rank_Yz,sendtag);
|
||||
comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag);
|
||||
comm.barrier();
|
||||
//**********************************************************************************
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
@@ -669,48 +663,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]);
|
||||
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag);
|
||||
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag);
|
||||
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag);
|
||||
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag);
|
||||
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag);
|
||||
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag);
|
||||
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag);
|
||||
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag);
|
||||
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag);
|
||||
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag);
|
||||
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag);
|
||||
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]);
|
||||
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag);
|
||||
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag);
|
||||
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag);
|
||||
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag);
|
||||
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag);
|
||||
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag);
|
||||
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag);
|
||||
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]);
|
||||
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag);
|
||||
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag);
|
||||
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag);
|
||||
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag);
|
||||
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag);
|
||||
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag);
|
||||
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag);
|
||||
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(comm);
|
||||
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag);
|
||||
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag);
|
||||
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag);
|
||||
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag);
|
||||
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag);
|
||||
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag);
|
||||
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag);
|
||||
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag);
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
comm.barrier();
|
||||
//......................................................................................
|
||||
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
|
||||
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
|
||||
@@ -846,42 +840,24 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag);
|
||||
comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag);
|
||||
comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag);
|
||||
comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag);
|
||||
comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag);
|
||||
comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag);
|
||||
comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag);
|
||||
comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag);
|
||||
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag);
|
||||
comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag);
|
||||
comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag);
|
||||
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag);
|
||||
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag);
|
||||
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag);
|
||||
comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag);
|
||||
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag);
|
||||
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag);
|
||||
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -914,7 +890,7 @@ int main(int argc, char **argv)
|
||||
free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz);
|
||||
*/ //......................................................................................
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -994,48 +970,49 @@ int main(int argc, char **argv)
|
||||
PackValues(sendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_X,sendtag);
|
||||
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_x,recvtag);
|
||||
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_x,sendtag);
|
||||
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_X,recvtag);
|
||||
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_Y,sendtag);
|
||||
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_y,recvtag);
|
||||
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_y,sendtag);
|
||||
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_Y,recvtag);
|
||||
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_Z,sendtag);
|
||||
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_z,recvtag);
|
||||
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_z,sendtag);
|
||||
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_Z,recvtag);
|
||||
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
|
||||
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
|
||||
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
|
||||
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
|
||||
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
|
||||
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
|
||||
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
|
||||
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
|
||||
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
|
||||
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
|
||||
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
|
||||
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
|
||||
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
|
||||
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
|
||||
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
|
||||
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
|
||||
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
|
||||
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
|
||||
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
|
||||
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
|
||||
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
|
||||
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
|
||||
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
|
||||
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
UnpackValues(recvList_x, recvCount_x,recvbuf_x, Phi, N);
|
||||
@@ -1064,8 +1041,8 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
comm.barrier();
|
||||
starttime = Utilities::MPI::time();
|
||||
//.........................................
|
||||
|
||||
sendtag = recvtag = 5;
|
||||
@@ -1158,42 +1135,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag);
|
||||
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag);
|
||||
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag);
|
||||
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag);
|
||||
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag);
|
||||
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag);
|
||||
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag);
|
||||
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag);
|
||||
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag);
|
||||
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag);
|
||||
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag);
|
||||
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag);
|
||||
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag);
|
||||
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag);
|
||||
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag);
|
||||
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag);
|
||||
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag);
|
||||
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag);
|
||||
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag);
|
||||
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag);
|
||||
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag);
|
||||
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag);
|
||||
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag);
|
||||
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag);
|
||||
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag);
|
||||
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag);
|
||||
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag);
|
||||
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag);
|
||||
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag);
|
||||
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag);
|
||||
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag);
|
||||
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag);
|
||||
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag);
|
||||
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag);
|
||||
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag);
|
||||
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1211,8 +1188,8 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Wait for completion of D3Q19 communication
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
//...................................................................................
|
||||
// Unpack the distributions on the device
|
||||
//...................................................................................
|
||||
@@ -1293,23 +1270,23 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Send all the D3Q7 distributions
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
req1[0] = comm.Isend(recvbuf_x, 2*recvCount_x,rank_X,sendtag);
|
||||
req2[0] = comm.Irecv(sendbuf_X, 2*sendCount_X,rank_x,recvtag);
|
||||
req1[1] = comm.Isend(recvbuf_X, 2*recvCount_X,rank_x,sendtag);
|
||||
req2[1] = comm.Irecv(sendbuf_x, 2*sendCount_x,rank_X,recvtag);
|
||||
req1[2] = comm.Isend(recvbuf_y, 2*recvCount_y,rank_Y,sendtag);
|
||||
req2[2] = comm.Irecv(sendbuf_Y, 2*sendCount_Y,rank_y,recvtag);
|
||||
req1[3] = comm.Isend(recvbuf_Y, 2*recvCount_Y,rank_y,sendtag);
|
||||
req2[3] = comm.Irecv(sendbuf_y, 2*sendCount_y,rank_Y,recvtag);
|
||||
req1[4] = comm.Isend(recvbuf_z, 2*recvCount_z,rank_Z,sendtag);
|
||||
req2[4] = comm.Irecv(sendbuf_Z, 2*sendCount_Z,rank_z,recvtag);
|
||||
req1[5] = comm.Isend(recvbuf_Z, 2*recvCount_Z,rank_z,sendtag);
|
||||
req2[5] = comm.Irecv(sendbuf_z, 2*sendCount_z,rank_Z,recvtag);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of D3Q7 communication
|
||||
MPI_Waitall(6,req1,stat1);
|
||||
MPI_Waitall(6,req2,stat2);
|
||||
comm.waitAll(6,req1);
|
||||
comm.waitAll(6,req2);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
UnpackDenD3Q7(sendList_x,sendCount_x,sendbuf_x,2,Den,N);
|
||||
@@ -1345,48 +1322,49 @@ int main(int argc, char **argv)
|
||||
PackValues(sendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]);
|
||||
//...................................................................................
|
||||
req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_X,sendtag);
|
||||
req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_x,recvtag);
|
||||
req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_x,sendtag);
|
||||
req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_X,recvtag);
|
||||
req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_Y,sendtag);
|
||||
req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_y,recvtag);
|
||||
req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_y,sendtag);
|
||||
req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_Y,recvtag);
|
||||
req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_Z,sendtag);
|
||||
req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_z,recvtag);
|
||||
req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_z,sendtag);
|
||||
req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_Z,recvtag);
|
||||
req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_XY,sendtag);
|
||||
req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_xy,recvtag);
|
||||
req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_xy,sendtag);
|
||||
req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_XY,recvtag);
|
||||
req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_xY,sendtag);
|
||||
req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_Xy,recvtag);
|
||||
req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_Xy,sendtag);
|
||||
req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_xY,recvtag);
|
||||
req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_XZ,sendtag);
|
||||
req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_xz,recvtag);
|
||||
req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_xz,sendtag);
|
||||
req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_XZ,recvtag);
|
||||
req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_xZ,sendtag);
|
||||
req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_Xz,recvtag);
|
||||
req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_Xz,sendtag);
|
||||
req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_xZ,recvtag);
|
||||
req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_YZ,sendtag);
|
||||
req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_yz,recvtag);
|
||||
req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_yz,sendtag);
|
||||
req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_YZ,recvtag);
|
||||
req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_yZ,sendtag);
|
||||
req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_Yz,recvtag);
|
||||
req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_Yz,sendtag);
|
||||
req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_yZ,recvtag);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
//...................................................................................
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
UnpackValues(recvList_x, recvCount_x,recvbuf_x, Phi, N);
|
||||
@@ -1409,14 +1387,14 @@ int main(int argc, char **argv)
|
||||
UnpackValues(recvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
// Iteration completed!
|
||||
timestep++;
|
||||
//...................................................................
|
||||
}
|
||||
//************************************************************************/
|
||||
|
||||
stoptime = MPI_Wtime();
|
||||
stoptime = Utilities::MPI::time();
|
||||
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
|
||||
cputime = stoptime - starttime;
|
||||
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
|
||||
@@ -1459,7 +1437,7 @@ int main(int argc, char **argv)
|
||||
fwrite(ColorGrad,8,3*N,COLORGRAD);
|
||||
fclose(COLORGRAD);
|
||||
// ****************************************************
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
#include "D3Q19.h"
|
||||
#include "D3Q7.h"
|
||||
#include "Color.h"
|
||||
#include "common/MPI_Helpers.h"
|
||||
#include "common/MPI.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@@ -62,7 +62,7 @@ inline void UnpackMeshData(int *list, int count, double *recvbuf, DoubleArray &V
|
||||
}
|
||||
}
|
||||
//***************************************************************************************
|
||||
inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
|
||||
inline void CommunicateMeshHalo(DoubleArray &MeshData, const Utilities::MPI& Communicator,
|
||||
double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
|
||||
double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy,
|
||||
double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz,
|
||||
@@ -111,42 +111,24 @@ inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
|
||||
PackMeshData(sendList_yZ, sendCount_yZ ,sendbuf_yZ, MeshData);
|
||||
PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
|
||||
recvbuf_X,recvCount_X,MPI_CHAR,rank_X,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
|
||||
recvbuf_x,recvCount_x,MPI_CHAR,rank_x,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
|
||||
recvbuf_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvbuf_y,recvCount_y,MPI_CHAR,rank_y,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
|
||||
recvbuf_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvbuf_z,recvCount_z,MPI_CHAR,rank_z,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
|
||||
recvbuf_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
|
||||
recvbuf_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvbuf_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
|
||||
recvbuf_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
|
||||
recvbuf_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvbuf_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvbuf_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvbuf_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
|
||||
recvbuf_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvbuf_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvbuf_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvbuf_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,Communicator,MPI_STATUS_IGNORE);
|
||||
comm.sendrecv(sendbuf_x,sendCount_x,rank_x,sendtag,recvbuf_X,recvCount_X,rank_X,recvtag);
|
||||
comm.sendrecv(sendbuf_X,sendCount_X,rank_X,sendtag,recvbuf_x,recvCount_x,rank_x,recvtag);
|
||||
comm.sendrecv(sendbuf_y,sendCount_y,rank_y,sendtag,recvbuf_Y,recvCount_Y,rank_Y,recvtag);
|
||||
comm.sendrecv(sendbuf_Y,sendCount_Y,rank_Y,sendtag,recvbuf_y,recvCount_y,rank_y,recvtag);
|
||||
comm.sendrecv(sendbuf_z,sendCount_z,rank_z,sendtag,recvbuf_Z,recvCount_Z,rank_Z,recvtag);
|
||||
comm.sendrecv(sendbuf_Z,sendCount_Z,rank_Z,sendtag,recvbuf_z,recvCount_z,rank_z,recvtag);
|
||||
comm.sendrecv(sendbuf_xy,sendCount_xy,rank_xy,sendtag,recvbuf_XY,recvCount_XY,rank_XY,recvtag);
|
||||
comm.sendrecv(sendbuf_XY,sendCount_XY,rank_XY,sendtag,recvbuf_xy,recvCount_xy,rank_xy,recvtag);
|
||||
comm.sendrecv(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag,recvbuf_xY,recvCount_xY,rank_xY,recvtag);
|
||||
comm.sendrecv(sendbuf_xY,sendCount_xY,rank_xY,sendtag,recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
|
||||
comm.sendrecv(sendbuf_xz,sendCount_xz,rank_xz,sendtag,recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
|
||||
comm.sendrecv(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag,recvbuf_xz,recvCount_xz,rank_xz,recvtag);
|
||||
comm.sendrecv(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag,recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
|
||||
comm.sendrecv(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag,recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
|
||||
comm.sendrecv(sendbuf_yz,sendCount_yz,rank_yz,sendtag,recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
|
||||
comm.sendrecv(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag,recvbuf_yz,recvCount_yz,rank_yz,recvtag);
|
||||
comm.sendrecv(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag,recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
|
||||
comm.sendrecv(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag,recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
|
||||
//........................................................................................
|
||||
UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData);
|
||||
UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData);
|
||||
@@ -172,15 +154,11 @@ inline void CommunicateMeshHalo(DoubleArray &MeshData, MPI_Comm Communicator,
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
//*****************************************
|
||||
// ***** MPI STUFF ****************
|
||||
//*****************************************
|
||||
// Initialize MPI
|
||||
int rank,nprocs;
|
||||
MPI_Init(&argc,&argv);
|
||||
MPI_Comm comm = MPI_COMM_WORLD;
|
||||
MPI_Comm_rank(comm,&rank);
|
||||
MPI_Comm_size(comm,&nprocs);
|
||||
Utilities::MPI comm( MPI_COMM_WORLD );
|
||||
int rank = comm.getRank();
|
||||
int nprocs = comm.getSize();
|
||||
// parallel domain size (# of sub-domains)
|
||||
int nprocx,nprocy,nprocz;
|
||||
int iproc,jproc,kproc;
|
||||
@@ -194,7 +172,6 @@ int main(int argc, char **argv)
|
||||
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
|
||||
//**********************************
|
||||
MPI_Request req1[18],req2[18];
|
||||
MPI_Status stat1[18],stat2[18];
|
||||
|
||||
if (rank == 0){
|
||||
printf("********************************************************\n");
|
||||
@@ -283,38 +260,39 @@ int main(int argc, char **argv)
|
||||
}
|
||||
// **************************************************************
|
||||
// Broadcast simulation parameters from rank 0 to all other procs
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
//.................................................
|
||||
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&xIntPos,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
|
||||
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(×tepMax,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&interval,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
|
||||
comm.bcast(&tau,1,0);
|
||||
comm.bcast(&alpha,1,0);
|
||||
comm.bcast(&beta,1,0);
|
||||
comm.bcast(&das,1,0);
|
||||
comm.bcast(&dbs,1,0);
|
||||
comm.bcast(&xIntPos,1,0);
|
||||
comm.bcast(&wp_saturation,1,0);
|
||||
comm.bcast(&pBC,1,0);
|
||||
comm.bcast(&Restart,1,0);
|
||||
comm.bcast(&din,1,0);
|
||||
comm.bcast(&dout,1,0);
|
||||
comm.bcast(&Fx,1,0);
|
||||
comm.bcast(&Fy,1,0);
|
||||
comm.bcast(&Fz,1,0);
|
||||
comm.bcast(×tepMax,1,0);
|
||||
comm.bcast(&interval,1,0);
|
||||
comm.bcast(&tol,1,0);
|
||||
|
||||
// Computational domain
|
||||
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
|
||||
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
|
||||
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
|
||||
comm.bcast(&Nz,1,0);
|
||||
// comm.bcast(&nBlocks,1,0);
|
||||
// comm.bcast(&nthreads,1,0);
|
||||
comm.bcast(&nprocx,1,0);
|
||||
comm.bcast(&nprocy,1,0);
|
||||
comm.bcast(&nprocz,1,0);
|
||||
comm.bcast(&nspheres,1,0);
|
||||
comm.bcast(&Lx,1,0);
|
||||
comm.bcast(&Ly,1,0);
|
||||
comm.bcast(&Lz,1,0);
|
||||
//.................................................
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
// **************************************************************
|
||||
// **************************************************************
|
||||
double Ps = -(das-dbs)/(das+dbs);
|
||||
@@ -346,7 +324,7 @@ int main(int argc, char **argv)
|
||||
printf("********************************************************\n");
|
||||
}
|
||||
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
kproc = rank/(nprocx*nprocy);
|
||||
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
|
||||
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
|
||||
@@ -685,14 +663,14 @@ int main(int argc, char **argv)
|
||||
//.......................................................................
|
||||
if (rank == 0) printf("Reading the sphere packing \n");
|
||||
if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad);
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
// Broadcast the sphere packing to all processes
|
||||
MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
|
||||
MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
|
||||
comm.bcast(cx,nspheres,0);
|
||||
comm.bcast(cy,nspheres,0);
|
||||
comm.bcast(cz,nspheres,0);
|
||||
comm.bcast(rad,nspheres,0);
|
||||
//...........................................................................
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
if (rank == 0) cout << "Domain set." << endl;
|
||||
//.......................................................................
|
||||
// sprintf(LocalRankString,"%05d",rank);
|
||||
@@ -725,7 +703,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
sum_local = 1.0*sum;
|
||||
MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
porosity = comm.sumReduce( sum_local );
|
||||
porosity = porosity*iVol_global;
|
||||
if (rank==0) printf("Media porosity = %f \n",porosity);
|
||||
|
||||
@@ -859,7 +837,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
if (rank==0) printf ("SendLists are ready on host\n");
|
||||
//......................................................................................
|
||||
// Use MPI to fill in the recvCounts form the associated processes
|
||||
@@ -870,89 +848,48 @@ int main(int argc, char **argv)
|
||||
//**********************************************************************************
|
||||
// Fill in the recieve counts using MPI
|
||||
sendtag = recvtag = 3;
|
||||
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag,comm,&req2[5]);
|
||||
req1[0] = comm.Isend(&sendCount_x,1,rank_x,sendtag);
|
||||
req2[0] = comm.Irecv(&recvCount_X,1,rank_X,recvtag);
|
||||
req1[1] = comm.Isend(&sendCount_X,1,rank_X,sendtag);
|
||||
req2[1] = comm.Irecv(&recvCount_x,1,rank_x,recvtag);
|
||||
req1[2] = comm.Isend(&sendCount_y,1,rank_y,sendtag);
|
||||
req2[2] = comm.Irecv(&recvCount_Y,1,rank_Y,recvtag);
|
||||
req1[3] = comm.Isend(&sendCount_Y,1,rank_Y,sendtag);
|
||||
req2[3] = comm.Irecv(&recvCount_y,1,rank_y,recvtag);
|
||||
req1[4] = comm.Isend(&sendCount_z,1,rank_z,sendtag);
|
||||
req2[4] = comm.Irecv(&recvCount_Z,1,rank_Z,recvtag);
|
||||
req1[5] = comm.Isend(&sendCount_Z,1,rank_Z,sendtag);
|
||||
req2[5] = comm.Irecv(&recvCount_z,1,rank_z,recvtag);
|
||||
|
||||
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
|
||||
req1[6] = comm.Isend(&sendCount_xy,1,rank_xy,sendtag);
|
||||
req2[6] = comm.Irecv(&recvCount_XY,1,rank_XY,recvtag);
|
||||
req1[7] = comm.Isend(&sendCount_XY,1,rank_XY,sendtag);
|
||||
req2[7] = comm.Irecv(&recvCount_xy,1,rank_xy,recvtag);
|
||||
req1[8] = comm.Isend(&sendCount_Xy,1,rank_Xy,sendtag);
|
||||
req2[8] = comm.Irecv(&recvCount_xY,1,rank_xY,recvtag);
|
||||
req1[9] = comm.Isend(&sendCount_xY,1,rank_xY,sendtag);
|
||||
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_Xy,recvtag);
|
||||
|
||||
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
|
||||
req1[10] = comm.Isend(&sendCount_xz,1,rank_xz,sendtag);
|
||||
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_XZ,recvtag);
|
||||
req1[11] = comm.Isend(&sendCount_XZ,1,rank_XZ,sendtag);
|
||||
req2[11] = comm.Irecv(&recvCount_xz,1,rank_xz,recvtag);
|
||||
req1[12] = comm.Isend(&sendCount_Xz,1,rank_Xz,sendtag);
|
||||
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_xZ,recvtag);
|
||||
req1[13] = comm.Isend(&sendCount_xZ,1,rank_xZ,sendtag);
|
||||
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_Xz,recvtag);
|
||||
|
||||
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(comm);
|
||||
/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm);
|
||||
MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm);
|
||||
MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm);
|
||||
MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm);
|
||||
MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
|
||||
MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm);
|
||||
MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm);
|
||||
MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Barrier(comm);
|
||||
*/ //**********************************************************************************
|
||||
req1[14] = comm.Isend(&sendCount_yz,1,rank_yz,sendtag);
|
||||
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_YZ,recvtag);
|
||||
req1[15] = comm.Isend(&sendCount_YZ,1,rank_YZ,sendtag);
|
||||
req2[15] = comm.Irecv(&recvCount_yz,1,rank_yz,recvtag);
|
||||
req1[16] = comm.Isend(&sendCount_Yz,1,rank_Yz,sendtag);
|
||||
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_yZ,recvtag);
|
||||
req1[17] = comm.Isend(&sendCount_yZ,1,rank_yZ,sendtag);
|
||||
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_Yz,recvtag);
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
comm.barrier();
|
||||
//......................................................................................
|
||||
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
|
||||
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
|
||||
@@ -982,48 +919,48 @@ int main(int argc, char **argv)
|
||||
// Use MPI to fill in the appropriate values for recvList
|
||||
// Fill in the recieve lists using MPI
|
||||
sendtag = recvtag = 4;
|
||||
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,comm,&req2[5]);
|
||||
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_x,sendtag);
|
||||
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_X,recvtag);
|
||||
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_X,sendtag);
|
||||
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_x,recvtag);
|
||||
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_y,sendtag);
|
||||
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_Y,recvtag);
|
||||
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_Y,sendtag);
|
||||
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_y,recvtag);
|
||||
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_z,sendtag);
|
||||
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_Z,recvtag);
|
||||
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_Z,sendtag);
|
||||
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_z,recvtag);
|
||||
|
||||
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,comm,&req2[9]);
|
||||
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_xy,sendtag);
|
||||
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_XY,recvtag);
|
||||
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_XY,sendtag);
|
||||
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_xy,recvtag);
|
||||
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy,sendtag);
|
||||
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_xY,recvtag);
|
||||
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_xY,sendtag);
|
||||
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy,recvtag);
|
||||
|
||||
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,comm,&req2[13]);
|
||||
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_xz,sendtag);
|
||||
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ,recvtag);
|
||||
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ,sendtag);
|
||||
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_xz,recvtag);
|
||||
req1[12] = comm.Isend(sendList_Xz,endCount_Xz,rank_Xz,sendtag);
|
||||
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ,recvtag);
|
||||
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ,sendtag);
|
||||
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz,recvtag);
|
||||
|
||||
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,comm,&req2[17]);
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
MPI_Barrier(comm);
|
||||
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_yz,sendtag);
|
||||
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ,recvtag);
|
||||
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ,sendtag);
|
||||
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_yz,recvtag);
|
||||
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz,sendtag);
|
||||
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ,recvtag);
|
||||
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ,sendtag);
|
||||
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz,recvtag);
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
comm.barrier();
|
||||
//......................................................................................
|
||||
for (int idx=0; idx<recvCount_x; idx++) recvList_x[idx] -= (Nx-2);
|
||||
for (int idx=0; idx<recvCount_X; idx++) recvList_X[idx] += (Nx-2);
|
||||
@@ -1138,7 +1075,7 @@ int main(int argc, char **argv)
|
||||
dvc_AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory
|
||||
dvc_AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory
|
||||
//......................................................................................
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
if (rank==0) printf ("Prepare to copy send/recv Lists to device \n");
|
||||
dvc_CopyToDevice(dvcSendList_x,sendList_x,sendCount_x*sizeof(int));
|
||||
dvc_CopyToDevice(dvcSendList_X,sendList_X,sendCount_X*sizeof(int));
|
||||
@@ -1245,42 +1182,24 @@ int main(int argc, char **argv)
|
||||
PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id);
|
||||
PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id);
|
||||
//......................................................................................
|
||||
MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag,
|
||||
recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag,
|
||||
recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag,
|
||||
recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag,
|
||||
recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag,
|
||||
recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag,
|
||||
recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag,
|
||||
recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag,
|
||||
recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag,
|
||||
recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag,
|
||||
recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag,
|
||||
recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag,
|
||||
recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag,
|
||||
recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag,
|
||||
recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag,
|
||||
recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag,
|
||||
recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag,
|
||||
recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag,
|
||||
recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
|
||||
comm.sendrecv(sendID_x,sendCount_x,rank_x,sendtag,recvID_X,recvCount_X,rank_X,recvtag);
|
||||
comm.sendrecv(sendID_X,sendCount_X,rank_X,sendtag,recvID_x,recvCount_x,rank_x,recvtag);
|
||||
comm.sendrecv(sendID_y,sendCount_y,rank_y,sendtag,recvID_Y,recvCount_Y,rank_Y,recvtag);
|
||||
comm.sendrecv(sendID_Y,sendCount_Y,rank_Y,sendtag,recvID_y,recvCount_y,rank_y,recvtag);
|
||||
comm.sendrecv(sendID_z,sendCount_z,rank_z,sendtag,recvID_Z,recvCount_Z,rank_Z,recvtag);
|
||||
comm.sendrecv(sendID_Z,sendCount_Z,rank_Z,sendtag,recvID_z,recvCount_z,rank_z,recvtag);
|
||||
comm.sendrecv(sendID_xy,sendCount_xy,rank_xy,sendtag,recvID_XY,recvCount_XY,rank_XY,recvtag);
|
||||
comm.sendrecv(sendID_XY,sendCount_XY,rank_XY,sendtag,recvID_xy,recvCount_xy,rank_xy,recvtag);
|
||||
comm.sendrecv(sendID_Xy,sendCount_Xy,rank_Xy,sendtag,recvID_xY,recvCount_xY,rank_xY,recvtag);
|
||||
comm.sendrecv(sendID_xY,sendCount_xY,rank_xY,sendtag,recvID_Xy,recvCount_Xy,rank_Xy,recvtag);
|
||||
comm.sendrecv(sendID_xz,sendCount_xz,rank_xz,sendtag,recvID_XZ,recvCount_XZ,rank_XZ,recvtag);
|
||||
comm.sendrecv(sendID_XZ,sendCount_XZ,rank_XZ,sendtag,recvID_xz,recvCount_xz,rank_xz,recvtag);
|
||||
comm.sendrecv(sendID_Xz,sendCount_Xz,rank_Xz,sendtag,recvID_xZ,recvCount_xZ,rank_xZ,recvtag);
|
||||
comm.sendrecv(sendID_xZ,sendCount_xZ,rank_xZ,sendtag,recvID_Xz,recvCount_Xz,rank_Xz,recvtag);
|
||||
comm.sendrecv(sendID_yz,sendCount_yz,rank_yz,sendtag,recvID_YZ,recvCount_YZ,rank_YZ,recvtag);
|
||||
comm.sendrecv(sendID_YZ,sendCount_YZ,rank_YZ,sendtag,recvID_yz,recvCount_yz,rank_yz,recvtag);
|
||||
comm.sendrecv(sendID_Yz,sendCount_Yz,rank_Yz,sendtag,recvID_yZ,recvCount_yZ,rank_yZ,recvtag);
|
||||
comm.sendrecv(sendID_yZ,sendCount_yZ,rank_yZ,sendtag,recvID_Yz,recvCount_Yz,rank_Yz,recvtag);
|
||||
//......................................................................................
|
||||
UnpackID(recvList_x, recvCount_x ,recvID_x, id);
|
||||
UnpackID(recvList_X, recvCount_X ,recvID_X, id);
|
||||
@@ -1348,7 +1267,7 @@ int main(int argc, char **argv)
|
||||
recvMeshData_YZ = new double [recvCount_YZ];
|
||||
recvMeshData_XZ = new double [recvCount_XZ];
|
||||
if (rank==0) printf ("Devices are ready to communicate. \n");
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
|
||||
//...........device phase ID.................................................
|
||||
if (rank==0) printf ("Copying phase ID to device \n");
|
||||
@@ -1528,8 +1447,8 @@ int main(int argc, char **argv)
|
||||
|
||||
//.......create and start timer............
|
||||
double starttime,stoptime,cputime;
|
||||
MPI_Barrier(comm);
|
||||
starttime = MPI_Wtime();
|
||||
comm.barrier();
|
||||
starttime = Utilities::MPI::time();
|
||||
//.........................................
|
||||
//...........................................................................
|
||||
// MAIN VARIABLES INITIALIZED HERE
|
||||
@@ -1600,7 +1519,7 @@ int main(int argc, char **argv)
|
||||
dvc_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
|
||||
dvc_CopyToDevice(Den,cDen,2*N*sizeof(double));
|
||||
dvc_Barrier();
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
}
|
||||
// Pack the buffers (zeros out the halo region)
|
||||
dvc_PackDenD3Q7(dvcRecvList_x,recvCount_x,recvbuf_x,2,Den,N);
|
||||
@@ -1639,48 +1558,48 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_x,sendtag);
|
||||
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag);
|
||||
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_X,sendtag);
|
||||
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag);
|
||||
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_y,sendtag);
|
||||
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag);
|
||||
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag);
|
||||
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag);
|
||||
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_z,sendtag);
|
||||
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag);
|
||||
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag);
|
||||
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag);
|
||||
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
|
||||
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
|
||||
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
|
||||
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
|
||||
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
|
||||
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
|
||||
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
|
||||
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
|
||||
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
|
||||
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
|
||||
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
|
||||
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
|
||||
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
|
||||
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
|
||||
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
|
||||
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
|
||||
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
|
||||
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
|
||||
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
|
||||
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
|
||||
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
|
||||
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
|
||||
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
|
||||
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
//...................................................................................
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
dvc_Barrier();
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
@@ -1726,7 +1645,7 @@ int main(int argc, char **argv)
|
||||
dvc_CopyToHost(Phase.data,Phi,N*sizeof(double));
|
||||
dvc_CopyToHost(Press.data,Pressure,N*sizeof(double));
|
||||
dvc_CopyToHost(Vel,Velocity,3*N*sizeof(double));
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
//...........................................................................
|
||||
|
||||
timestep=0;
|
||||
@@ -1822,42 +1741,42 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the distributions
|
||||
MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_x,sendtag);
|
||||
req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_X,recvtag);
|
||||
req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_X,sendtag);
|
||||
req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_x,recvtag);
|
||||
req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_y,sendtag);
|
||||
req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_Y,recvtag);
|
||||
req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_Y,sendtag);
|
||||
req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_y,recvtag);
|
||||
req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_z,sendtag);
|
||||
req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_Z,recvtag);
|
||||
req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_Z,sendtag);
|
||||
req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_z,recvtag);
|
||||
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
|
||||
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
|
||||
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
|
||||
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
|
||||
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
|
||||
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
|
||||
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
|
||||
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
|
||||
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
|
||||
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
|
||||
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
|
||||
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
|
||||
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
|
||||
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
|
||||
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
|
||||
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
|
||||
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
|
||||
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
|
||||
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
|
||||
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
|
||||
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
|
||||
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
|
||||
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
|
||||
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
|
||||
//...................................................................................
|
||||
|
||||
//*************************************************************************
|
||||
@@ -1874,8 +1793,8 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Wait for completion of D3Q19 communication
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
|
||||
//...................................................................................
|
||||
// Unpack the distributions on the device
|
||||
@@ -1958,23 +1877,23 @@ int main(int argc, char **argv)
|
||||
|
||||
//...................................................................................
|
||||
// Send all the D3Q7 distributions
|
||||
MPI_Isend(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
req1[0] = comm.Isend(recvbuf_x,2*recvCount_x,rank_x,sendtag);
|
||||
req2[0] = comm.Irecv(sendbuf_X,2*sendCount_X,rank_X,recvtag);
|
||||
req1[1] = comm.Isend(recvbuf_X,2*recvCount_X,rank_X,sendtag);
|
||||
req2[1] = comm.Irecv(sendbuf_x,2*sendCount_x,rank_x,recvtag);
|
||||
req1[2] = comm.Isend(recvbuf_y,2*recvCount_y,rank_y,sendtag);
|
||||
req2[2] = comm.Irecv(sendbuf_Y,2*sendCount_Y,rank_Y,recvtag);
|
||||
req1[3] = comm.Isend(recvbuf_Y,2*recvCount_Y,rank_Y,sendtag);
|
||||
req2[3] = comm.Irecv(sendbuf_y,2*sendCount_y,rank_y,recvtag);
|
||||
req1[4] = comm.Isend(recvbuf_z,2*recvCount_z,rank_z,sendtag);
|
||||
req2[4] = comm.Irecv(sendbuf_Z,2*sendCount_Z,rank_Z,recvtag);
|
||||
req1[5] = comm.Isend(recvbuf_Z,2*recvCount_Z,rank_Z,sendtag);
|
||||
req2[5] = comm.Irecv(sendbuf_z,2*sendCount_z,rank_z,recvtag);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of D3Q7 communication
|
||||
MPI_Waitall(6,req1,stat1);
|
||||
MPI_Waitall(6,req2,stat2);
|
||||
comm.waitAll(6,req1);
|
||||
comm.waitAll(6,req2);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
dvc_UnpackDenD3Q7(dvcSendList_x,sendCount_x,sendbuf_x,2,Den,N);
|
||||
@@ -2013,48 +1932,48 @@ int main(int argc, char **argv)
|
||||
//...................................................................................
|
||||
// Send / Recv all the phase indcator field values
|
||||
//...................................................................................
|
||||
MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]);
|
||||
MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]);
|
||||
MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]);
|
||||
MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]);
|
||||
MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]);
|
||||
MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]);
|
||||
MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]);
|
||||
MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]);
|
||||
MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]);
|
||||
MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]);
|
||||
MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]);
|
||||
MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]);
|
||||
MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]);
|
||||
MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]);
|
||||
MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]);
|
||||
MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]);
|
||||
MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]);
|
||||
MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]);
|
||||
MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]);
|
||||
MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]);
|
||||
MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]);
|
||||
MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]);
|
||||
MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]);
|
||||
MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]);
|
||||
MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]);
|
||||
MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]);
|
||||
MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]);
|
||||
MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]);
|
||||
MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]);
|
||||
MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]);
|
||||
MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]);
|
||||
MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]);
|
||||
MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]);
|
||||
MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]);
|
||||
MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]);
|
||||
MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]);
|
||||
req1[0] = comm.Isend(sendbuf_x,sendCount_x,rank_x,sendtag);
|
||||
req2[0] = comm.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag);
|
||||
req1[1] = comm.Isend(sendbuf_X,sendCount_X,rank_X,sendtag);
|
||||
req2[1] = comm.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag);
|
||||
req1[2] = comm.Isend(sendbuf_y,sendCount_y,rank_y,sendtag);
|
||||
req2[2] = comm.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag);
|
||||
req1[3] = comm.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag);
|
||||
req2[3] = comm.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag);
|
||||
req1[4] = comm.Isend(sendbuf_z,sendCount_z,rank_z,sendtag);
|
||||
req2[4] = comm.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag);
|
||||
req1[5] = comm.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag);
|
||||
req2[5] = comm.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag);
|
||||
req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag);
|
||||
req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag);
|
||||
req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag);
|
||||
req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag);
|
||||
req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag);
|
||||
req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag);
|
||||
req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag);
|
||||
req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
|
||||
req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag);
|
||||
req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
|
||||
req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag);
|
||||
req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag);
|
||||
req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag);
|
||||
req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
|
||||
req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag);
|
||||
req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
|
||||
req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag);
|
||||
req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
|
||||
req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag);
|
||||
req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag);
|
||||
req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag);
|
||||
req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
|
||||
req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag);
|
||||
req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
// Wait for completion of Indicator Field communication
|
||||
//...................................................................................
|
||||
MPI_Waitall(18,req1,stat1);
|
||||
MPI_Waitall(18,req2,stat2);
|
||||
comm.waitAll(18,req1);
|
||||
comm.waitAll(18,req2);
|
||||
dvc_Barrier();
|
||||
//...................................................................................
|
||||
//...................................................................................
|
||||
@@ -2084,7 +2003,7 @@ int main(int argc, char **argv)
|
||||
dvc_UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N);
|
||||
dvc_UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N);
|
||||
//...................................................................................
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
|
||||
// Iteration completed!
|
||||
timestep++;
|
||||
@@ -2364,27 +2283,27 @@ int main(int argc, char **argv)
|
||||
//...........................................................................
|
||||
}
|
||||
//...........................................................................
|
||||
MPI_Barrier(comm);
|
||||
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
comm.barrier();
|
||||
nwp_volume_global = comm.sumReduce( nwp_volume );
|
||||
awn_global = comm.sumReduce( awn );
|
||||
ans_global = comm.sumReduce( ans );
|
||||
aws_global = comm.sumReduce( aws );
|
||||
lwns_global = comm.sumReduce( lwns );
|
||||
As_global = comm.sumReduce( As );
|
||||
Jwn_global = comm.sumReduce( Jwn );
|
||||
efawns_global = comm.sumReduce( efawns );
|
||||
// Phase averages
|
||||
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm);
|
||||
MPI_Barrier(comm);
|
||||
vol_w_global = comm.sumReduce( vol_w );
|
||||
vol_n_global = comm.sumReduce( vol_n );
|
||||
paw_global = comm.sumReduce( paw );
|
||||
pan_global = comm.sumReduce( pan );
|
||||
vaw_global(0) = comm.sumReduce( vaw(0) );
|
||||
van_global(0) = comm.sumReduce( van(0) );
|
||||
vawn_global(0) = comm.sumReduce( vawn(0) );
|
||||
Gwn_global(0) = comm.sumReduce( Gwn(0) );
|
||||
Gns_global(0) = comm.sumReduce( Gns(0) );
|
||||
Gws_global(0) = comm.sumReduce( Gws(0) );
|
||||
comm.barrier();
|
||||
//.........................................................................
|
||||
// Compute the change in the total surface energy based on the defined interval
|
||||
// See McClure, Prins and Miller (2013)
|
||||
@@ -2451,8 +2370,8 @@ int main(int argc, char **argv)
|
||||
}
|
||||
//************************************************************************/
|
||||
dvc_Barrier();
|
||||
MPI_Barrier(comm);
|
||||
stoptime = MPI_Wtime();
|
||||
comm.barrier();
|
||||
stoptime = Utilities::MPI::time();
|
||||
if (rank==0) printf("-------------------------------------------------------------------\n");
|
||||
// Compute the walltime per timestep
|
||||
cputime = (stoptime - starttime)/timestep;
|
||||
@@ -2489,7 +2408,7 @@ int main(int argc, char **argv)
|
||||
*/ //************************************************************************/
|
||||
|
||||
// ****************************************************
|
||||
MPI_Barrier(comm);
|
||||
comm.barrier();
|
||||
MPI_Finalize();
|
||||
// ****************************************************
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user