#ifndef Domain_INC #define Domain_INC // Created by James McClure // Copyright 2008-2013 #include #include #include #include #include #include #include // std::exception #include #include "common/Array.h" #include "common/Utilities.h" #include "common/MPI_Helpers.h" #include "common/Communication.h" using namespace std; struct Domain{ // Default constructor Domain(int nx, int ny, int nz, int rnk, int npx, int npy, int npz, double lx, double ly, double lz, int BC); // Destructor ~Domain(); // Basic domain information int Nx,Ny,Nz,N; int iproc,jproc,kproc; int nprocx,nprocy,nprocz; double Lx,Ly,Lz,Volume; int rank; int BoundaryCondition; RankInfoStruct rank_info; MPI_Group Group; // Group of processors associated with this domain MPI_Comm Comm; // MPI Communicator for this domain //********************************** // MPI ranks for all 18 neighbors //********************************** int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z; int rank_xy,rank_XY,rank_xY,rank_Xy; int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** //...................................................................................... // Get the actual D3Q19 communication counts (based on location of solid phase) // Discrete velocity set symmetry implies the sendcount = recvcount //...................................................................................... int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z; int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ; int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ; //...................................................................................... int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z; int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ; int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ; //...................................................................................... int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z; int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ; int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ; //...................................................................................... int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z; int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ; int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ; //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ; //...................................................................................... int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z; int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ; int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ; //...................................................................................... double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z; double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ; double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ; double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z; double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ; double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ; // Solid indicator function char *id; // Blob information IntArray BlobLabel; IntArray BlobGraph; void InitializeRanks(); void CommInit(MPI_Comm comm); void CommunicateMeshHalo(DoubleArray &Mesh); void BlobComm(MPI_Comm comm); void AssignBlobConnections(); private: inline int getRankForBlock( int i, int j, int k ) { int i2 = (i+nprocx)%nprocx; int j2 = (j+nprocy)%nprocy; int k2 = (k+nprocz)%nprocz; return i2 + j2*nprocx + k2*nprocx*nprocy; } inline void PackBlobData(int *list, int count, int *sendbuf, int *data){ // Fill in the phase ID values from neighboring processors // This packs up the values that need to be sent from one processor to another int idx,n; for (idx=0; idx fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); int count = 0; while (count < timesteps){ // Communicate the halo of values fillData.fill(Distance); // Execute the next timestep for (k=1;k 0.0 ){ for (q=0; q<26; q++){ Cqx = 1.0*D3Q27[q][0]; Cqy = 1.0*D3Q27[q][1]; Cqz = 1.0*D3Q27[q][2]; // get the associated neighbor in = i + D3Q27[q][0]; jn = j + D3Q27[q][1]; kn = k + D3Q27[q][2]; // make sure the neighbor is in the domain (periodic BC) /* if (in < 0 ) in +=Nx; * don't need this in parallel since MPI handles the halos if (jn < 0 ) jn +=Ny; if (kn < 0 ) kn +=Nz; if (!(in < Nx) ) in -=Nx; if (!(jn < Ny) ) jn -=Ny; if (!(kn < Nz) ) kn -=Nz; // symmetric boundary if (in < 0 ) in = i; if (jn < 0 ) jn = j; if (kn < 0 ) kn = k; if (!(in < Nx) ) in = i; if (!(jn < Ny) ) jn = k; if (!(kn < Nz) ) kn = k; */ // Compute the gradient using upwind finite differences Dqx = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqx; Dqy = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqy; Dqz = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqz; // Only include upwind derivatives if (sign*(nx*Cqx + ny*Cqy + nz*Cqz) < 0.0 ){ Dx += Dqx; Dy += Dqy; Dz += Dqz; W += weights[q]; } } // Normalize by the weight to get the approximation to the gradient Dx /= W; Dy /= W; Dz /= W; norm = sqrt(Dx*Dx+Dy*Dy+Dz*Dz); } else{ norm = 0.0; } Distance(i,j,k) += dt*sign*(1.0 - norm); // Disallow any change in phase // if (Distance(i,j,k)*2.0*(ID[n]-1.0) < 0) Distance(i,j,k) = -Distance(i,j,k); } } } count++; } } inline void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad) { // Read in the full sphere pack //...... READ IN THE SPHERES................................... cout << "Reading the packing file..." << endl; FILE *fid = fopen("pack.out","rb"); INSIST(fid!=NULL,"Error opening pack.out"); //.........Trash the header lines.......... char line[100]; fgetl(line, 100, fid); fgetl(line, 100, fid); fgetl(line, 100, fid); fgetl(line, 100, fid); fgetl(line, 100, fid); //........read the spheres.................. // We will read until a blank like or end-of-file is reached int count = 0; while ( !feof(fid) && fgets(line,100,fid)!=NULL ) { char* line2 = line; List_cx[count] = strtod(line2,&line2); List_cy[count] = strtod(line2,&line2); List_cz[count] = strtod(line2,&line2); List_rad[count] = strtod(line2,&line2); count++; } cout << "Number of spheres extracted is: " << count << endl; INSIST( count==nspheres, "Specified number of spheres is probably incorrect!" ); // ............................................................. } inline void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz) { // Use sphere lists to determine which nodes are in porespace // Write out binary file for nodes char value; int N = Nx*Ny*Nz; // Domain size, including the halo double hx,hy,hz; double x,y,z; double cx,cy,cz,r; int imin,imax,jmin,jmax,kmin,kmax; int p,i,j,k,n; //............................................ double min_x,min_y,min_z; // double max_x,max_y,max_z; //............................................ // Lattice spacing for the entire domain // It should generally be true that hx=hy=hz // Otherwise, you will end up with ellipsoids hx = Lx/(Nx*nprocx-1); hy = Ly/(Ny*nprocy-1); hz = Lz/(Nz*nprocz-1); //............................................ // Get maximum and minimum for this domain // Halo is included ! min_x = double(iproc*Nx-1)*hx; min_y = double(jproc*Ny-1)*hy; min_z = double(kproc*Nz-1)*hz; // max_x = ((iproc+1)*Nx+1)*hx; // max_y = ((jproc+1)*Ny+1)*hy; // max_z = ((kproc+1)*Nz+1)*hz; //............................................ //............................................ // Pre-initialize local ID for (n=0;nNx) imin = Nx; if (imax<0) imax = 0; if (imax>Nx) imax = Nx; if (jmin<0) jmin = 0; if (jmin>Ny) jmin = Ny; if (jmax<0) jmax = 0; if (jmax>Ny) jmax = Ny; if (kmin<0) kmin = 0; if (kmin>Nz) kmin = Nz; if (kmax<0) kmax = 0; if (kmax>Nz) kmax = Nz; // Loop over the domain for this sphere (may be null) for (i=imin;iNx) imin = Nx; if (imax<0) imax = 0; if (imax>Nx) imax = Nx; if (jmin<0) jmin = 0; if (jmin>Ny) jmin = Ny; if (jmax<0) jmax = 0; if (jmax>Ny) jmax = Ny; if (kmin<0) kmin = 0; if (kmin>Nz) kmin = Nz; if (kmax<0) kmax = 0; if (kmax>Nz) kmax = Nz; // Loop over the domain for this sphere (may be null) for (i=imin;i