// Created by James McClure // Copyright 2008-2013 #include #include #include #include #include #include #include // std::exception #include #include "common/Domain.h" #include "common/Array.h" #include "common/Utilities.h" #include "common/MPI_Helpers.h" #include "common/Communication.h" static int MAX_BLOB_COUNT=50; using namespace std; /******************************************************** * Constructor/Destructor * ********************************************************/ Domain::Domain(int nx, int ny, int nz, int rnk, int npx, int npy, int npz, double lx, double ly, double lz, int BC): Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), nprocx(0), nprocy(0), nprocz(0), Lx(0), Ly(0), Lz(0), Volume(0), rank(0), BoundaryCondition(0), Group(MPI_GROUP_NULL), Comm(MPI_COMM_NULL), rank_x(0), rank_y(0), rank_z(0), rank_X(0), rank_Y(0), rank_Z(0), rank_xy(0), rank_XY(0), rank_xY(0), rank_Xy(0), rank_xz(0), rank_XZ(0), rank_xZ(0), rank_Xz(0), rank_yz(0), rank_YZ(0), rank_yZ(0), rank_Yz(0), sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), sendList_x(NULL), sendList_y(NULL), sendList_z(NULL), sendList_X(NULL), sendList_Y(NULL), sendList_Z(NULL), sendList_xy(NULL), sendList_yz(NULL), sendList_xz(NULL), sendList_Xy(NULL), sendList_Yz(NULL), sendList_xZ(NULL), sendList_xY(NULL), sendList_yZ(NULL), sendList_Xz(NULL), sendList_XY(NULL), sendList_YZ(NULL), sendList_XZ(NULL), sendBuf_x(NULL), sendBuf_y(NULL), sendBuf_z(NULL), sendBuf_X(NULL), sendBuf_Y(NULL), sendBuf_Z(NULL), sendBuf_xy(NULL), sendBuf_yz(NULL), sendBuf_xz(NULL), sendBuf_Xy(NULL), sendBuf_Yz(NULL), sendBuf_xZ(NULL), sendBuf_xY(NULL), sendBuf_yZ(NULL), sendBuf_Xz(NULL), sendBuf_XY(NULL), sendBuf_YZ(NULL), sendBuf_XZ(NULL), recvCount_x(0), recvCount_y(0), recvCount_z(0), recvCount_X(0), recvCount_Y(0), recvCount_Z(0), recvCount_xy(0), recvCount_yz(0), recvCount_xz(0), recvCount_Xy(0), recvCount_Yz(0), recvCount_xZ(0), recvCount_xY(0), recvCount_yZ(0), recvCount_Xz(0), recvCount_XY(0), recvCount_YZ(0), recvCount_XZ(0), recvList_x(NULL), recvList_y(NULL), recvList_z(NULL), recvList_X(NULL), recvList_Y(NULL), recvList_Z(NULL), recvList_xy(NULL), recvList_yz(NULL), recvList_xz(NULL), recvList_Xy(NULL), recvList_Yz(NULL), recvList_xZ(NULL), recvList_xY(NULL), recvList_yZ(NULL), recvList_Xz(NULL), recvList_XY(NULL), recvList_YZ(NULL), recvList_XZ(NULL), recvBuf_x(NULL), recvBuf_y(NULL), recvBuf_z(NULL), recvBuf_X(NULL), recvBuf_Y(NULL), recvBuf_Z(NULL), recvBuf_xy(NULL), recvBuf_yz(NULL), recvBuf_xz(NULL), recvBuf_Xy(NULL), recvBuf_Yz(NULL), recvBuf_xZ(NULL), recvBuf_xY(NULL), recvBuf_yZ(NULL), recvBuf_Xz(NULL), recvBuf_XY(NULL), recvBuf_YZ(NULL), recvBuf_XZ(NULL), sendData_x(NULL), sendData_y(NULL), sendData_z(NULL), sendData_X(NULL), sendData_Y(NULL), sendData_Z(NULL), sendData_xy(NULL), sendData_yz(NULL), sendData_xz(NULL), sendData_Xy(NULL), sendData_Yz(NULL), sendData_xZ(NULL), sendData_xY(NULL), sendData_yZ(NULL), sendData_Xz(NULL), sendData_XY(NULL), sendData_YZ(NULL), sendData_XZ(NULL), recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), id(NULL) { Volume = nx*ny*nx*npx*npy*npz*1.0; Nx = nx+2; Ny = ny+2; Nz = nz+2; Lx = lx, Ly = ly, Lz = lz; rank = rnk; nprocx=npx; nprocy=npy; nprocz=npz; N = Nx*Ny*Nz; id = new char[N]; memset(id,0,N); BlobLabel.resize(Nx,Ny,Nz); BlobGraph.resize(18,MAX_BLOB_COUNT,MAX_BLOB_COUNT); BoundaryCondition = BC; rank_info=RankInfoStruct(rank,nprocx,nprocy,nprocz); } Domain::~Domain() { // Free sendList delete [] sendList_x; delete [] sendList_y; delete [] sendList_z; delete [] sendList_X; delete [] sendList_Y; delete [] sendList_Z; delete [] sendList_xy; delete [] sendList_yz; delete [] sendList_xz; delete [] sendList_Xy; delete [] sendList_Yz; delete [] sendList_xZ; delete [] sendList_xY; delete [] sendList_yZ; delete [] sendList_Xz; delete [] sendList_XY; delete [] sendList_YZ; delete [] sendList_XZ; // Free sendBuf delete [] sendBuf_x; delete [] sendBuf_y; delete [] sendBuf_z; delete [] sendBuf_X; delete [] sendBuf_Y; delete [] sendBuf_Z; delete [] sendBuf_xy; delete [] sendBuf_yz; delete [] sendBuf_xz; delete [] sendBuf_Xy; delete [] sendBuf_Yz; delete [] sendBuf_xZ; delete [] sendBuf_xY; delete [] sendBuf_yZ; delete [] sendBuf_Xz; delete [] sendBuf_XY; delete [] sendBuf_YZ; delete [] sendBuf_XZ; // Free recvList delete [] recvList_x; delete [] recvList_y; delete [] recvList_z; delete [] recvList_X; delete [] recvList_Y; delete [] recvList_Z; delete [] recvList_xy; delete [] recvList_yz; delete [] recvList_xz; delete [] recvList_Xy; delete [] recvList_Yz; delete [] recvList_xZ; delete [] recvList_xY; delete [] recvList_yZ; delete [] recvList_Xz; delete [] recvList_XY; delete [] recvList_YZ; delete [] recvList_XZ; // Free recvBuf delete [] recvBuf_x; delete [] recvBuf_y; delete [] recvBuf_z; delete [] recvBuf_X; delete [] recvBuf_Y; delete [] recvBuf_Z; delete [] recvBuf_xy; delete [] recvBuf_yz; delete [] recvBuf_xz; delete [] recvBuf_Xy; delete [] recvBuf_Yz; delete [] recvBuf_xZ; delete [] recvBuf_xY; delete [] recvBuf_yZ; delete [] recvBuf_Xz; delete [] recvBuf_XY; delete [] recvBuf_YZ; delete [] recvBuf_XZ; // Free sendData delete [] sendData_x; delete [] sendData_y; delete [] sendData_z; delete [] sendData_X; delete [] sendData_Y; delete [] sendData_Z; delete [] sendData_xy; delete [] sendData_xY; delete [] sendData_Xy; delete [] sendData_XY; delete [] sendData_xz; delete [] sendData_xZ; delete [] sendData_Xz; delete [] sendData_XZ; delete [] sendData_yz; delete [] sendData_yZ; delete [] sendData_Yz; delete [] sendData_YZ; // Free recvData delete [] recvData_x; delete [] recvData_y; delete [] recvData_z; delete [] recvData_X; delete [] recvData_Y; delete [] recvData_Z; delete [] recvData_xy; delete [] recvData_xY; delete [] recvData_Xy; delete [] recvData_XY; delete [] recvData_xz; delete [] recvData_xZ; delete [] recvData_Xz; delete [] recvData_XZ; delete [] recvData_yz; delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ; // Free id delete [] id; // Free the communicator if ( Group!=MPI_GROUP_NULL ) { MPI_Group_free(&Group); MPI_Comm_free(&Comm); } } int Domain::VoxelConnection(int n) { const int d[26][3] = {{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}, {1,1,0},{1,-1,0},{-1,1,0},{-1,-1,0},{1,0,1},{-1,0,1}, {1,0,-1},{-1,0,-1},{0,1,1},{0,-1,1},{0,1,-1},{0,-1,-1}, {1,1,1},{1,1,-1},{1,-1,1},{1,-1,-1},{-1,1,1},{-1,1,-1}, {-1,-1,1},{-1,-1,-1}}; // directions to neighbors int returnVal = -1; int x,y,z; // Get the 3-D indices x = n%Nx; y = (n/Nx)%Ny; z = n/(Nx*Ny); int nodx,nody,nodz; for (int p=0;p<26;p++){ nodx=x+d[p][0]; // Get the neighbor and guarantee it is in the domain if (nodx < 0 ){ nodx = 0; } if (nodx > Nx-1 ){ nodx = Nx-1; } nody=y+d[p][1]; if (nody < 0 ){ nody = 0; } if (nody > Ny-1 ){ nody = Ny-1; } nodz=z+d[p][2]; if (nodz < 0 ){ nodz = 0; } if (nodz > Nz-1 ){ nodz = Nz-1; } if (BlobLabel(nodx,nody,nodz) > returnVal ) returnVal = BlobLabel(nodx,nody,nodz); } return returnVal; } void Domain::getBlobConnections(int * List, int count, int neighbor, int direction){ int idx,n,localValue,neighborValue; int x,y,z; for (idx=0; idx -1){ localValue = VoxelConnection(n); printf("Blob (%i,%i) connects to neighbor blob (%i,%i)", localValue, rank, neighborValue, neighbor); BlobGraph(direction,localValue,neighbor) = 1; // Set the BlobGraph to TRUE for this pair } } } void Domain::InitializeRanks() { // map the rank to the block index iproc = rank%nprocx; jproc = (rank/nprocx)%nprocy; kproc = rank/(nprocx*nprocy); // set up the neighbor ranks int i = iproc; int j = jproc; int k = kproc; rank_X = getRankForBlock(i+1,j,k); rank_x = getRankForBlock(i-1,j,k); rank_Y = getRankForBlock(i,j+1,k); rank_y = getRankForBlock(i,j-1,k); rank_Z = getRankForBlock(i,j,k+1); rank_z = getRankForBlock(i,j,k-1); rank_XY = getRankForBlock(i+1,j+1,k); rank_xy = getRankForBlock(i-1,j-1,k); rank_Xy = getRankForBlock(i+1,j-1,k); rank_xY = getRankForBlock(i-1,j+1,k); rank_XZ = getRankForBlock(i+1,j,k+1); rank_xz = getRankForBlock(i-1,j,k-1); rank_Xz = getRankForBlock(i+1,j,k-1); rank_xZ = getRankForBlock(i-1,j,k+1); rank_YZ = getRankForBlock(i,j+1,k+1); rank_yz = getRankForBlock(i,j-1,k-1); rank_Yz = getRankForBlock(i,j+1,k-1); rank_yZ = getRankForBlock(i,j-1,k+1); } void Domain::CommInit(MPI_Comm Communicator){ int i,j,k,n; int sendtag = 21; int recvtag = 21; //...................................................................................... //Get the ranks of each process and it's neighbors // map the rank to the block index iproc = rank%nprocx; jproc = (rank/nprocx)%nprocy; kproc = rank/(nprocx*nprocy); // set up the neighbor ranks i = iproc; j = jproc; k = kproc; rank_X = getRankForBlock(i+1,j,k); rank_x = getRankForBlock(i-1,j,k); rank_Y = getRankForBlock(i,j+1,k); rank_y = getRankForBlock(i,j-1,k); rank_Z = getRankForBlock(i,j,k+1); rank_z = getRankForBlock(i,j,k-1); rank_XY = getRankForBlock(i+1,j+1,k); rank_xy = getRankForBlock(i-1,j-1,k); rank_Xy = getRankForBlock(i+1,j-1,k); rank_xY = getRankForBlock(i-1,j+1,k); rank_XZ = getRankForBlock(i+1,j,k+1); rank_xz = getRankForBlock(i-1,j,k-1); rank_Xz = getRankForBlock(i+1,j,k-1); rank_xZ = getRankForBlock(i-1,j,k+1); rank_YZ = getRankForBlock(i,j+1,k+1); rank_yz = getRankForBlock(i,j-1,k-1); rank_Yz = getRankForBlock(i,j+1,k-1); rank_yZ = getRankForBlock(i,j-1,k+1); //...................................................................................... MPI_Comm_group(Communicator,&Group); MPI_Comm_create(Communicator,Group,&Comm); //...................................................................................... MPI_Request req1[18], req2[18]; MPI_Status stat1[18],stat2[18]; //...................................................................................... sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0; sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0; sendCount_xY = sendCount_yZ = sendCount_Xz = sendCount_XY = sendCount_YZ = sendCount_XZ = 0; //...................................................................................... for (k=1; k