diff --git a/common/Domain.cpp b/common/Domain.cpp index 75f2082b..d975ce25 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -113,6 +113,59 @@ Domain::Domain( std::shared_ptr db, MPI_Comm Communicator): rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); MPI_Barrier(Comm); } + +Domain::~Domain() +{ + // Free sendList + delete [] sendList_x; delete [] sendList_y; delete [] sendList_z; + delete [] sendList_X; delete [] sendList_Y; delete [] sendList_Z; + delete [] sendList_xy; delete [] sendList_yz; delete [] sendList_xz; + delete [] sendList_Xy; delete [] sendList_Yz; delete [] sendList_xZ; + delete [] sendList_xY; delete [] sendList_yZ; delete [] sendList_Xz; + delete [] sendList_XY; delete [] sendList_YZ; delete [] sendList_XZ; + // Free sendBuf + delete [] sendBuf_x; delete [] sendBuf_y; delete [] sendBuf_z; + delete [] sendBuf_X; delete [] sendBuf_Y; delete [] sendBuf_Z; + delete [] sendBuf_xy; delete [] sendBuf_yz; delete [] sendBuf_xz; + delete [] sendBuf_Xy; delete [] sendBuf_Yz; delete [] sendBuf_xZ; + delete [] sendBuf_xY; delete [] sendBuf_yZ; delete [] sendBuf_Xz; + delete [] sendBuf_XY; delete [] sendBuf_YZ; delete [] sendBuf_XZ; + // Free recvList + delete [] recvList_x; delete [] recvList_y; delete [] recvList_z; + delete [] recvList_X; delete [] recvList_Y; delete [] recvList_Z; + delete [] recvList_xy; delete [] recvList_yz; delete [] recvList_xz; + delete [] recvList_Xy; delete [] recvList_Yz; delete [] recvList_xZ; + delete [] recvList_xY; delete [] recvList_yZ; delete [] recvList_Xz; + delete [] recvList_XY; delete [] recvList_YZ; delete [] recvList_XZ; + // Free recvBuf + delete [] recvBuf_x; delete [] recvBuf_y; delete [] recvBuf_z; + delete [] recvBuf_X; delete [] recvBuf_Y; delete [] recvBuf_Z; + delete [] recvBuf_xy; delete [] recvBuf_yz; delete [] recvBuf_xz; + delete [] recvBuf_Xy; delete [] recvBuf_Yz; delete [] recvBuf_xZ; + delete [] recvBuf_xY; delete [] recvBuf_yZ; delete [] recvBuf_Xz; + delete [] recvBuf_XY; delete [] recvBuf_YZ; delete [] recvBuf_XZ; + // Free sendData + delete [] sendData_x; delete [] sendData_y; delete [] sendData_z; + delete [] sendData_X; delete [] sendData_Y; delete [] sendData_Z; + delete [] sendData_xy; delete [] sendData_xY; delete [] sendData_Xy; + delete [] sendData_XY; delete [] sendData_xz; delete [] sendData_xZ; + delete [] sendData_Xz; delete [] sendData_XZ; delete [] sendData_yz; + delete [] sendData_yZ; delete [] sendData_Yz; delete [] sendData_YZ; + // Free recvData + delete [] recvData_x; delete [] recvData_y; delete [] recvData_z; + delete [] recvData_X; delete [] recvData_Y; delete [] recvData_Z; + delete [] recvData_xy; delete [] recvData_xY; delete [] recvData_Xy; + delete [] recvData_XY; delete [] recvData_xz; delete [] recvData_xZ; + delete [] recvData_Xz; delete [] recvData_XZ; delete [] recvData_yz; + delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ; + // Free id + delete [] id; + // Free the communicator + if ( Comm != MPI_COMM_WORLD && Comm != MPI_COMM_NULL ) { + MPI_Comm_free(&Comm); + } +} + void Domain::initialize( std::shared_ptr db ) { d_db = db; @@ -172,58 +225,324 @@ void Domain::initialize( std::shared_ptr db ) MPI_Comm_size( Comm, &nprocs ); INSIST(nprocs == nproc[0]*nproc[1]*nproc[2],"Fatal error in processor count!"); } -Domain::~Domain() -{ - // Free sendList - delete [] sendList_x; delete [] sendList_y; delete [] sendList_z; - delete [] sendList_X; delete [] sendList_Y; delete [] sendList_Z; - delete [] sendList_xy; delete [] sendList_yz; delete [] sendList_xz; - delete [] sendList_Xy; delete [] sendList_Yz; delete [] sendList_xZ; - delete [] sendList_xY; delete [] sendList_yZ; delete [] sendList_Xz; - delete [] sendList_XY; delete [] sendList_YZ; delete [] sendList_XZ; - // Free sendBuf - delete [] sendBuf_x; delete [] sendBuf_y; delete [] sendBuf_z; - delete [] sendBuf_X; delete [] sendBuf_Y; delete [] sendBuf_Z; - delete [] sendBuf_xy; delete [] sendBuf_yz; delete [] sendBuf_xz; - delete [] sendBuf_Xy; delete [] sendBuf_Yz; delete [] sendBuf_xZ; - delete [] sendBuf_xY; delete [] sendBuf_yZ; delete [] sendBuf_Xz; - delete [] sendBuf_XY; delete [] sendBuf_YZ; delete [] sendBuf_XZ; - // Free recvList - delete [] recvList_x; delete [] recvList_y; delete [] recvList_z; - delete [] recvList_X; delete [] recvList_Y; delete [] recvList_Z; - delete [] recvList_xy; delete [] recvList_yz; delete [] recvList_xz; - delete [] recvList_Xy; delete [] recvList_Yz; delete [] recvList_xZ; - delete [] recvList_xY; delete [] recvList_yZ; delete [] recvList_Xz; - delete [] recvList_XY; delete [] recvList_YZ; delete [] recvList_XZ; - // Free recvBuf - delete [] recvBuf_x; delete [] recvBuf_y; delete [] recvBuf_z; - delete [] recvBuf_X; delete [] recvBuf_Y; delete [] recvBuf_Z; - delete [] recvBuf_xy; delete [] recvBuf_yz; delete [] recvBuf_xz; - delete [] recvBuf_Xy; delete [] recvBuf_Yz; delete [] recvBuf_xZ; - delete [] recvBuf_xY; delete [] recvBuf_yZ; delete [] recvBuf_Xz; - delete [] recvBuf_XY; delete [] recvBuf_YZ; delete [] recvBuf_XZ; - // Free sendData - delete [] sendData_x; delete [] sendData_y; delete [] sendData_z; - delete [] sendData_X; delete [] sendData_Y; delete [] sendData_Z; - delete [] sendData_xy; delete [] sendData_xY; delete [] sendData_Xy; - delete [] sendData_XY; delete [] sendData_xz; delete [] sendData_xZ; - delete [] sendData_Xz; delete [] sendData_XZ; delete [] sendData_yz; - delete [] sendData_yZ; delete [] sendData_Yz; delete [] sendData_YZ; - // Free recvData - delete [] recvData_x; delete [] recvData_y; delete [] recvData_z; - delete [] recvData_X; delete [] recvData_Y; delete [] recvData_Z; - delete [] recvData_xy; delete [] recvData_xY; delete [] recvData_Xy; - delete [] recvData_XY; delete [] recvData_xz; delete [] recvData_xZ; - delete [] recvData_Xz; delete [] recvData_XZ; delete [] recvData_yz; - delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ; - // Free id - delete [] id; - // Free the communicator - if ( Comm != MPI_COMM_WORLD && Comm != MPI_COMM_NULL ) { - MPI_Comm_free(&Comm); - } -} +void Domain::Decomp(std::shared_ptr domain_db ) +{ + //....................................................................... + // Reading the domain information file + //....................................................................... + int nprocs, nprocx, nprocy, nprocz, nx, ny, nz; + int64_t global_Nx,global_Ny,global_Nz; + int64_t i,j,k,n; + int BC=0; + int64_t xStart,yStart,zStart; + int checkerSize; + //int inlet_layers_x, inlet_layers_y, inlet_layers_z; + //int outlet_layers_x, outlet_layers_y, outlet_layers_z; + xStart=yStart=zStart=0; + inlet_layers_x = 0; + inlet_layers_y = 0; + inlet_layers_z = 0; + outlet_layers_x = 0; + outlet_layers_y = 0; + outlet_layers_z = 0; + checkerSize = 32; + + // read the input database + //auto db = std::make_shared( filename ); + //auto domain_db = db->getDatabase( "Domain" ); + + // Read domain parameters + auto Filename = domain_db->getScalar( "Filename" ); + //auto L = domain_db->getVector( "L" ); + auto size = domain_db->getVector( "n" ); + auto SIZE = domain_db->getVector( "N" ); + auto nproc = domain_db->getVector( "nproc" ); + if (domain_db->keyExists( "offset" )){ + auto offset = domain_db->getVector( "offset" ); + xStart = offset[0]; + yStart = offset[1]; + zStart = offset[2]; + } + if (domain_db->keyExists( "InletLayers" )){ + auto InletCount = domain_db->getVector( "InletLayers" ); + inlet_layers_x = InletCount[0]; + inlet_layers_y = InletCount[1]; + inlet_layers_z = InletCount[2]; + } + if (domain_db->keyExists( "OutletLayers" )){ + auto OutletCount = domain_db->getVector( "OutletLayers" ); + outlet_layers_x = OutletCount[0]; + outlet_layers_y = OutletCount[1]; + outlet_layers_z = OutletCount[2]; + } + if (domain_db->keyExists( "checkerSize" )){ + checkerSize = domain_db->getScalar( "checkerSize" ); + } + else { + checkerSize = SIZE[0]; + } + auto ReadValues = domain_db->getVector( "ReadValues" ); + auto WriteValues = domain_db->getVector( "WriteValues" ); + auto ReadType = domain_db->getScalar( "ReadType" ); + if (ReadType == "8bit"){ + } + else if (ReadType == "16bit"){ + } + else{ + printf("INPUT ERROR: Valid ReadType are 8bit, 16bit \n"); + ReadType = "8bit"; + } + + nx = size[0]; + ny = size[1]; + nz = size[2]; + nprocx = nproc[0]; + nprocy = nproc[1]; + nprocz = nproc[2]; + global_Nx = SIZE[0]; + global_Ny = SIZE[1]; + global_Nz = SIZE[2]; + + printf("Input media: %s\n",Filename.c_str()); + printf("Relabeling %lu values\n",ReadValues.size()); + for (int idx=0; idx 0){ + // use checkerboard pattern + printf("Checkerboard pattern at x inlet for %i layers \n",inlet_layers_x); + for (int k = 0; k 0){ + printf("Checkerboard pattern at y inlet for %i layers \n",inlet_layers_y); + // use checkerboard pattern + for (int k = 0; k 0){ + printf("Checkerboard pattern at z inlet for %i layers \n",inlet_layers_z); + // use checkerboard pattern + for (int k = zStart; k < zStart+inlet_layers_z; k++){ + for (int j = 0; j 0){ + // use checkerboard pattern + printf("Checkerboard pattern at x outlet for %i layers \n",outlet_layers_x); + for (int k = 0; k 0){ + printf("Checkerboard pattern at y outlet for %i layers \n",outlet_layers_y); + // use checkerboard pattern + for (int k = 0; k 0){ + printf("Checkerboard pattern at z outlet for %i layers \n",outlet_layers_z); + // use checkerboard pattern + for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){ + for (int j = 0; j LabelCount(ReadValues.size(),0); + // Set up the sub-domains + if (rank==0){ + printf("Distributing subdomains across %i processors \n",nprocs); + printf("Process grid: %i x %i x %i \n",nprocx,nprocy,nprocz); + printf("Subdomain size: %i x %i x %i \n",nx,ny,nz); + printf("Size of transition region: %ld \n", z_transition_size); + + for (int kp=0; kp