diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 1bc32c5f..0c4caa81 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -413,12 +413,12 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis for (k=2; k 0 ){ - Map(n) = idx++; - //neighborList[idx++] = n; // index of self in regular layout - } + // Local index (regular layout) + n = k*Nx*Ny + j*Nx + i; + if (id[n] > 0 ){ + Map(n) = idx++; + //neighborList[idx++] = n; // index of self in regular layout + } } } } @@ -543,14 +543,6 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis } ScaLBL_CopyToDevice(dvcSendList_x,TempBuffer,sendCount_x*sizeof(int)); - - - - - - - - ScaLBL_CopyToHost(TempBuffer,dvcSendList_y,sendCount_y*sizeof(int)); for (i=0; iLastInterior(); idx++){ - if (idx == ScaLBL_Comm->LastExterior()) idx = ScaLBL_Comm->FirstInterior(); + for (int idx=0; idxLastExterior(); idx++){ + int n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n"); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ int n = TmpMap[idx]; if (n > Nx*Ny*Nz){ printf("Bad value! idx=%i \n"); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 22d18656..9c8d4dfe 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -61,7 +61,7 @@ ADD_LBPM_TEST_1_2_4( TestBlobIdentify ) #ADD_LBPM_TEST_PARALLEL( TestTwoPhase 8 ) #ADD_LBPM_TEST_PARALLEL( TestBlobAnalyze 8 ) ADD_LBPM_TEST_PARALLEL( TestSegDist 8 ) -#ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 ) +ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 ) #ADD_LBPM_TEST_PARALLEL( TestMassConservationD3Q7 1 ) ADD_LBPM_TEST_1_2_4( testCommunication ) ADD_LBPM_TEST_1_2_4( testUtilities ) diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index e67ce5c9..0c3988c8 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -13,8 +13,8 @@ using namespace std; std::shared_ptr loadInputs( int nprocs ) { - auto db = std::make_shared( "Domain.in" ); - const int dim = 50; + auto db = std::make_shared(); + const int dim = 8; db->putScalar( "BC", 0 ); if ( nprocs == 1 ){ db->putVector( "nproc", { 1, 1, 1 } ); @@ -180,17 +180,6 @@ int main(int argc, char **argv) int check; { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - //***************************************** - // MPI ranks for all 18 neighbors - //********************************** - int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z; - int rank_xy,rank_XY,rank_xY,rank_Xy; - int rank_xz,rank_XZ,rank_xZ,rank_Xz; - int rank_yz,rank_YZ,rank_yZ,rank_Yz; - //********************************** MPI_Request req1[18],req2[18]; MPI_Status stat1[18],stat2[18]; @@ -210,31 +199,15 @@ int main(int argc, char **argv) // Load inputs auto db = loadInputs( nprocs ); + /* auto filename = argv[1]; + auto input_db = std::make_shared( filename ); + auto db = input_db->getDatabase( "Domain" ); + */ int Nx = db->getVector( "n" )[0]; int Ny = db->getVector( "n" )[1]; int Nz = db->getVector( "n" )[2]; - - if (rank==0){ - printf("********************************************************\n"); - printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz); - printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz); - printf("********************************************************\n"); - } - - MPI_Barrier(comm); - kproc = rank/(nprocx*nprocy); - jproc = (rank-nprocx*nprocy*kproc)/nprocx; - iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; - - double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - Domain Dm(db); - - - InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, - rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, - rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, - rank_yz, rank_YZ, rank_yZ, rank_Yz ); + auto Dm = std::shared_ptr(new Domain(db,comm)); // full domain for analysis Nx += 2; Ny += 2; @@ -253,25 +226,39 @@ int main(int argc, char **argv) char *id; id = new char[Nx*Ny*Nz]; - - if (rank==0) printf("Assigning phase ID from file \n"); + /* if (rank==0) printf("Assigning phase ID from file \n"); if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); FILE *IDFILE = fopen(LocalRankFilename,"rb"); if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx"); fread(id,1,N,IDFILE); fclose(IDFILE); - + */ // Setup the domain for (k=0;kid[n] = id[n]; } } } - Dm.CommInit(); + Dm->CommInit(); + int iproc,jproc,kproc; + int nprocx,nprocy,nprocz; + iproc = Dm->iproc(); + jproc = Dm->jproc(); + kproc = Dm->kproc(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); + + if (rank==0){ + printf("********************************************************\n"); + printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz); + printf("Parallel domain size = %i x %i x %i\n",Dm->nprocx(),Dm->nprocy(),Dm->nprocz()); + printf("********************************************************\n"); + } //....................................................................... // Compute the media porosity @@ -292,7 +279,8 @@ int main(int argc, char **argv) } } MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); - porosity = 1.0-sum*iVol_global; + double iVol_global=1.f/double((Nx-2)*(Ny-2)*(Nz-2)*nprocx*nprocy*nprocz); + porosity = 1.0-sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); //....................................................................... @@ -301,20 +289,19 @@ int main(int argc, char **argv) if (rank == 0) cout << "Domain set." << endl; //........................................................................... - //........................................................................... if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) ScaLBL_Communicator ScaLBL_Comm(Dm); - if (rank==0) printf ("Set up memory efficient layout \n"); - int neighborSize=18*Np*sizeof(int); - int *neighborList; + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); + auto neighborList= new int[18*Npad]; IntArray Map(Nx,Ny,Nz); - neighborList= new int[18*Np]; - ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + Map.fill(-2); + Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); - + int neighborSize=18*Np*sizeof(int); //......................device distributions................................. dist_mem_size = Np*sizeof(double); if (rank==0) printf ("Allocating distributions \n"); @@ -380,7 +367,7 @@ int main(int argc, char **argv) */ if (rank==0) printf("Setting the distributions, size = : %i\n", Np); //........................................................................... - GlobalFlipScaLBL_D3Q19_Init(fq_host, Map, Np, Nx-2, Ny-2, Nz-2,iproc,jproc,kproc,nprocx,nprocy,nprocz); + GlobalFlipScaLBL_D3Q19_Init(fq_host, Map, Np, Nx-2, Ny-2, Nz-2, iproc,jproc,kproc,nprocx,nprocy,nprocz); ScaLBL_CopyToDevice(fq, fq_host, 19*dist_mem_size); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -395,7 +382,6 @@ int main(int argc, char **argv) ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - //........................................................................... ScaLBL_CopyToHost(fq_host,fq,19*Np*sizeof(double)); check = GlobalCheckDebugDist(fq_host, Map, Np, Nx-2, Ny-2, Nz-2,iproc,jproc,kproc,nprocx,nprocy,nprocz,0,ScaLBL_Comm.next);