//************************************************************************* // Lattice Boltzmann Simulator for Single Phase Flow in Porous Media // James E. McCLure //************************************************************************* #include #include #include #include "common/ScaLBL.h" #include "common/MPI_Helpers.h" using namespace std; std::shared_ptr loadInputs( int nprocs ) { //auto db = std::make_shared( "Domain.in" ); auto db = std::make_shared(); db->putScalar( "BC", 0 ); db->putVector( "nproc", { 1, 1, 1 } ); db->putVector( "n", { 5, 5, 5 } ); db->putScalar( "nspheres", 1 ); db->putVector( "L", { 1, 1, 1 } ); return db; } //*************************************************************************************** int main(int argc, char **argv) { //***************************************** // ***** MPI STUFF **************** //***************************************** // Initialize MPI int rank,nprocs; MPI_Init(&argc,&argv); MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); int check=0; { int i,j,k,n; static int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}, {1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0}, {1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1}, {0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}}; if (rank == 0){ printf("********************************************************\n"); printf("Running unit test: TestMap \n"); printf("********************************************************\n"); } // Load inputs auto db = loadInputs( nprocs ); int Nx = db->getVector( "n" )[0]; int Ny = db->getVector( "n" )[1]; int Nz = db->getVector( "n" )[2]; int nprocx = db->getVector( "nproc" )[0]; int nprocy = db->getVector( "nproc" )[1]; int nprocz = db->getVector( "nproc" )[2]; std::shared_ptr Dm(new Domain(db,comm)); Nx += 2; Ny += 2; Nz += 2; int N = Nx*Ny*Nz; //....................................................................... int Np = 0; for (k=1;kid[n] = 1; Np++; } } } Dm->CommInit(); // Create a communicator for the device (will use optimized layout) std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); //Create a second communicator based on the regular data layout std::shared_ptr ScaLBL_Comm_Regular(new ScaLBL_Communicator(Dm)); if (rank==0){ printf("Total domain size = %i \n",N); printf("Reduced domain size = %i \n",Np); } // LBM variables if (rank==0) printf ("Set up the neighborlist \n"); int Npad=Np+32; int neighborSize=18*Npad*sizeof(int); int *neighborList; IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); // Check the neighborlist printf("Check neighborlist: exterior %i, first interior %i last interior %i \n",ScaLBL_Comm->LastExterior(),ScaLBL_Comm->FirstInterior(),ScaLBL_Comm->LastInterior()); for (int idx=0; idxLastExterior(); idx++){ for (int q=0; q<18; q++){ int nn = neighborList[q*Np+idx]%Np; if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx); } } for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ for (int q=0; q<18; q++){ int nn = neighborList[q*Np+idx]%Np; if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx); } } //......................device distributions................................. int *NeighborList; int *dvcMap; //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Npad); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device map and neighbor list \n"); int *TmpMap; TmpMap=new int[Np*sizeof(int)]; for (k=1; kfirst_interior; idxlast_interior; idx++){ n = TmpMap[idx]; k = n/(Nx*Ny); j = (n-Nx*Ny*k)/Nx; i = n-Nx*Ny*k-Nx*j; for (int q=1; q<19; q++){ int nn = neighborList[(q-1)*Np+idx]; double value=fq[nn]; // 3D index of neighbor int iq=i-D3Q19[q-1][0]; int jq=j-D3Q19[q-1][1]; int kq=k-D3Q19[q-1][2]; if (iq==0) iq=1; if (jq==0) jq=1; if (kq==0) kq=1; if (iq==Nx-1) iq=Nx-2; if (jq==Ny-1) jq=Ny-2; if (kq==Nz-1) kq=Nz-2; double check = kq*100.f+jq*10.f+iq*1.f+q*0.01; if (value != check) printf("Neighbor q=%i, i=%i,j=%i,k=%i: %f \n",q,iq,jq,kq,value); } } delete [] TmpMap; } // **************************************************** MPI_Barrier(comm); MPI_Finalize(); // **************************************************** return check; }