//************************************************************************* // Lattice Boltzmann Simulator for Single Phase Flow in Porous Media // James E. McCLure //************************************************************************* #include #include #include #include "common/ScaLBL.h" #include "common/MPI.h" using namespace std; std::shared_ptr loadInputs( int nprocs ) { //auto db = std::make_shared( "Domain.in" ); auto db = std::make_shared(); db->putScalar( "BC", 0 ); db->putVector( "nproc", { 1, 1, 1 } ); db->putVector( "n", { 5, 5, 5 } ); db->putScalar( "nspheres", 1 ); db->putVector( "L", { 1, 1, 1 } ); return db; } //*************************************************************************************** int main(int argc, char **argv) { // Initialize MPI Utilities::startup( argc, argv ); Utilities::MPI comm( MPI_COMM_WORLD ); int check=0; { int i,j,k,n; static int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}, {1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0}, {1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1}, {0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}}; int rank = comm.getRank(); if (rank == 0){ printf("********************************************************\n"); printf("Running unit test: TestMap \n"); printf("********************************************************\n"); } // Load inputs auto db = loadInputs( comm.getSize() ); int Nx = db->getVector( "n" )[0]; int Ny = db->getVector( "n" )[1]; int Nz = db->getVector( "n" )[2]; auto Dm = std::make_shared(db,comm); Nx += 2; Ny += 2; Nz += 2; int N = Nx*Ny*Nz; //....................................................................... int Np = 0; for (k=1;kid[n] = 1; Np++; } } } Dm->CommInit(); // Create a communicator for the device (will use optimized layout) std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); //Create a second communicator based on the regular data layout std::shared_ptr ScaLBL_Comm_Regular(new ScaLBL_Communicator(Dm)); if (rank==0){ printf("Total domain size = %i \n",N); printf("Reduced domain size = %i \n",Np); } // LBM variables if (rank==0) printf ("Set up the neighborlist \n"); int Npad=Np+32; int neighborSize=18*Npad*sizeof(int); int *neighborList; IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np); comm.barrier(); // Check the neighborlist printf("Check neighborlist: exterior %i, first interior %i last interior %i \n",ScaLBL_Comm->LastExterior(),ScaLBL_Comm->FirstInterior(),ScaLBL_Comm->LastInterior()); for (int idx=0; idxLastExterior(); idx++){ for (int q=0; q<18; q++){ int nn = neighborList[q*Np+idx]%Np; if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx); } } for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ for (int q=0; q<18; q++){ int nn = neighborList[q*Np+idx]%Np; if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx); } } //......................device distributions................................. int *NeighborList; int *dvcMap; //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Npad); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device map and neighbor list \n"); int *TmpMap; TmpMap=new int[Np*sizeof(int)]; for (k=1; kfirst_interior; idxlast_interior; idx++){ n = TmpMap[idx]; k = n/(Nx*Ny); j = (n-Nx*Ny*k)/Nx; i = n-Nx*Ny*k-Nx*j; for (int q=1; q<19; q++){ int nn = neighborList[(q-1)*Np+idx]; double value=fq[nn]; // 3D index of neighbor int iq=i-D3Q19[q-1][0]; int jq=j-D3Q19[q-1][1]; int kq=k-D3Q19[q-1][2]; if (iq==0) iq=1; if (jq==0) jq=1; if (kq==0) kq=1; if (iq==Nx-1) iq=Nx-2; if (jq==Ny-1) jq=Ny-2; if (kq==Nz-1) kq=Nz-2; double check = kq*100.f+jq*10.f+iq*1.f+q*0.01; if (value != check) printf("Neighbor q=%i, i=%i,j=%i,k=%i: %f \n",q,iq,jq,kq,value); } } delete [] TmpMap; } Utilities::shutdown(); return check; }