From 151b748bddb7ae9bc96b8767c586cbec6ccebf20 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 22 Nov 2016 14:31:10 -0500 Subject: [PATCH 1/7] Added help flag to lbpm_nondarcy_simulator --- tests/lbpm_nondarcy_simulator.cpp | 976 ++++++++++++++++-------------- 1 file changed, 514 insertions(+), 462 deletions(-) diff --git a/tests/lbpm_nondarcy_simulator.cpp b/tests/lbpm_nondarcy_simulator.cpp index c9ca1213..a4c21a11 100644 --- a/tests/lbpm_nondarcy_simulator.cpp +++ b/tests/lbpm_nondarcy_simulator.cpp @@ -26,514 +26,566 @@ using namespace std; int main(int argc, char **argv) { - //***************************************** - // ***** MPI STUFF **************** - //***************************************** - // Initialize MPI - int rank,nprocs; - MPI_Init(&argc,&argv); - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&nprocs); - { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; + std::string help("--help"); + std::string arg1(""); + if (argc > 1) arg1=argv[1]; + if (help.compare(arg1) == 0){ + printf("********************************************************** \n"); + printf("Pore-scale lattice Boltzmann simulator for non-Darcy flow in porous media \n \n"); + printf("Simulate non-Darcy flow in porous media \n"); + printf(" MPI-based lattice Boltzmann simulator \n"); + printf(" Multi-relaxation time (MRT) D3Q19 \n \n"); + printf(" Launch with MPI (e.g.) \n \n"); + printf(" mpirun -np $NUMPROCS $LBPM_WIA_DIR/lbpm_nondarcy_simulator \n \n"); + printf("**********CITATION********** \n"); + printf(" Dye, A.L., McClure, J.E., Gray, W.G. and C.T. Miller\n"); + printf(" Description of Non-Darcy Flows in Porous Medium Systems \n"); + printf(" Physical Review E 87 (3), 033012 \n \n"); + printf("**********INPUT********** \n"); + printf("1. Domain.in (describes the simulation domain and domain decomposition) \n"); + printf(" ----(e.g. Domain.in)-----\n"); + printf(" nprocx nprocy nprocz (process grid)\n"); + printf(" Nx Ny Nz (local sub-domain)\n"); + printf(" Lx Ly Lz (physical domain size) \n"); + printf(" --------------------------\n"); + printf("2. SignDist.xxxxx (Distance map of porespace) \n"); + printf(" - one file for each MPI process \n"); + printf(" - double precision values \n"); + printf(" - dimensions are [Nx+2,Ny+2,Nz+2] (include halo)\n \n"); + //printf("3. parameters for LBM are hard-coded! \n \n"); + printf("**********OUTPUT********** \n"); + printf("1. nondary.csv - list of averaged quantities obtained from steady-state flow fields\n"); + printf(" - D32 - Sauter mean grain diamter \n"); + printf(" - vx - average velocity in the x-direction \n"); + printf(" - vy - average velocity in the y-direction \n"); + printf(" - vz - average velocity in the z-direction \n"); + printf(" - Fx - body force applied in the x-direction \n"); + printf(" - Fy - body force applied in the y-direction \n"); + printf(" - Fz - body force applied in the z-direction \n"); + printf(" - Fo - Gallilei number \n"); + printf(" - Re - Reynolds number \n"); + printf("********************************************************** \n"); + /*printf("*******DIMENSIONLESS FORCHEIMER EQUATION********\n"); + printf(" - force = |F| (total magnitude of force) \n"); + printf(" - velocity = F dot v / |F| (flow velocity aligned with force) \n"); + printf(" - Fo = density*D32^3*(density*force) / (viscosity^2) \n"); + printf(" - Re = density*D32*velocity / viscosity \n"); + printf(" - Fo = a*Re + b*Re^2 \n"); + */ + // ************************************************************************* + + } + else { + //***************************************** - // MPI ranks for all 18 neighbors - //********************************** - int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z; - int rank_xy,rank_XY,rank_xY,rank_Xy; - int rank_xz,rank_XZ,rank_xZ,rank_Xz; - int rank_yz,rank_YZ,rank_yZ,rank_Yz; - //********************************** - MPI_Request req1[18],req2[18]; - MPI_Status stat1[18],stat2[18]; + // ***** MPI STUFF **************** + //***************************************** + // Initialize MPI + int rank,nprocs; + MPI_Init(&argc,&argv); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); + { + // parallel domain size (# of sub-domains) + int nprocx,nprocy,nprocz; + int iproc,jproc,kproc; + //***************************************** + // MPI ranks for all 18 neighbors + //********************************** + int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z; + int rank_xy,rank_XY,rank_xY,rank_Xy; + int rank_xz,rank_XZ,rank_xZ,rank_Xz; + int rank_yz,rank_YZ,rank_yZ,rank_Yz; + //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; - double REYNOLDS_NUMBER = 100.f; - if (argc > 1){ - REYNOLDS_NUMBER=strtod(argv[1],NULL); - } - if (rank == 0){ - printf("********************************************************\n"); - printf("Simulating Single Phase Non-Darcy Curve, Re < %f \n",REYNOLDS_NUMBER); - printf("********************************************************\n"); - } + double REYNOLDS_NUMBER = 100.f; + if (argc > 1){ + REYNOLDS_NUMBER=strtod(argv[1],NULL); + } + if (rank == 0){ + printf("********************************************************\n"); + printf("Simulating Single Phase Non-Darcy Curve, Re < %f \n",REYNOLDS_NUMBER); + printf("********************************************************\n"); + } - // Variables that specify the computational domain - string FILENAME; - int Nx,Ny,Nz; // local sub-domain size - int nspheres; // number of spheres in the packing - double Lx,Ly,Lz; // Domain length - double D = 1.0; // reference length for non-dimensionalization - // Color Model parameters - int timestepMax, interval; - double tau,Fx,Fy,Fz,tol,err; - double din,dout; - bool pBC,Restart; - int i,j,k,n; + // Variables that specify the computational domain + string FILENAME; + int Nx,Ny,Nz; // local sub-domain size + int nspheres; // number of spheres in the packing + double Lx,Ly,Lz; // Domain length + double D = 1.0; // reference length for non-dimensionalization + // Color Model parameters + int timestepMax, interval; + double tau,Fx,Fy,Fz,tol,err; + double din,dout; + bool pBC,Restart; + int i,j,k,n; - int RESTART_INTERVAL=20000; + int RESTART_INTERVAL=20000; - if (rank==0){ + if (rank==0){ + + //....................................................................... + // Reading the domain information file + //....................................................................... + ifstream domain("Domain.in"); + domain >> nprocx; + domain >> nprocy; + domain >> nprocz; + domain >> Nx; + domain >> Ny; + domain >> Nz; + domain >> nspheres; + domain >> Lx; + domain >> Ly; + domain >> Lz; + //....................................................................... + + /* + * Set simulation parameters internally + */ + tau=1.f; + Fx = 0.f; + Fy = 0.f; + Fz = 1.0e-7; + pBC = 0; + din = 1.0; + dout = 1.0; + timestepMax = nprocz*Nz*100; + interval = 500; + tol = 1.0e-4; + + + } + // ************************************************************** + // Broadcast simulation parameters from rank 0 to all other procs + MPI_Barrier(comm); + //................................................. + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + //MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + // MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + // Computational domain + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); + //................................................. + MPI_Barrier(comm); + + RESTART_INTERVAL=interval; + // ************************************************************** + // ************************************************************** + double rlxA = 1.f/tau; + double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA); + + + if (nprocs != nprocx*nprocy*nprocz){ + printf("nprocx = %i \n",nprocx); + printf("nprocy = %i \n",nprocy); + printf("nprocz = %i \n",nprocz); + INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); + } + + if (rank==0){ + printf("********************************************************\n"); + printf("tau = %f \n", tau); + printf("Force(x) = %f \n", Fx); + printf("Force(y) = %f \n", Fy); + printf("Force(z) = %f \n", Fz); + printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); + printf("Process grid = %i x %i x %i\n",nprocx,nprocy,nprocz); + printf("********************************************************\n"); + } + + double viscosity=(tau-0.5)/3.0; + // Initialized domain and averaging framework for Two-Phase Flow + int BC=pBC; + Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); + TwoPhase Averages(Dm); + + InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, + rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, + rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, + rank_yz, rank_YZ, rank_yZ, rank_Yz ); + + MPI_Barrier(comm); + + Nx += 2; Ny += 2; Nz += 2; + + int N = Nx*Ny*Nz; + int dist_mem_size = N*sizeof(double); //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; + if (rank == 0) printf("Read input media... \n"); //....................................................................... - /* - * Set simulation parameters internally - */ - tau=1.f; - Fx = 0.f; - Fy = 0.f; - Fz = 1.0e-7; - pBC = 0; - din = 1.0; - dout = 1.0; - timestepMax = nprocz*Nz*100; - interval = 500; - tol = 1.0e-4; + //....................................................................... + // Filenames used + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + char tmpstr[10]; + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + // printf("Local File Name = %s \n",LocalRankFilename); + // .......... READ THE INPUT FILE ....................................... + // char value; + char *id; + id = new char[N]; + int sum = 0; + double sum_local; + double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); + if (pBC) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); + double porosity, pore_vol; + //........................................................................... + if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); - //MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); - // MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); - MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); - MPI_Bcast(×tepMax,1,MPI_INT,0,comm); - MPI_Bcast(&interval,1,MPI_INT,0,comm); - MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); - // Computational domain - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); + //....................................................................... + sprintf(LocalRankString,"%05d",rank); + // sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + // WriteLocalSolidID(LocalRankFilename, id, N); + sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); + ReadBinaryFile(LocalRankFilename, Averages.SDs.data(), N); + MPI_Barrier(comm); + if (rank == 0) cout << "Domain set." << endl; - RESTART_INTERVAL=interval; - // ************************************************************** - // ************************************************************** - double rlxA = 1.f/tau; - double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA); - - - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } - - if (rank==0){ - printf("********************************************************\n"); - printf("tau = %f \n", tau); - printf("Force(x) = %f \n", Fx); - printf("Force(y) = %f \n", Fy); - printf("Force(z) = %f \n", Fz); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("Process grid = %i x %i x %i\n",nprocx,nprocy,nprocz); - printf("********************************************************\n"); - } - - double viscosity=(tau-0.5)/3.0; - // Initialized domain and averaging framework for Two-Phase Flow - int BC=pBC; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); - TwoPhase Averages(Dm); - - InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, - rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, - rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, - rank_yz, rank_YZ, rank_yZ, rank_Yz ); - - MPI_Barrier(comm); - - Nx += 2; Ny += 2; Nz += 2; - - int N = Nx*Ny*Nz; - int dist_mem_size = N*sizeof(double); - - //....................................................................... - if (rank == 0) printf("Read input media... \n"); - //....................................................................... - - //....................................................................... - // Filenames used - char LocalRankString[8]; - char LocalRankFilename[40]; - char LocalRestartFile[40]; - char tmpstr[10]; - sprintf(LocalRankString,"%05d",rank); - sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); - sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - - // printf("Local File Name = %s \n",LocalRankFilename); - // .......... READ THE INPUT FILE ....................................... - // char value; - char *id; - id = new char[N]; - int sum = 0; - double sum_local; - double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (pBC) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); - double porosity, pore_vol; - //........................................................................... - if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; - - //....................................................................... - sprintf(LocalRankString,"%05d",rank); - // sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); - // WriteLocalSolidID(LocalRankFilename, id, N); - sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); - ReadBinaryFile(LocalRankFilename, Averages.SDs.data(), N); - MPI_Barrier(comm); - if (rank == 0) cout << "Domain set." << endl; - - //....................................................................... - // Assign the phase ID field based on the signed distance - //....................................................................... - for (k=0;k 0.0){ - id[n] = 2; - } - // compute the porosity (actual interface location used) - if (Averages.SDs(n) > 0.0){ - sum++; - } - } - } - } - - // Set up kstart, kfinish so that the reservoirs are excluded from averaging - int kstart,kfinish; - kstart = 1; - kfinish = Nz-1; - if (pBC && kproc==0) kstart = 4; - if (pBC && kproc==nprocz-1) kfinish = Nz-4; - - // Compute the pore volume - sum_local = 0.0; - for ( k=kstart;k 0){ - sum_local += 1.0; - } - } - } - } - MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); - // MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); - porosity = pore_vol*iVol_global; - if (rank==0) printf("Media porosity = %f \n",porosity); - //......................................................... - // If pressure boundary conditions are applied remove solid - if (pBC && kproc == 0){ - for (k=0; k<3; k++){ + //....................................................................... + // Assign the phase ID field based on the signed distance + //....................................................................... + for (k=0;k 0.0){ + id[n] = 2; + } + // compute the porosity (actual interface location used) + if (Averages.SDs(n) > 0.0){ + sum++; + } + } + } + } + + // Set up kstart, kfinish so that the reservoirs are excluded from averaging + int kstart,kfinish; + kstart = 1; + kfinish = Nz-1; + if (pBC && kproc==0) kstart = 4; + if (pBC && kproc==nprocz-1) kfinish = Nz-4; + + // Compute the pore volume + sum_local = 0.0; + for ( k=kstart;k 0){ + sum_local += 1.0; + } + } + } + } + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); + // MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); + porosity = pore_vol*iVol_global; + if (rank==0) printf("Media porosity = %f \n",porosity); + //......................................................... + // If pressure boundary conditions are applied remove solid + if (pBC && kproc == 0){ + for (k=0; k<3; k++){ + for (j=0;j tol ){ - Re = 0.f; - // Generate a bunch of points until sufficiently high Re is obtained - while (Re < REYNOLDS_NUMBER){ - // Increase the external force and simulate to steady state - Fz = 2.0*Fz; + //************************************************************************* + // Fused Color Gradient and Collision + //************************************************************************* + MRT( ID,f_even,f_odd,rlxA,rlxB,Fx,Fy,Fz,Nx,Ny,Nz); + //************************************************************************* - err = vel_prev = 1.0; - if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol); - //************ MAIN ITERATION LOOP ***************************************/ - while (timestep < timestepMax && err > tol ){ + //************************************************************************* + // Pack and send the D3Q19 distributions + ScaLBL_Comm.SendD3Q19(f_even, f_odd); + //************************************************************************* + // Swap the distributions for momentum transport + //************************************************************************* + SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + //************************************************************************* + // Wait for communications to complete and unpack the distributions + ScaLBL_Comm.RecvD3Q19(f_even, f_odd); + //************************************************************************* - //************************************************************************* - // Fused Color Gradient and Collision - //************************************************************************* - MRT( ID,f_even,f_odd,rlxA,rlxB,Fx,Fy,Fz,Nx,Ny,Nz); - //************************************************************************* + if (pBC && kproc == 0) { + PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); + } - //************************************************************************* - // Pack and send the D3Q19 distributions - ScaLBL_Comm.SendD3Q19(f_even, f_odd); - //************************************************************************* - // Swap the distributions for momentum transport - //************************************************************************* - SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); - //************************************************************************* - // Wait for communications to complete and unpack the distributions - ScaLBL_Comm.RecvD3Q19(f_even, f_odd); - //************************************************************************* - - if (pBC && kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - } - - if (pBC && kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - } - //................................................................................... - DeviceBarrier(); - MPI_Barrier(comm); - - // Timestep completed! - timestep++; - - - if (rank==0){ - // write out csv file - printf("D32 Fx Fy Fz vx vy vz err1d Fo Re K err\n"); - } - - if (timestep%500 == 0){ - //........................................................................... - // Copy the data for for the analysis timestep - //........................................................................... - // Copy the phase from the GPU -> CPU - //........................................................................... + if (pBC && kproc == nprocz-1){ + PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + } + //................................................................................... DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - ComputeVelocityD3Q19(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); - CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); - CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); - CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); + MPI_Barrier(comm); - // Way more work than necessary -- this is just to get the solid interfacial area!! - Averages.Initialize(); - Averages.UpdateMeshValues(); - Averages.ComputeLocal(); - Averages.Reduce(); + // Timestep completed! + timestep++; - vawx = -Averages.vaw_global(0); - vawy = -Averages.vaw_global(1); - vawz = -Averages.vaw_global(2); - - // Compute local measures - err = Re; // previous Reynolds number - D32 = 6.0*(Dm.Volume-Averages.vol_w_global)/Averages.As_global; - mag_force = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - Fo = D32*D32*D32*mag_force/viscosity/viscosity; - // .... 1-D flow should be aligned with force ... - velocity = vawx*Fx/mag_force + vawy*Fy/mag_force + vawz*Fz/mag_force; - err1D = fabs(velocity-sqrt(vawx*vawx+vawy*vawy+vawz*vawz))/velocity; - //.......... Computation of the Reynolds number Re .............. - Re = D32*velocity/viscosity; - err = fabs(Re-err); if (rank==0){ - // ************* DIMENSIONLESS FORCHEIMER EQUATION ************************* - // Dye, A.L., McClure, J.E., Gray, W.G. and C.T. Miller - // Description of Non-Darcy Flows in Porous Medium Systems - // Physical Review E 87 (3), 033012 - // Fo := density*D32^3*(density*force) / (viscosity^2) - // Re := density*D32*velocity / viscosity - // Fo = a*Re + b*Re^2 - // ************************************************************************* - printf("%f ",D32); - printf("%.5g,%.5g,%.5g ",Fx,Fy,Fz); - printf("%.5g,%.5g,%.5g ",vawx,vawy,vawz); - printf("%.5g ",err1D); - printf("%5g ", Fo); - printf("%.5g ", Re); - printf("%.5g ", Re/Fo); - printf("%.5g\n", err); + // write out csv file + printf("D32 Fx Fy Fz vx vy vz err1d Fo Re K err\n"); + } + + if (timestep%500 == 0){ + //........................................................................... + // Copy the data for for the analysis timestep + //........................................................................... + // Copy the phase from the GPU -> CPU + //........................................................................... + DeviceBarrier(); + ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ComputeVelocityD3Q19(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); + CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); + CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); + CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); + CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); + + // Way more work than necessary -- this is just to get the solid interfacial area!! + Averages.Initialize(); + Averages.UpdateMeshValues(); + Averages.ComputeLocal(); + Averages.Reduce(); + + vawx = -Averages.vaw_global(0); + vawy = -Averages.vaw_global(1); + vawz = -Averages.vaw_global(2); + + // Compute local measures + err = Re; // previous Reynolds number + D32 = 6.0*(Dm.Volume-Averages.vol_w_global)/Averages.As_global; + mag_force = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + Fo = D32*D32*D32*mag_force/viscosity/viscosity; + // .... 1-D flow should be aligned with force ... + velocity = vawx*Fx/mag_force + vawy*Fy/mag_force + vawz*Fz/mag_force; + err1D = fabs(velocity-sqrt(vawx*vawx+vawy*vawy+vawz*vawz))/velocity; + //.......... Computation of the Reynolds number Re .............. + Re = D32*velocity/viscosity; + err = fabs(Re-err); + + if (rank==0){ + // ************* DIMENSIONLESS FORCHEIMER EQUATION ************************* + // Dye, A.L., McClure, J.E., Gray, W.G. and C.T. Miller + // Description of Non-Darcy Flows in Porous Medium Systems + // Physical Review E 87 (3), 033012 + // Fo := density*D32^3*(density*force) / (viscosity^2) + // Re := density*D32*velocity / viscosity + // Fo = a*Re + b*Re^2 + // ************************************************************************* + printf("%f ",D32); + printf("%.5g,%.5g,%.5g ",Fx,Fy,Fz); + printf("%.5g,%.5g,%.5g ",vawx,vawy,vawz); + printf("%.5g ",err1D); + printf("%5g ", Fo); + printf("%.5g ", Re); + printf("%.5g ", Re/Fo); + printf("%.5g\n", err); + } } } - } - // Write steady state variables to csv file - if (rank==0){ - fprintf(NONDARCY,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",D32,Fx,Fy,Fz,vawx,vawy,vawz,Re,Fo); - fflush(NONDARCY); + // Write steady state variables to csv file + if (rank==0){ + fprintf(NONDARCY,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",D32,Fx,Fy,Fz,vawx,vawy,vawz,Re,Fo); + fflush(NONDARCY); + } } + //************************************************************************/ + fclose(NONDARCY); + DeviceBarrier(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Nx*Ny*Nz)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + + NULL_USE(RESTART_INTERVAL); } - //************************************************************************/ - fclose(NONDARCY); - DeviceBarrier(); + // **************************************************** MPI_Barrier(comm); - stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Nx*Ny*Nz)/cputime/1000000; - - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); - - NULL_USE(RESTART_INTERVAL); + MPI_Finalize(); + // **************************************************** } - // **************************************************** - MPI_Barrier(comm); - MPI_Finalize(); - // **************************************************** } From b22a1b21202932df02652b1b57accfe92d928e9c Mon Sep 17 00:00:00 2001 From: zherexli Date: Tue, 22 Nov 2016 17:27:21 -0500 Subject: [PATCH 2/7] add a TODO list for all python preprocess/postprocess scripts --- .../Morphological_Analysis/morphological_analysis_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflows/Morphological_Analysis/morphological_analysis_utils.py b/workflows/Morphological_Analysis/morphological_analysis_utils.py index 9869bb92..8b0eb5e3 100644 --- a/workflows/Morphological_Analysis/morphological_analysis_utils.py +++ b/workflows/Morphological_Analysis/morphological_analysis_utils.py @@ -1,5 +1,7 @@ #!/usr/bin/env python - +# TODO: double check the issue of the "view of the array" for all the following functions +# make sure it is the copy of the array, not the view of the array is used for any +# subprocesses. import numpy as np import scipy.stats as stats import scipy.ndimage.morphology as morphology From b7a5ea342f5522c6dcd70638398195dc183498fb Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 23 Nov 2016 16:16:48 -0500 Subject: [PATCH 3/7] replacing ScaLBL functions in tests/*cpp --- common/ScaLBL.h | 584 ++++++++++---------- tests/Blob.cpp | 115 ++++ tests/TestBubble.cpp | 728 ++++++++++++------------- tests/TestCommD3Q19.cpp | 28 +- tests/TestDistanceFromSegmented.cpp | 4 + tests/TestMassConservationD3Q7.cpp | 54 +- tests/lb2_CMT_wia.cpp | 94 ++-- tests/lb2_Color_blob_wia_mpi.cpp | 748 +++++++++++++------------- tests/lbpm_color_macro_simulator.cpp | 198 +++---- tests/lbpm_color_simulator.cpp | 170 +++--- tests/lbpm_permeability_simulator.cpp | 48 +- 11 files changed, 1442 insertions(+), 1329 deletions(-) create mode 100644 tests/Blob.cpp create mode 100644 tests/TestDistanceFromSegmented.cpp diff --git a/common/ScaLBL.h b/common/ScaLBL.h index c32e0de1..896c8335 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -9,80 +9,74 @@ */ #include "Domain.h" -extern "C" void AllocateDeviceMemory(void** address, size_t size); +extern "C" void ScaLBL_AllocateDeviceMemory(void** address, size_t size); //extern "C" void FreeDeviceMemory(void** address); -extern "C" void CopyToDevice(void* dest, const void* source, size_t size); +extern "C" void ScaLBL_CopyToDevice(void* dest, const void* source, size_t size); -extern "C" void CopyToHost(void* dest, const void* source, size_t size); +extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size); -extern "C" void DeviceBarrier(); +extern "C" void ScaLBL_DeviceBarrier(); -extern "C" void PackDist(int q, int *list, int start, int count, double *sendbuf, double *dist, int N); +extern "C" void ScaLBL_D3Q19_Pack(int q, int *list, int start, int count, double *sendbuf, double *dist, int N); -extern "C" void UnpackDist(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count,double *recvbuf, double *dist, int Nx, int Ny, int Nz); +extern "C" void ScaLBL_D3Q19_Unpack(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count,double *recvbuf, double *dist, int Nx, int Ny, int Nz); -extern "C" void PackValues(int *list, int count, double *sendbuf, double *Data, int N); +extern "C" void ScaLBL_Scalar_Pack(int *list, int count, double *sendbuf, double *Data, int N); -extern "C" void UnpackValues(int *list, int count, double *recvbuf, double *Data, int N); +extern "C" void ScaLBL_Scalar_Unpack(int *list, int count, double *recvbuf, double *Data, int N); -extern "C" void PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N); +extern "C" void ScaLBL_PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N); -extern "C" void UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N); +extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N); -extern "C" void InitD3Q7(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz); +extern "C" void ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz); -extern "C" void SwapD3Q7(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz); +extern "C" void ScaLBL_D3Q7_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz); -extern "C" void ComputeDensityD3Q7(char *ID, double *disteven, double *distodd, double *Den, +extern "C" void ScaLBL_D3Q7_Density(char *ID, double *disteven, double *distodd, double *Den, int Nx, int Ny, int Nz); -extern "C" void InitD3Q19(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz); +extern "C" void ScaLBL_D3Q19_Init(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz); -extern "C" void SwapD3Q19(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz); +extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz); -extern "C" void MRT(char *ID, double *f_even, double *f_odd, double rlxA, double rlxB, +extern "C" void ScaLBL_D3Q19_MRT(char *ID, double *f_even, double *f_odd, double rlxA, double rlxB, double Fx, double Fy, double Fz,int Nx, int Ny, int Nz); -extern "C" void ComputeVelocityD3Q19(char *ID, double *disteven, double *distodd, double *vel, +extern "C" void ScaLBL_D3Q19_Velocity(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz); -extern "C" void ComputePressureD3Q19(const char *ID, const double *disteven, const double *distodd, +extern "C" void ScaLBL_D3Q19_Pressure(const char *ID, const double *disteven, const double *distodd, double *Pressure, int Nx, int Ny, int Nz); -extern "C" void PressureBC_inlet(double *disteven, double *distodd, double din, +extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din, int Nx, int Ny, int Nz); -extern "C" void PressureBC_outlet(double *disteven, double *distodd, double dout, +extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout, int Nx, int Ny, int Nz, int outlet); -extern "C" void InitDenColor(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz); +extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz); -extern "C" void InitDenColorDistance(char *ID, double *Den, double *Phi, double *Distance, +extern "C" void ScaLBL_ColorDistance_Init(char *ID, double *Den, double *Phi, double *Distance, double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz); -extern "C" void ComputeColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz); +extern "C" void ScaLBL_D3Q19_ColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz); -extern "C" void ColorCollide( char *ID, double *disteven, double *distodd, double *ColorGrad, - double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB, - double alpha, double beta, double Fx, double Fy, double Fz, bool pBC); - -extern "C" void ColorCollideOpt( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, +extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB, double alpha, double beta, double Fx, double Fy, double Fz); -extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, - double beta, int Nx, int Ny, int Nz, bool pBC); +extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, + double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC); -extern "C" void ComputePhi(char *ID, double *Phi, double *Den, int N); -extern "C" void MassColorCollideD3Q7(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, - double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC); +extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N); -extern "C" void ColorBC_inlet(double *Phi, double *Den, double *A_even, double *A_odd, +extern "C" void ScaLBL_Color_BC_z(double *Phi, double *Den, double *A_even, double *A_odd, double *B_even, double *B_odd, int Nx, int Ny, int Nz); -extern "C" void ColorBC_outlet(double *Phi, double *Den, double *A_even, double *A_odd, +extern "C" void ScaLBL_Color_BC_Z(double *Phi, double *Den, double *A_even, double *A_odd, double *B_even, double *B_odd, int Nx, int Ny, int Nz); extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz, @@ -91,7 +85,7 @@ extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, do extern "C" void ScaLBL_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, double uz, int Nx, int Ny, int Nz, int outlet); -extern "C" void SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); +extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); class ScaLBL_Communicator{ public: @@ -234,122 +228,122 @@ ScaLBL_Communicator::ScaLBL_Communicator(Domain &Dm){ recvCount_YZ=Dm.recvCount_YZ; recvCount_XZ=Dm.recvCount_XZ; //...................................................................................... - AllocateDeviceMemory((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_X, 5*sendCount_X*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_y, 5*sendCount_y*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_z, 5*sendCount_z*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double)); // Allocatevoid * memory - AllocateDeviceMemory((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_X, 5*sendCount_X*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_y, 5*sendCount_y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_z, 5*sendCount_z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double)); // Allocatevoid * memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory //...................................................................................... - AllocateDeviceMemory((void **) &recvbuf_x, 5*recvCount_x*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_X, 5*recvCount_X*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_y, 5*recvCount_y*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_z, 5*recvCount_z*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory - AllocateDeviceMemory((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_x, 5*recvCount_x*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_X, 5*recvCount_X*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_y, 5*recvCount_y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_z, 5*recvCount_z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory //...................................................................................... - AllocateDeviceMemory((void **) &dvcSendList_x, sendCount_x*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_X, sendCount_X*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_y, sendCount_y*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_Y, sendCount_Y*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_z, sendCount_z*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_Z, sendCount_Z*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_xy, sendCount_xy*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_xY, sendCount_xY*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_Xy, sendCount_Xy*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_XY, sendCount_XY*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_xz, sendCount_xz*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_xZ, sendCount_xZ*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_Xz, sendCount_Xz*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_XZ, sendCount_XZ*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_yz, sendCount_yz*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_yZ, sendCount_yZ*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_Yz, sendCount_Yz*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcSendList_YZ, sendCount_YZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_x, sendCount_x*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_X, sendCount_X*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_y, sendCount_y*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Y, sendCount_Y*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_z, sendCount_z*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Z, sendCount_Z*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xy, sendCount_xy*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xY, sendCount_xY*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Xy, sendCount_Xy*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_XY, sendCount_XY*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xz, sendCount_xz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xZ, sendCount_xZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Xz, sendCount_Xz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_XZ, sendCount_XZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_yz, sendCount_yz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_yZ, sendCount_yZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Yz, sendCount_Yz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_YZ, sendCount_YZ*sizeof(int)); // Allocate device memory //...................................................................................... - AllocateDeviceMemory((void **) &dvcRecvList_x, recvCount_x*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_X, recvCount_X*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_y, recvCount_y*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_Y, recvCount_Y*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_z, recvCount_z*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_Z, recvCount_Z*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_xy, recvCount_xy*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_xY, recvCount_xY*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_Xy, recvCount_Xy*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_XY, recvCount_XY*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_xz, recvCount_xz*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_xZ, recvCount_xZ*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_Xz, recvCount_Xz*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_XZ, recvCount_XZ*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_yz, recvCount_yz*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_yZ, recvCount_yZ*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory - AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_x, recvCount_x*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_X, recvCount_X*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_y, recvCount_y*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Y, recvCount_Y*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_z, recvCount_z*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Z, recvCount_Z*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xy, recvCount_xy*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xY, recvCount_xY*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Xy, recvCount_Xy*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_XY, recvCount_XY*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xz, recvCount_xz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xZ, recvCount_xZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Xz, recvCount_Xz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_XZ, recvCount_XZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_yz, recvCount_yz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_yZ, recvCount_yZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory //...................................................................................... - CopyToDevice(dvcSendList_x,Dm.sendList_x,sendCount_x*sizeof(int)); - CopyToDevice(dvcSendList_X,Dm.sendList_X,sendCount_X*sizeof(int)); - CopyToDevice(dvcSendList_y,Dm.sendList_y,sendCount_y*sizeof(int)); - CopyToDevice(dvcSendList_Y,Dm.sendList_Y,sendCount_Y*sizeof(int)); - CopyToDevice(dvcSendList_z,Dm.sendList_z,sendCount_z*sizeof(int)); - CopyToDevice(dvcSendList_Z,Dm.sendList_Z,sendCount_Z*sizeof(int)); - CopyToDevice(dvcSendList_xy,Dm.sendList_xy,sendCount_xy*sizeof(int)); - CopyToDevice(dvcSendList_XY,Dm.sendList_XY,sendCount_XY*sizeof(int)); - CopyToDevice(dvcSendList_xY,Dm.sendList_xY,sendCount_xY*sizeof(int)); - CopyToDevice(dvcSendList_Xy,Dm.sendList_Xy,sendCount_Xy*sizeof(int)); - CopyToDevice(dvcSendList_xz,Dm.sendList_xz,sendCount_xz*sizeof(int)); - CopyToDevice(dvcSendList_XZ,Dm.sendList_XZ,sendCount_XZ*sizeof(int)); - CopyToDevice(dvcSendList_xZ,Dm.sendList_xZ,sendCount_xZ*sizeof(int)); - CopyToDevice(dvcSendList_Xz,Dm.sendList_Xz,sendCount_Xz*sizeof(int)); - CopyToDevice(dvcSendList_yz,Dm.sendList_yz,sendCount_yz*sizeof(int)); - CopyToDevice(dvcSendList_YZ,Dm.sendList_YZ,sendCount_YZ*sizeof(int)); - CopyToDevice(dvcSendList_yZ,Dm.sendList_yZ,sendCount_yZ*sizeof(int)); - CopyToDevice(dvcSendList_Yz,Dm.sendList_Yz,sendCount_Yz*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_x,Dm.sendList_x,sendCount_x*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_X,Dm.sendList_X,sendCount_X*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_y,Dm.sendList_y,sendCount_y*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_Y,Dm.sendList_Y,sendCount_Y*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_z,Dm.sendList_z,sendCount_z*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_Z,Dm.sendList_Z,sendCount_Z*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_xy,Dm.sendList_xy,sendCount_xy*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_XY,Dm.sendList_XY,sendCount_XY*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_xY,Dm.sendList_xY,sendCount_xY*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_Xy,Dm.sendList_Xy,sendCount_Xy*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_xz,Dm.sendList_xz,sendCount_xz*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_XZ,Dm.sendList_XZ,sendCount_XZ*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_xZ,Dm.sendList_xZ,sendCount_xZ*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_Xz,Dm.sendList_Xz,sendCount_Xz*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_yz,Dm.sendList_yz,sendCount_yz*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_YZ,Dm.sendList_YZ,sendCount_YZ*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_yZ,Dm.sendList_yZ,sendCount_yZ*sizeof(int)); + ScaLBL_CopyToDevice(dvcSendList_Yz,Dm.sendList_Yz,sendCount_Yz*sizeof(int)); //...................................................................................... - CopyToDevice(dvcRecvList_x,Dm.recvList_x,recvCount_x*sizeof(int)); - CopyToDevice(dvcRecvList_X,Dm.recvList_X,recvCount_X*sizeof(int)); - CopyToDevice(dvcRecvList_y,Dm.recvList_y,recvCount_y*sizeof(int)); - CopyToDevice(dvcRecvList_Y,Dm.recvList_Y,recvCount_Y*sizeof(int)); - CopyToDevice(dvcRecvList_z,Dm.recvList_z,recvCount_z*sizeof(int)); - CopyToDevice(dvcRecvList_Z,Dm.recvList_Z,recvCount_Z*sizeof(int)); - CopyToDevice(dvcRecvList_xy,Dm.recvList_xy,recvCount_xy*sizeof(int)); - CopyToDevice(dvcRecvList_XY,Dm.recvList_XY,recvCount_XY*sizeof(int)); - CopyToDevice(dvcRecvList_xY,Dm.recvList_xY,recvCount_xY*sizeof(int)); - CopyToDevice(dvcRecvList_Xy,Dm.recvList_Xy,recvCount_Xy*sizeof(int)); - CopyToDevice(dvcRecvList_xz,Dm.recvList_xz,recvCount_xz*sizeof(int)); - CopyToDevice(dvcRecvList_XZ,Dm.recvList_XZ,recvCount_XZ*sizeof(int)); - CopyToDevice(dvcRecvList_xZ,Dm.recvList_xZ,recvCount_xZ*sizeof(int)); - CopyToDevice(dvcRecvList_Xz,Dm.recvList_Xz,recvCount_Xz*sizeof(int)); - CopyToDevice(dvcRecvList_yz,Dm.recvList_yz,recvCount_yz*sizeof(int)); - CopyToDevice(dvcRecvList_YZ,Dm.recvList_YZ,recvCount_YZ*sizeof(int)); - CopyToDevice(dvcRecvList_yZ,Dm.recvList_yZ,recvCount_yZ*sizeof(int)); - CopyToDevice(dvcRecvList_Yz,Dm.recvList_Yz,recvCount_Yz*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_x,Dm.recvList_x,recvCount_x*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_X,Dm.recvList_X,recvCount_X*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_y,Dm.recvList_y,recvCount_y*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_Y,Dm.recvList_Y,recvCount_Y*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_z,Dm.recvList_z,recvCount_z*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_Z,Dm.recvList_Z,recvCount_Z*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_xy,Dm.recvList_xy,recvCount_xy*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_XY,Dm.recvList_XY,recvCount_XY*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_xY,Dm.recvList_xY,recvCount_xY*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_Xy,Dm.recvList_Xy,recvCount_Xy*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_xz,Dm.recvList_xz,recvCount_xz*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_XZ,Dm.recvList_XZ,recvCount_XZ*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_xZ,Dm.recvList_xZ,recvCount_xZ*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_Xz,Dm.recvList_Xz,recvCount_Xz*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_yz,Dm.recvList_yz,recvCount_yz*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_YZ,Dm.recvList_YZ,recvCount_YZ*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_yZ,Dm.recvList_yZ,recvCount_yZ*sizeof(int)); + ScaLBL_CopyToDevice(dvcRecvList_Yz,Dm.recvList_Yz,recvCount_Yz*sizeof(int)); //...................................................................................... MPI_Barrier(MPI_COMM_SCALBL); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //...................................................................................... SendCount = sendCount_x+sendCount_X+sendCount_y+sendCount_Y+sendCount_z+sendCount_Z+ sendCount_xy+sendCount_Xy+sendCount_xY+sendCount_XY+ @@ -381,68 +375,68 @@ void ScaLBL_Communicator::SendD3Q19(double *f_even, double *f_odd){ } // assign tag of 19 to D3Q19 communication sendtag = recvtag = 19; - DeviceBarrier(); + ScaLBL_DeviceBarrier(); // Pack the distributions - PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N); - PackDist(4,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(5,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(6,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(7,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N); //...Packing for X face(1,7,9,11,13)................................ - PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N); - PackDist(3,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(4,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(5,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(6,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); //...Packing for y face(4,8,9,16,18)................................. - PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,f_even,N); - PackDist(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_even,N); - PackDist(4,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); - PackDist(8,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_even,N); - PackDist(9,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,0,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_even,N); //...Packing for Y face(3,7,10,15,17)................................. - PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(5,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); - PackDist(7,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(8,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); //...Packing for z face(6,12,13,16,17)................................ - PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,f_even,N); - PackDist(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_even,N); - PackDist(6,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); - PackDist(8,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_even,N); - PackDist(8,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,0,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); //...Packing for Z face(5,11,14,15,18)................................ - PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(5,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(7,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); - PackDist(7,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(9,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); //...Pack the xy edge (8)................................ - PackDist(4,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_even,N); //...Pack the Xy edge (9)................................ - PackDist(4,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_odd,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_odd,N); //...Pack the xY edge (10)................................ - PackDist(5,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_even,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_even,N); //...Pack the XY edge (7)................................ - PackDist(3,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_odd,N); //...Pack the xz edge (12)................................ - PackDist(6,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_even,N); //...Pack the xZ edge (14)................................ - PackDist(7,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_even,N); //...Pack the Xz edge (13)................................ - PackDist(6,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_odd,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_odd,N); //...Pack the XZ edge (11)................................ - PackDist(5,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_odd,N); //...Pack the xz edge (12)................................ //...Pack the yz edge (16)................................ - PackDist(8,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_even,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_even,N); //...Pack the yZ edge (18)................................ - PackDist(9,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_even,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_even,N); //...Pack the Yz edge (17)................................ - PackDist(8,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_odd,N); //...Pack the YZ edge (15)................................ - PackDist(7,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_odd,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_odd,N); //................................................................................... //................................................................................... @@ -490,76 +484,76 @@ void ScaLBL_Communicator::RecvD3Q19(double *f_even, double *f_odd){ // Wait for completion of D3Q19 communication MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //................................................................................... // Unpack the distributions on the device //................................................................................... //...Map recieve list for the X face: q=2,8,10,12,13 ................................. - UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(3,-1,-1,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(4,-1,1,0,dvcRecvList_X,2*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(5,-1,0,-1,dvcRecvList_X,3*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(6,-1,0,1,dvcRecvList_X,4*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_X,2*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_X,3*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_X,4*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the x face: q=1,7,9,11,13.................................. - UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(4,1,1,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(5,1,-1,0,dvcRecvList_x,2*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(6,1,0,1,dvcRecvList_x,3*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(7,1,0,-1,dvcRecvList_x,4*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_x,2*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_x,3*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_x,4*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the y face: q=4,8,9,16,18 ................................... - UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(3,-1,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(5,1,-1,0,dvcRecvList_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,f_even,Nx,Ny,Nz); - UnpackDist(7,0,-1,-1,dvcRecvList_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,-1,1,dvcRecvList_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the Y face: q=3,7,10,15,17 .................................. - UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(4,1,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(4,-1,1,0,dvcRecvList_y,2*recvCount_y,recvCount_y,recvbuf_y,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,1,1,dvcRecvList_y,3*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(9,0,1,-1,dvcRecvList_y,4*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_y,2*recvCount_y,recvCount_y,recvbuf_y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_y,3*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_y,4*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the z face<<<6,12,13,16,17).............................................. - UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(5,-1,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(7,1,0,-1,dvcRecvList_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); - UnpackDist(7,0,-1,-1,dvcRecvList_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(9,0,1,-1,dvcRecvList_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); //...Map recieve list for the Z face<<<5,11,14,15,18).............................................. - UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(6,1,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(6,-1,0,1,dvcRecvList_z,2*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,1,1,dvcRecvList_z,3*recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(8,0,-1,1,dvcRecvList_z,4*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_z,2*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_z,3*recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_z,4*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); //.................................................................................. //...Map recieve list for the xy edge <<<8)................................ - UnpackDist(3,-1,-1,0,dvcRecvList_XY,0,recvCount_XY,recvbuf_XY,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_XY,0,recvCount_XY,recvbuf_XY,f_odd,Nx,Ny,Nz); //...Map recieve list for the Xy edge <<<9)................................ - UnpackDist(5,1,-1,0,dvcRecvList_xY,0,recvCount_xY,recvbuf_xY,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_xY,0,recvCount_xY,recvbuf_xY,f_even,Nx,Ny,Nz); //...Map recieve list for the xY edge <<<10)................................ - UnpackDist(4,-1,1,0,dvcRecvList_Xy,0,recvCount_Xy,recvbuf_Xy,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_Xy,0,recvCount_Xy,recvbuf_Xy,f_odd,Nx,Ny,Nz); //...Map recieve list for the XY edge <<<7)................................ - UnpackDist(4,1,1,0,dvcRecvList_xy,0,recvCount_xy,recvbuf_xy,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_xy,0,recvCount_xy,recvbuf_xy,f_even,Nx,Ny,Nz); //...Map recieve list for the xz edge <<<12)................................ - UnpackDist(5,-1,0,-1,dvcRecvList_XZ,0,recvCount_XZ,recvbuf_XZ,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_XZ,0,recvCount_XZ,recvbuf_XZ,f_odd,Nx,Ny,Nz); //...Map recieve list for the xZ edge <<<14)................................ - UnpackDist(6,-1,0,1,dvcRecvList_Xz,0,recvCount_Xz,recvbuf_Xz,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_Xz,0,recvCount_Xz,recvbuf_Xz,f_odd,Nx,Ny,Nz); //...Map recieve list for the Xz edge <<<13)................................ - UnpackDist(7,1,0,-1,dvcRecvList_xZ,0,recvCount_xZ,recvbuf_xZ,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_xZ,0,recvCount_xZ,recvbuf_xZ,f_even,Nx,Ny,Nz); //...Map recieve list for the XZ edge <<<11)................................ - UnpackDist(6,1,0,1,dvcRecvList_xz,0,recvCount_xz,recvbuf_xz,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_xz,0,recvCount_xz,recvbuf_xz,f_even,Nx,Ny,Nz); //...Map recieve list for the yz edge <<<16)................................ - UnpackDist(7,0,-1,-1,dvcRecvList_YZ,0,recvCount_YZ,recvbuf_YZ,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_YZ,0,recvCount_YZ,recvbuf_YZ,f_odd,Nx,Ny,Nz); //...Map recieve list for the yZ edge <<<18)................................ - UnpackDist(8,0,-1,1,dvcRecvList_Yz,0,recvCount_Yz,recvbuf_Yz,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_Yz,0,recvCount_Yz,recvbuf_Yz,f_odd,Nx,Ny,Nz); //...Map recieve list for the Yz edge <<<17)................................ - UnpackDist(9,0,1,-1,dvcRecvList_yZ,0,recvCount_yZ,recvbuf_yZ,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_yZ,0,recvCount_yZ,recvbuf_yZ,f_even,Nx,Ny,Nz); //...Map recieve list for the YZ edge <<<15)................................ - UnpackDist(8,0,1,1,dvcRecvList_yz,0,recvCount_yz,recvbuf_yz,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_yz,0,recvCount_yz,recvbuf_yz,f_even,Nx,Ny,Nz); //................................................................................... Lock=false; // unlock the communicator after communications complete //................................................................................... @@ -573,27 +567,27 @@ void ScaLBL_Communicator::BiSendD3Q7(double *A_even, double *A_odd, double *B_ev else{ Lock=true; } - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //................................................................................... sendtag = recvtag = 7; //................................................................................... - PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,A_even,N); - PackDist(1,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,B_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,0,sendCount_x,sendbuf_x,A_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,B_even,N); //...Packing for X face(1,7,9,11,13)................................ - PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,A_odd,N); - PackDist(0,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,B_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,0,sendCount_X,sendbuf_X,A_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,B_odd,N); //...Packing for y face(4,8,9,16,18)................................. - PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,A_even,N); - PackDist(2,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,B_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,0,sendCount_y,sendbuf_y,A_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,B_even,N); //...Packing for Y face(3,7,10,15,17)................................. - PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,A_odd,N); - PackDist(1,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,B_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,A_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,B_odd,N); //...Packing for z face(6,12,13,16,17)................................ - PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,A_even,N); - PackDist(3,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,B_even,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,0,sendCount_z,sendbuf_z,A_even,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,B_even,N); //...Packing for Z face(5,11,14,15,18)................................ - PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,A_odd,N); - PackDist(2,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,B_odd,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,A_odd,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,B_odd,N); //................................................................................... //................................................................................... @@ -617,32 +611,32 @@ void ScaLBL_Communicator::BiRecvD3Q7(double *A_even, double *A_odd, double *B_ev // Wait for completion of D3Q19 communication MPI_Waitall(6,req1,stat1); MPI_Waitall(6,req2,stat2); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //................................................................................... // Unpack the distributions on the device //................................................................................... //...Map recieve list for the X face: q=2,8,10,12,13 ................................ - UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,A_odd,Nx,Ny,Nz); - UnpackDist(0,-1,0,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,B_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the x face: q=1,7,9,11,13.................................. - UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,A_even,Nx,Ny,Nz); - UnpackDist(1,1,0,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,B_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the y face: q=4,8,9,16,18 ................................. - UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,A_odd,Nx,Ny,Nz); - UnpackDist(1,0,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,B_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the Y face: q=3,7,10,15,17 ................................ - UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,A_even,Nx,Ny,Nz); - UnpackDist(2,0,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,B_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the z face<<<6,12,13,16,17)................................ - UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,A_odd,Nx,Ny,Nz); - UnpackDist(2,0,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,B_odd,Nx,Ny,Nz); //...Map recieve list for the Z face<<<5,11,14,15,18)................................ - UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,A_even,Nx,Ny,Nz); - UnpackDist(3,0,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,B_even,Nx,Ny,Nz); //.................................................................................. Lock=false; // unlock the communicator after communications complete //................................................................................... @@ -655,28 +649,28 @@ void ScaLBL_Communicator::SendHalo(double *data){ else{ Lock=true; } - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //................................................................................... sendtag = recvtag = 1; //................................................................................... - PackValues(dvcSendList_x, sendCount_x,sendbuf_x, data, N); - PackValues(dvcSendList_y, sendCount_y,sendbuf_y, data, N); - PackValues(dvcSendList_z, sendCount_z,sendbuf_z, data, N); - PackValues(dvcSendList_X, sendCount_X,sendbuf_X, data, N); - PackValues(dvcSendList_Y, sendCount_Y,sendbuf_Y, data, N); - PackValues(dvcSendList_Z, sendCount_Z,sendbuf_Z, data, N); - PackValues(dvcSendList_xy, sendCount_xy,sendbuf_xy, data, N); - PackValues(dvcSendList_xY, sendCount_xY,sendbuf_xY, data, N); - PackValues(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, data, N); - PackValues(dvcSendList_XY, sendCount_XY,sendbuf_XY, data, N); - PackValues(dvcSendList_xz, sendCount_xz,sendbuf_xz, data, N); - PackValues(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, data, N); - PackValues(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, data, N); - PackValues(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, data, N); - PackValues(dvcSendList_yz, sendCount_yz,sendbuf_yz, data, N); - PackValues(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, data, N); - PackValues(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, data, N); - PackValues(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, data, N); + ScaLBL_Scalar_Pack(dvcSendList_x, sendCount_x,sendbuf_x, data, N); + ScaLBL_Scalar_Pack(dvcSendList_y, sendCount_y,sendbuf_y, data, N); + ScaLBL_Scalar_Pack(dvcSendList_z, sendCount_z,sendbuf_z, data, N); + ScaLBL_Scalar_Pack(dvcSendList_X, sendCount_X,sendbuf_X, data, N); + ScaLBL_Scalar_Pack(dvcSendList_Y, sendCount_Y,sendbuf_Y, data, N); + ScaLBL_Scalar_Pack(dvcSendList_Z, sendCount_Z,sendbuf_Z, data, N); + ScaLBL_Scalar_Pack(dvcSendList_xy, sendCount_xy,sendbuf_xy, data, N); + ScaLBL_Scalar_Pack(dvcSendList_xY, sendCount_xY,sendbuf_xY, data, N); + ScaLBL_Scalar_Pack(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, data, N); + ScaLBL_Scalar_Pack(dvcSendList_XY, sendCount_XY,sendbuf_XY, data, N); + ScaLBL_Scalar_Pack(dvcSendList_xz, sendCount_xz,sendbuf_xz, data, N); + ScaLBL_Scalar_Pack(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, data, N); + ScaLBL_Scalar_Pack(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, data, N); + ScaLBL_Scalar_Pack(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, data, N); + ScaLBL_Scalar_Pack(dvcSendList_yz, sendCount_yz,sendbuf_yz, data, N); + ScaLBL_Scalar_Pack(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, data, N); + ScaLBL_Scalar_Pack(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, data, N); + ScaLBL_Scalar_Pack(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, data, N); //................................................................................... // Send / Recv all the phase indcator field values //................................................................................... @@ -723,27 +717,27 @@ void ScaLBL_Communicator::RecvHalo(double *data){ //................................................................................... MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... - UnpackValues(dvcRecvList_x, recvCount_x,recvbuf_x, data, N); - UnpackValues(dvcRecvList_y, recvCount_y,recvbuf_y, data, N); - UnpackValues(dvcRecvList_z, recvCount_z,recvbuf_z, data, N); - UnpackValues(dvcRecvList_X, recvCount_X,recvbuf_X, data, N); - UnpackValues(dvcRecvList_Y, recvCount_Y,recvbuf_Y, data, N); - UnpackValues(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N); - UnpackValues(dvcRecvList_xy, recvCount_xy,recvbuf_xy, data, N); - UnpackValues(dvcRecvList_xY, recvCount_xY,recvbuf_xY, data, N); - UnpackValues(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, data, N); - UnpackValues(dvcRecvList_XY, recvCount_XY,recvbuf_XY, data, N); - UnpackValues(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N); - UnpackValues(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N); - UnpackValues(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N); - UnpackValues(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N); - UnpackValues(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N); - UnpackValues(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N); - UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N); - UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Y, recvCount_Y,recvbuf_Y, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xy, recvCount_xy,recvbuf_xy, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xY, recvCount_xY,recvbuf_xY, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_XY, recvCount_XY,recvbuf_XY, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N); //................................................................................... Lock=false; // unlock the communicator after communications complete //................................................................................... diff --git a/tests/Blob.cpp b/tests/Blob.cpp new file mode 100644 index 00000000..c4239636 --- /dev/null +++ b/tests/Blob.cpp @@ -0,0 +1,115 @@ +#include +#include +#include +#include + +#define MAX_LOCAL_BLOB_COUNT 500 + +using namespace std; + +struct BlobInfo{ + + BlobInfo(int size){ + Values = new double [40*size]; + Ranks = new int [19*size]; + RankLabel = new int [19*size]; + }; + double * Values; + int * Ranks; + int * RankLabel; + // local rank and blob_id + + int rank(int i){return Ranks[19*i];} + int rank_x(int i){return Ranks[19*i+1];} + int rank_y(int i){return Ranks[19*i+2];} + int rank_z(int i){return Ranks[19*i+3];} + int rank_X(int i){return Ranks[19*i+4];} + int rank_Y(int i){return Ranks[19*i+5];} + int rank_Z(int i){return Ranks[19*i+6];} + int rank_xy(int i){return Ranks[19*i+7];} + int rank_xY(int i){return Ranks[19*i+8];} + int rank_Xy(int i){return Ranks[19*i+9];} + int rank_XY(int i){return Ranks[19*i+10];} + int rank_xz(int i){return Ranks[19*i+11];} + int rank_xZ(int i){return Ranks[19*i+12];} + int rank_Xz(int i){return Ranks[19*i+13];} + int rank_XZ(int i){return Ranks[19*i+14];} + int rank_yz(int i){return Ranks[19*i+15];} + int rank_yZ(int i){return Ranks[19*i+16];} + int rank_Yz(int i){return Ranks[19*i+17];} + int rank_YZ(int i){return Ranks[19*i+18];} + + int blobID(int i){return RankLabel[19*i];} + int blobID_x(int i){return RankLabel[19*i+1];} + int blobID_y(int i){return RankLabel[19*i+2];} + int blobID_z(int i){return RankLabel[19*i+3];} + int blobID_X(int i){return RankLabel[19*i+4];} + int blobID_Y(int i){return RankLabel[19*i+5];} + int blobID_Z(int i){return RankLabel[19*i+6];} + int blobID_xy(int i){return RankLabel[19*i+7];} + int blobID_xY(int i){return RankLabel[19*i+8];} + int blobID_Xy(int i){return RankLabel[19*i+9];} + int blobID_XY(int i){return RankLabel[19*i+10];} + int blobID_xz(int i){return RankLabel[19*i+11];} + int blobID_xZ(int i){return RankLabel[19*i+12];} + int blobID_Xz(int i){return RankLabel[19*i+13];} + int blobID_XZ(int i){return RankLabel[19*i+14];} + int blobID_yz(int i){return RankLabel[19*i+15];} + int blobID_yZ(int i){return RankLabel[19*i+16];} + int blobID_Yz(int i){return RankLabel[19*i+17];} + int blobID_YZ(int i){return RankLabel[19*i+18];} + + double voln(int i){return Values[40*i];} + double pn(int i){return Values[40*i+1];} + double vnx(int i){return Values[40*i+2];} + double vny(int i){return Values[40*i+3];} + double vnz(int i){return Values[40*i+4];} + double vwnx(int i){return Values[40*i+5];} + double vwny(int i){return Values[40*i+6];} + double vwnz(int i){return Values[40*i+7];} + double Jwn(int i){return Values[40*i+8];} + double Kwn(int i){return Values[40*i+9];} + double awn(int i){return Values[40*i+10];} + double ans(int i){return Values[40*i+11];} + double Gwnxx(int i){return Values[40*i+12];} + double Gwnyy(int i){return Values[40*i+13];} + double Gwnzz(int i){return Values[40*i+14];} + double Gwnxy(int i){return Values[40*i+15];} + double Gwnxz(int i){return Values[40*i+16];} + double Gwnyz(int i){return Values[40*i+17];} + double Gnsxx(int i){return Values[40*i+18];} + double Gnsyy(int i){return Values[40*i+19];} + double Gnszz(int i){return Values[40*i+20];} + double Gnsxy(int i){return Values[40*i+21];} + double Gnsxz(int i){return Values[40*i+22];} + double Gnsyz(int i){return Values[40*i+23];} + double lwns(int i){return Values[40*i+24];} + double cospwns(int i){return Values[40*i+25];} + double vwnsx(int i){return Values[40*i+26];} + double vwnsy(int i){return Values[40*i+27];} + double vwnsz(int i){return Values[40*i+28];} + double cx(int i){return Values[40*i+29];} + double cy(int i){return Values[40*i+30];} + double cz(int i){return Values[40*i+31];} + +}; + +int main(void){ + + int Nx,Ny,Nz,N; + Nx = Ny = Nz = 100; + N = Nx*Ny*Nz; + + BlobInfo Blobs(MAX_LOCAL_BLOB_COUNT); +// LocalBlobs = new BlobInfo[MAX_LOCAL_BLOB_COUNT]; + + double * spheres; + spheres = new double [N]; + + int nblobs = 10; + + for (int i=0; i CPU //........................................................................... - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - CopyToHost(Phase.data(),Phi,N*sizeof(double)); - CopyToHost(Press.data(),Pressure,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_CopyToHost(Phase.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Press.data(),Pressure,N*sizeof(double)); MPI_Barrier(comm); //........................................................................... @@ -1370,71 +1370,71 @@ int main(int argc, char **argv) //************************************************************************* // Fused Color Gradient and Collision //************************************************************************* - ColorCollideOpt( ID,f_even,f_odd,Phi,ColorGrad, + ScaLBL_D3Q19_ColorCollide( ID,f_even,f_odd,Phi,ColorGrad, Velocity,Nx,Ny,Nz,rlxA,rlxB,alpha,beta,Fx,Fy,Fz); //************************************************************************* //................................................................................... - PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N); - PackDist(4,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(5,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(6,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(7,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N); //...Packing for X face(1,7,9,11,13)................................ - PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N); - PackDist(3,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(4,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(5,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(6,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); //...Packing for y face(4,8,9,16,18)................................. - PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,f_even,N); - PackDist(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_even,N); - PackDist(4,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); - PackDist(8,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_even,N); - PackDist(9,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,0,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_even,N); //...Packing for Y face(3,7,10,15,17)................................. - PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(5,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); - PackDist(7,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(8,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); //...Packing for z face(6,12,13,16,17)................................ - PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,f_even,N); - PackDist(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_even,N); - PackDist(6,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); - PackDist(8,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_even,N); - PackDist(8,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,0,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); //...Packing for Z face(5,11,14,15,18)................................ - PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(5,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(7,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); - PackDist(7,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(9,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); //...Pack the xy edge (8)................................ - PackDist(4,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_even,N); //...Pack the Xy edge (9)................................ - PackDist(4,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_odd,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_odd,N); //...Pack the xY edge (10)................................ - PackDist(5,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_even,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_even,N); //...Pack the XY edge (7)................................ - PackDist(3,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_odd,N); //...Pack the xz edge (12)................................ - PackDist(6,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_even,N); //...Pack the xZ edge (14)................................ - PackDist(7,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_even,N); //...Pack the Xz edge (13)................................ - PackDist(6,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_odd,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_odd,N); //...Pack the XZ edge (11)................................ - PackDist(5,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_odd,N); //...Pack the xz edge (12)................................ //...Pack the yz edge (16)................................ - PackDist(8,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_even,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_even,N); //...Pack the yZ edge (18)................................ - PackDist(9,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_even,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_even,N); //...Pack the Yz edge (17)................................ - PackDist(8,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_odd,N); //...Pack the YZ edge (15)................................ - PackDist(7,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_odd,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_odd,N); //................................................................................... //................................................................................... @@ -1482,14 +1482,14 @@ int main(int argc, char **argv) //************************************************************************* // DensityStreamD3Q7(ID, Den, Copy, Phi, ColorGrad, Velocity, beta, Nx, Ny, Nz, pBC); //************************************************************************* - MassColorCollideD3Q7(ID, A_even, A_odd, B_even, B_odd, Den, Phi, + ScaLBL_D3Q7_ColorCollideMass(ID, A_even, A_odd, B_even, B_odd, Den, Phi, ColorGrad, Velocity, beta, N, pBC); //************************************************************************* // Swap the distributions for momentum transport //************************************************************************* - SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_D3Q19_Swap(ID, f_even, f_odd, Nx, Ny, Nz); //************************************************************************* //................................................................................... @@ -1501,90 +1501,90 @@ int main(int argc, char **argv) // Unpack the distributions on the device //................................................................................... //...Map recieve list for the X face: q=2,8,10,12,13 ................................. - UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(3,-1,-1,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(4,-1,1,0,dvcRecvList_X,2*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(5,-1,0,-1,dvcRecvList_X,3*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(6,-1,0,1,dvcRecvList_X,4*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_X,2*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_X,3*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_X,4*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the x face: q=1,7,9,11,13.................................. - UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(4,1,1,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(5,1,-1,0,dvcRecvList_x,2*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(6,1,0,1,dvcRecvList_x,3*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(7,1,0,-1,dvcRecvList_x,4*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_x,2*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_x,3*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_x,4*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the y face: q=4,8,9,16,18 ................................... - UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(3,-1,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(5,1,-1,0,dvcRecvList_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,f_even,Nx,Ny,Nz); - UnpackDist(7,0,-1,-1,dvcRecvList_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,-1,1,dvcRecvList_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the Y face: q=3,7,10,15,17 .................................. - UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(4,1,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(4,-1,1,0,dvcRecvList_y,2*recvCount_y,recvCount_y,recvbuf_y,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,1,1,dvcRecvList_y,3*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(9,0,1,-1,dvcRecvList_y,4*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_y,2*recvCount_y,recvCount_y,recvbuf_y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_y,3*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_y,4*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the z face<<<6,12,13,16,17).............................................. - UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(5,-1,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(7,1,0,-1,dvcRecvList_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); - UnpackDist(7,0,-1,-1,dvcRecvList_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(9,0,1,-1,dvcRecvList_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); //...Map recieve list for the Z face<<<5,11,14,15,18).............................................. - UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(6,1,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(6,-1,0,1,dvcRecvList_z,2*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,1,1,dvcRecvList_z,3*recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(8,0,-1,1,dvcRecvList_z,4*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_z,2*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_z,3*recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_z,4*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); //.................................................................................. //...Map recieve list for the xy edge <<<8)................................ - UnpackDist(3,-1,-1,0,dvcRecvList_XY,0,recvCount_XY,recvbuf_XY,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_XY,0,recvCount_XY,recvbuf_XY,f_odd,Nx,Ny,Nz); //...Map recieve list for the Xy edge <<<9)................................ - UnpackDist(5,1,-1,0,dvcRecvList_xY,0,recvCount_xY,recvbuf_xY,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_xY,0,recvCount_xY,recvbuf_xY,f_even,Nx,Ny,Nz); //...Map recieve list for the xY edge <<<10)................................ - UnpackDist(4,-1,1,0,dvcRecvList_Xy,0,recvCount_Xy,recvbuf_Xy,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_Xy,0,recvCount_Xy,recvbuf_Xy,f_odd,Nx,Ny,Nz); //...Map recieve list for the XY edge <<<7)................................ - UnpackDist(4,1,1,0,dvcRecvList_xy,0,recvCount_xy,recvbuf_xy,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_xy,0,recvCount_xy,recvbuf_xy,f_even,Nx,Ny,Nz); //...Map recieve list for the xz edge <<<12)................................ - UnpackDist(5,-1,0,-1,dvcRecvList_XZ,0,recvCount_XZ,recvbuf_XZ,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_XZ,0,recvCount_XZ,recvbuf_XZ,f_odd,Nx,Ny,Nz); //...Map recieve list for the xZ edge <<<14)................................ - UnpackDist(6,-1,0,1,dvcRecvList_Xz,0,recvCount_Xz,recvbuf_Xz,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_Xz,0,recvCount_Xz,recvbuf_Xz,f_odd,Nx,Ny,Nz); //...Map recieve list for the Xz edge <<<13)................................ - UnpackDist(7,1,0,-1,dvcRecvList_xZ,0,recvCount_xZ,recvbuf_xZ,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_xZ,0,recvCount_xZ,recvbuf_xZ,f_even,Nx,Ny,Nz); //...Map recieve list for the XZ edge <<<11)................................ - UnpackDist(6,1,0,1,dvcRecvList_xz,0,recvCount_xz,recvbuf_xz,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_xz,0,recvCount_xz,recvbuf_xz,f_even,Nx,Ny,Nz); //...Map recieve list for the yz edge <<<16)................................ - UnpackDist(7,0,-1,-1,dvcRecvList_YZ,0,recvCount_YZ,recvbuf_YZ,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_YZ,0,recvCount_YZ,recvbuf_YZ,f_odd,Nx,Ny,Nz); //...Map recieve list for the yZ edge <<<18)................................ - UnpackDist(8,0,-1,1,dvcRecvList_Yz,0,recvCount_Yz,recvbuf_Yz,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_Yz,0,recvCount_Yz,recvbuf_Yz,f_odd,Nx,Ny,Nz); //...Map recieve list for the Yz edge <<<17)................................ - UnpackDist(9,0,1,-1,dvcRecvList_yZ,0,recvCount_yZ,recvbuf_yZ,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_yZ,0,recvCount_yZ,recvbuf_yZ,f_even,Nx,Ny,Nz); //...Map recieve list for the YZ edge <<<15)................................ - UnpackDist(8,0,1,1,dvcRecvList_yz,0,recvCount_yz,recvbuf_yz,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_yz,0,recvCount_yz,recvbuf_yz,f_even,Nx,Ny,Nz); //................................................................................... //................................................................................... - PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,A_even,N); - PackDist(1,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,B_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,0,sendCount_x,sendbuf_x,A_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,B_even,N); //...Packing for X face(1,7,9,11,13)................................ - PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,A_odd,N); - PackDist(0,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,B_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,0,sendCount_X,sendbuf_X,A_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,B_odd,N); //...Packing for y face(4,8,9,16,18)................................. - PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,A_even,N); - PackDist(2,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,B_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,0,sendCount_y,sendbuf_y,A_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,B_even,N); //...Packing for Y face(3,7,10,15,17)................................. - PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,A_odd,N); - PackDist(1,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,B_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,A_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,B_odd,N); //...Packing for z face(6,12,13,16,17)................................ - PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,A_even,N); - PackDist(3,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,B_even,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,0,sendCount_z,sendbuf_z,A_even,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,B_even,N); //...Packing for Z face(5,11,14,15,18)................................ - PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,A_odd,N); - PackDist(2,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,B_odd,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,A_odd,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,B_odd,N); //................................................................................... //................................................................................... @@ -1603,8 +1603,8 @@ int main(int argc, char **argv) MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); //................................................................................... - SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz); - SwapD3Q7(ID, B_even, B_odd, Nx, Ny, Nz); + ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz); + ScaLBL_D3Q7_Swap(ID, B_even, B_odd, Nx, Ny, Nz); //................................................................................... // Wait for completion of D3Q19 communication @@ -1614,59 +1614,59 @@ int main(int argc, char **argv) // Unpack the distributions on the device //................................................................................... //...Map recieve list for the X face: q=2,8,10,12,13 ................................. - UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,A_odd,Nx,Ny,Nz); - UnpackDist(0,-1,0,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,B_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the x face: q=1,7,9,11,13.................................. - UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,A_even,Nx,Ny,Nz); - UnpackDist(1,1,0,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,B_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the y face: q=4,8,9,16,18 ................................... - UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,A_odd,Nx,Ny,Nz); - UnpackDist(1,0,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,B_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the Y face: q=3,7,10,15,17 .................................. - UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,A_even,Nx,Ny,Nz); - UnpackDist(2,0,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,B_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the z face<<<6,12,13,16,17).............................................. - UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,A_odd,Nx,Ny,Nz); - UnpackDist(2,0,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,B_odd,Nx,Ny,Nz); //...Map recieve list for the Z face<<<5,11,14,15,18).............................................. - UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,A_even,Nx,Ny,Nz); - UnpackDist(3,0,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,B_even,Nx,Ny,Nz); //.................................................................................. //.................................................................................. - ComputeDensityD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); - ComputeDensityD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); + ScaLBL_D3Q7_Density(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); + ScaLBL_D3Q7_Density(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); //************************************************************************* // Compute the phase indicator field //************************************************************************* - // ComputePhi(ID, Phi, Copy, Den, N); - ComputePhi(ID, Phi, Den, N); + // ScaLBL_ComputePhaseField(ID, Phi, Copy, Den, N); + ScaLBL_ComputePhaseField(ID, Phi, Den, N); //************************************************************************* //................................................................................... - PackValues(dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); - PackValues(dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); - PackValues(dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); - PackValues(dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); - PackValues(dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); - PackValues(dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); - PackValues(dvcSendList_xy, sendCount_xy,sendbuf_xy, Phi, N); - PackValues(dvcSendList_xY, sendCount_xY,sendbuf_xY, Phi, N); - PackValues(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, Phi, N); - PackValues(dvcSendList_XY, sendCount_XY,sendbuf_XY, Phi, N); - PackValues(dvcSendList_xz, sendCount_xz,sendbuf_xz, Phi, N); - PackValues(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, Phi, N); - PackValues(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, Phi, N); - PackValues(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, Phi, N); - PackValues(dvcSendList_yz, sendCount_yz,sendbuf_yz, Phi, N); - PackValues(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, Phi, N); - PackValues(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, Phi, N); - PackValues(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_xy, sendCount_xy,sendbuf_xy, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_xY, sendCount_xY,sendbuf_xY, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_XY, sendCount_XY,sendbuf_XY, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_xz, sendCount_xz,sendbuf_xz, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_yz, sendCount_yz,sendbuf_yz, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N); //................................................................................... // Send / Recv all the phase indcator field values //................................................................................... @@ -1712,44 +1712,44 @@ int main(int argc, char **argv) //................................................................................... MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... - /* UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); + /* ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); */ - UnpackValues(dvcRecvList_x, recvCount_x,recvbuf_x, Phi, N); - UnpackValues(dvcRecvList_y, recvCount_y,recvbuf_y, Phi, N); - UnpackValues(dvcRecvList_z, recvCount_z,recvbuf_z, Phi, N); - UnpackValues(dvcRecvList_X, recvCount_X,recvbuf_X, Phi, N); - UnpackValues(dvcRecvList_Y, recvCount_Y,recvbuf_Y, Phi, N); - UnpackValues(dvcRecvList_Z, recvCount_Z,recvbuf_Z, Phi, N); - UnpackValues(dvcRecvList_xy, recvCount_xy,recvbuf_xy, Phi, N); - UnpackValues(dvcRecvList_xY, recvCount_xY,recvbuf_xY, Phi, N); - UnpackValues(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, Phi, N); - UnpackValues(dvcRecvList_XY, recvCount_XY,recvbuf_XY, Phi, N); - UnpackValues(dvcRecvList_xz, recvCount_xz,recvbuf_xz, Phi, N); - UnpackValues(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, Phi, N); - UnpackValues(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, Phi, N); - UnpackValues(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, Phi, N); - UnpackValues(dvcRecvList_yz, recvCount_yz,recvbuf_yz, Phi, N); - UnpackValues(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, Phi, N); - UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N); - UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Y, recvCount_Y,recvbuf_Y, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xy, recvCount_xy,recvbuf_xy, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xY, recvCount_xY,recvbuf_xY, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_XY, recvCount_XY,recvbuf_XY, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N); //................................................................................... if (pBC && kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (pBC && kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } //................................................................................... @@ -1761,9 +1761,9 @@ int main(int argc, char **argv) if (timestep%RESTART_INTERVAL == 0){ // Copy the data to the CPU - CopyToHost(cDistEven,f_even,10*N*sizeof(double)); - CopyToHost(cDistOdd,f_odd,9*N*sizeof(double)); - CopyToHost(cDen,Den,2*N*sizeof(double)); + ScaLBL_CopyToHost(cDistEven,f_even,10*N*sizeof(double)); + ScaLBL_CopyToHost(cDistOdd,f_odd,9*N*sizeof(double)); + ScaLBL_CopyToHost(cDen,Den,2*N*sizeof(double)); // Read in the restart file to CPU buffers WriteCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N); } @@ -1772,12 +1772,12 @@ int main(int argc, char **argv) // End the bubble loop //........................................................................... // Copy the phase indicator field for the later timestep - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - CopyToHost(Phase_tminus.data(),Phi,N*sizeof(double)); - CopyToHost(Phase_tplus.data(),Phi,N*sizeof(double)); - CopyToHost(Phase.data(),Phi,N*sizeof(double)); - CopyToHost(Press.data(),Pressure,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_CopyToHost(Phase_tminus.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Phase_tplus.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Phase.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Press.data(),Pressure,N*sizeof(double)); double temp=0.5/beta; for (n=0; n CPU //........................................................................... - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - CopyToHost(Phase.data,Phi,N*sizeof(double)); - CopyToHost(Press.data,Pressure,N*sizeof(double)); - CopyToHost(Vel_x.data,&Velocity[0],N*sizeof(double)); - CopyToHost(Vel_y.data,&Velocity[N],N*sizeof(double)); - CopyToHost(Vel_z.data,&Velocity[2*N],N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_CopyToHost(Phase.data,Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Press.data,Pressure,N*sizeof(double)); + ScaLBL_CopyToHost(Vel_x.data,&Velocity[0],N*sizeof(double)); + ScaLBL_CopyToHost(Vel_y.data,&Velocity[N],N*sizeof(double)); + ScaLBL_CopyToHost(Vel_z.data,&Velocity[2*N],N*sizeof(double)); MPI_Barrier(comm); //........................................................................... @@ -1528,71 +1528,71 @@ int main(int argc, char **argv) //************************************************************************* // Fused Color Gradient and Collision //************************************************************************* - ColorCollideOpt( ID,f_even,f_odd,Phi,ColorGrad, + ScaLBL_D3Q19_ColorCollide( ID,f_even,f_odd,Phi,ColorGrad, Velocity,Nx,Ny,Nz,rlxA,rlxB,alpha,beta,Fx,Fy,Fz); //************************************************************************* //................................................................................... - PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N); - PackDist(4,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(5,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(6,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - PackDist(7,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N); //...Packing for X face(1,7,9,11,13)................................ - PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N); - PackDist(3,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(4,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(5,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - PackDist(6,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); //...Packing for y face(4,8,9,16,18)................................. - PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,f_even,N); - PackDist(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_even,N); - PackDist(4,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); - PackDist(8,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_even,N); - PackDist(9,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,0,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_even,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_even,N); //...Packing for Y face(3,7,10,15,17)................................. - PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(5,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); - PackDist(7,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); - PackDist(8,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); //...Packing for z face(6,12,13,16,17)................................ - PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,f_even,N); - PackDist(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_even,N); - PackDist(6,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); - PackDist(8,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_even,N); - PackDist(8,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,0,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_even,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); //...Packing for Z face(5,11,14,15,18)................................ - PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(5,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(7,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); - PackDist(7,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); - PackDist(9,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); //...Pack the xy edge (8)................................ - PackDist(4,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_even,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_even,N); //...Pack the Xy edge (9)................................ - PackDist(4,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_odd,N); + ScaLBL_D3Q19_Pack(4,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_odd,N); //...Pack the xY edge (10)................................ - PackDist(5,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_even,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_even,N); //...Pack the XY edge (7)................................ - PackDist(3,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_odd,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_odd,N); //...Pack the xz edge (12)................................ - PackDist(6,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_even,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_even,N); //...Pack the xZ edge (14)................................ - PackDist(7,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_even,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_even,N); //...Pack the Xz edge (13)................................ - PackDist(6,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_odd,N); + ScaLBL_D3Q19_Pack(6,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_odd,N); //...Pack the XZ edge (11)................................ - PackDist(5,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_odd,N); + ScaLBL_D3Q19_Pack(5,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_odd,N); //...Pack the xz edge (12)................................ //...Pack the yz edge (16)................................ - PackDist(8,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_even,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_even,N); //...Pack the yZ edge (18)................................ - PackDist(9,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_even,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_even,N); //...Pack the Yz edge (17)................................ - PackDist(8,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_odd,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_odd,N); //...Pack the YZ edge (15)................................ - PackDist(7,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_odd,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_odd,N); //................................................................................... //................................................................................... @@ -1640,14 +1640,14 @@ int main(int argc, char **argv) //************************************************************************* // DensityStreamD3Q7(ID, Den, Copy, Phi, ColorGrad, Velocity, beta, Nx, Ny, Nz, pBC, S); //************************************************************************* - MassColorCollideD3Q7(ID, A_even, A_odd, B_even, B_odd, Den, Phi, + ScaLBL_D3Q7_ColorCollideMass(ID, A_even, A_odd, B_even, B_odd, Den, Phi, ColorGrad, Velocity, beta, N, pBC); //************************************************************************* // Swap the distributions for momentum transport //************************************************************************* - SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_D3Q19_Swap(ID, f_even, f_odd, Nx, Ny, Nz); //************************************************************************* //................................................................................... @@ -1659,90 +1659,90 @@ int main(int argc, char **argv) // Unpack the distributions on the device //................................................................................... //...Map recieve list for the X face: q=2,8,10,12,13 ................................. - UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(3,-1,-1,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(4,-1,1,0,dvcRecvList_X,2*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(5,-1,0,-1,dvcRecvList_X,3*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); - UnpackDist(6,-1,0,1,dvcRecvList_X,4*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_X,2*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_X,3*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_X,4*recvCount_X,recvCount_X,recvbuf_X,f_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the x face: q=1,7,9,11,13.................................. - UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(4,1,1,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(5,1,-1,0,dvcRecvList_x,2*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(6,1,0,1,dvcRecvList_x,3*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); - UnpackDist(7,1,0,-1,dvcRecvList_x,4*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_x,2*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_x,3*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_x,4*recvCount_x,recvCount_x,recvbuf_x,f_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the y face: q=4,8,9,16,18 ................................... - UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(3,-1,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(5,1,-1,0,dvcRecvList_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,f_even,Nx,Ny,Nz); - UnpackDist(7,0,-1,-1,dvcRecvList_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,-1,1,dvcRecvList_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,f_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the Y face: q=3,7,10,15,17 .................................. - UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(4,1,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(4,-1,1,0,dvcRecvList_y,2*recvCount_y,recvCount_y,recvbuf_y,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,1,1,dvcRecvList_y,3*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); - UnpackDist(9,0,1,-1,dvcRecvList_y,4*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_y,2*recvCount_y,recvCount_y,recvbuf_y,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_y,3*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_y,4*recvCount_y,recvCount_y,recvbuf_y,f_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the z face<<<6,12,13,16,17).............................................. - UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(5,-1,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(7,1,0,-1,dvcRecvList_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); - UnpackDist(7,0,-1,-1,dvcRecvList_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); - UnpackDist(9,0,1,-1,dvcRecvList_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,f_even,Nx,Ny,Nz); //...Map recieve list for the Z face<<<5,11,14,15,18).............................................. - UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(6,1,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(6,-1,0,1,dvcRecvList_z,2*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); - UnpackDist(8,0,1,1,dvcRecvList_z,3*recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); - UnpackDist(8,0,-1,1,dvcRecvList_z,4*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_z,2*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_z,3*recvCount_z,recvCount_z,recvbuf_z,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_z,4*recvCount_z,recvCount_z,recvbuf_z,f_odd,Nx,Ny,Nz); //.................................................................................. //...Map recieve list for the xy edge <<<8)................................ - UnpackDist(3,-1,-1,0,dvcRecvList_XY,0,recvCount_XY,recvbuf_XY,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,-1,-1,0,dvcRecvList_XY,0,recvCount_XY,recvbuf_XY,f_odd,Nx,Ny,Nz); //...Map recieve list for the Xy edge <<<9)................................ - UnpackDist(5,1,-1,0,dvcRecvList_xY,0,recvCount_xY,recvbuf_xY,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,1,-1,0,dvcRecvList_xY,0,recvCount_xY,recvbuf_xY,f_even,Nx,Ny,Nz); //...Map recieve list for the xY edge <<<10)................................ - UnpackDist(4,-1,1,0,dvcRecvList_Xy,0,recvCount_Xy,recvbuf_Xy,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,-1,1,0,dvcRecvList_Xy,0,recvCount_Xy,recvbuf_Xy,f_odd,Nx,Ny,Nz); //...Map recieve list for the XY edge <<<7)................................ - UnpackDist(4,1,1,0,dvcRecvList_xy,0,recvCount_xy,recvbuf_xy,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(4,1,1,0,dvcRecvList_xy,0,recvCount_xy,recvbuf_xy,f_even,Nx,Ny,Nz); //...Map recieve list for the xz edge <<<12)................................ - UnpackDist(5,-1,0,-1,dvcRecvList_XZ,0,recvCount_XZ,recvbuf_XZ,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(5,-1,0,-1,dvcRecvList_XZ,0,recvCount_XZ,recvbuf_XZ,f_odd,Nx,Ny,Nz); //...Map recieve list for the xZ edge <<<14)................................ - UnpackDist(6,-1,0,1,dvcRecvList_Xz,0,recvCount_Xz,recvbuf_Xz,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,-1,0,1,dvcRecvList_Xz,0,recvCount_Xz,recvbuf_Xz,f_odd,Nx,Ny,Nz); //...Map recieve list for the Xz edge <<<13)................................ - UnpackDist(7,1,0,-1,dvcRecvList_xZ,0,recvCount_xZ,recvbuf_xZ,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,1,0,-1,dvcRecvList_xZ,0,recvCount_xZ,recvbuf_xZ,f_even,Nx,Ny,Nz); //...Map recieve list for the XZ edge <<<11)................................ - UnpackDist(6,1,0,1,dvcRecvList_xz,0,recvCount_xz,recvbuf_xz,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(6,1,0,1,dvcRecvList_xz,0,recvCount_xz,recvbuf_xz,f_even,Nx,Ny,Nz); //...Map recieve list for the yz edge <<<16)................................ - UnpackDist(7,0,-1,-1,dvcRecvList_YZ,0,recvCount_YZ,recvbuf_YZ,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(7,0,-1,-1,dvcRecvList_YZ,0,recvCount_YZ,recvbuf_YZ,f_odd,Nx,Ny,Nz); //...Map recieve list for the yZ edge <<<18)................................ - UnpackDist(8,0,-1,1,dvcRecvList_Yz,0,recvCount_Yz,recvbuf_Yz,f_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,-1,1,dvcRecvList_Yz,0,recvCount_Yz,recvbuf_Yz,f_odd,Nx,Ny,Nz); //...Map recieve list for the Yz edge <<<17)................................ - UnpackDist(9,0,1,-1,dvcRecvList_yZ,0,recvCount_yZ,recvbuf_yZ,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(9,0,1,-1,dvcRecvList_yZ,0,recvCount_yZ,recvbuf_yZ,f_even,Nx,Ny,Nz); //...Map recieve list for the YZ edge <<<15)................................ - UnpackDist(8,0,1,1,dvcRecvList_yz,0,recvCount_yz,recvbuf_yz,f_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(8,0,1,1,dvcRecvList_yz,0,recvCount_yz,recvbuf_yz,f_even,Nx,Ny,Nz); //................................................................................... //................................................................................... - PackDist(1,dvcSendList_x,0,sendCount_x,sendbuf_x,A_even,N); - PackDist(1,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,B_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,0,sendCount_x,sendbuf_x,A_even,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,B_even,N); //...Packing for X face(1,7,9,11,13)................................ - PackDist(0,dvcSendList_X,0,sendCount_X,sendbuf_X,A_odd,N); - PackDist(0,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,B_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,0,sendCount_X,sendbuf_X,A_odd,N); + ScaLBL_D3Q19_Pack(0,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,B_odd,N); //...Packing for y face(4,8,9,16,18)................................. - PackDist(2,dvcSendList_y,0,sendCount_y,sendbuf_y,A_even,N); - PackDist(2,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,B_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,0,sendCount_y,sendbuf_y,A_even,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,B_even,N); //...Packing for Y face(3,7,10,15,17)................................. - PackDist(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,A_odd,N); - PackDist(1,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,B_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,A_odd,N); + ScaLBL_D3Q19_Pack(1,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,B_odd,N); //...Packing for z face(6,12,13,16,17)................................ - PackDist(3,dvcSendList_z,0,sendCount_z,sendbuf_z,A_even,N); - PackDist(3,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,B_even,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,0,sendCount_z,sendbuf_z,A_even,N); + ScaLBL_D3Q19_Pack(3,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,B_even,N); //...Packing for Z face(5,11,14,15,18)................................ - PackDist(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,A_odd,N); - PackDist(2,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,B_odd,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,A_odd,N); + ScaLBL_D3Q19_Pack(2,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,B_odd,N); //................................................................................... //................................................................................... @@ -1761,8 +1761,8 @@ int main(int argc, char **argv) MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); //................................................................................... - SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz); - SwapD3Q7(ID, B_even, B_odd, Nx, Ny, Nz); + ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz); + ScaLBL_D3Q7_Swap(ID, B_even, B_odd, Nx, Ny, Nz); //................................................................................... // Wait for completion of D3Q19 communication @@ -1772,59 +1772,59 @@ int main(int argc, char **argv) // Unpack the distributions on the device //................................................................................... //...Map recieve list for the X face: q=2,8,10,12,13 ................................. - UnpackDist(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,A_odd,Nx,Ny,Nz); - UnpackDist(0,-1,0,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,0,recvCount_X,recvbuf_X,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(0,-1,0,0,dvcRecvList_X,recvCount_X,recvCount_X,recvbuf_X,B_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the x face: q=1,7,9,11,13.................................. - UnpackDist(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,A_even,Nx,Ny,Nz); - UnpackDist(1,1,0,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,0,recvCount_x,recvbuf_x,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,1,0,0,dvcRecvList_x,recvCount_x,recvCount_x,recvbuf_x,B_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the y face: q=4,8,9,16,18 ................................... - UnpackDist(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,A_odd,Nx,Ny,Nz); - UnpackDist(1,0,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,0,recvCount_Y,recvbuf_Y,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(1,0,-1,0,dvcRecvList_Y,recvCount_Y,recvCount_Y,recvbuf_Y,B_odd,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the Y face: q=3,7,10,15,17 .................................. - UnpackDist(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,A_even,Nx,Ny,Nz); - UnpackDist(2,0,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,0,recvCount_y,recvbuf_y,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,1,0,dvcRecvList_y,recvCount_y,recvCount_y,recvbuf_y,B_even,Nx,Ny,Nz); //................................................................................... //...Map recieve list for the z face<<<6,12,13,16,17).............................................. - UnpackDist(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,A_odd,Nx,Ny,Nz); - UnpackDist(2,0,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,0,recvCount_Z,recvbuf_Z,A_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(2,0,0,-1,dvcRecvList_Z,recvCount_Z,recvCount_Z,recvbuf_Z,B_odd,Nx,Ny,Nz); //...Map recieve list for the Z face<<<5,11,14,15,18).............................................. - UnpackDist(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,A_even,Nx,Ny,Nz); - UnpackDist(3,0,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,B_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,0,recvCount_z,recvbuf_z,A_even,Nx,Ny,Nz); + ScaLBL_D3Q19_Unpack(3,0,0,1,dvcRecvList_z,recvCount_z,recvCount_z,recvbuf_z,B_even,Nx,Ny,Nz); //.................................................................................. //.................................................................................. - ComputeDensityD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); - ComputeDensityD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); + ScaLBL_D3Q7_Density(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); + ScaLBL_D3Q7_Density(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); //************************************************************************* // Compute the phase indicator field //************************************************************************* -// ComputePhi(ID, Phi, Copy, Den, N); - ComputePhi(ID, Phi, Den, N); +// ScaLBL_ComputePhaseField(ID, Phi, Copy, Den, N); + ScaLBL_ComputePhaseField(ID, Phi, Den, N); //************************************************************************* //................................................................................... - PackValues(dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); - PackValues(dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); - PackValues(dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); - PackValues(dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); - PackValues(dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); - PackValues(dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); - PackValues(dvcSendList_xy, sendCount_xy,sendbuf_xy, Phi, N); - PackValues(dvcSendList_xY, sendCount_xY,sendbuf_xY, Phi, N); - PackValues(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, Phi, N); - PackValues(dvcSendList_XY, sendCount_XY,sendbuf_XY, Phi, N); - PackValues(dvcSendList_xz, sendCount_xz,sendbuf_xz, Phi, N); - PackValues(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, Phi, N); - PackValues(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, Phi, N); - PackValues(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, Phi, N); - PackValues(dvcSendList_yz, sendCount_yz,sendbuf_yz, Phi, N); - PackValues(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, Phi, N); - PackValues(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, Phi, N); - PackValues(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_xy, sendCount_xy,sendbuf_xy, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_xY, sendCount_xY,sendbuf_xY, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_XY, sendCount_XY,sendbuf_XY, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_xz, sendCount_xz,sendbuf_xz, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_yz, sendCount_yz,sendbuf_yz, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, Phi, N); + ScaLBL_Scalar_Pack(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, Phi, N); //................................................................................... // Send / Recv all the phase indcator field values //................................................................................... @@ -1870,45 +1870,45 @@ int main(int argc, char **argv) //................................................................................... MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... -/* UnpackValues(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); - UnpackValues(faceGrid, packThreads, dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); +/* ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_x, sendCount_x,sendbuf_x, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_y, sendCount_y,sendbuf_y, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_z, sendCount_z,sendbuf_z, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_X, sendCount_X,sendbuf_X, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_Y, sendCount_Y,sendbuf_Y, Phi, N); + ScaLBL_Scalar_Unpack(faceGrid, packThreads, dvcSendList_Z, sendCount_Z,sendbuf_Z, Phi, N); */ - UnpackValues(dvcRecvList_x, recvCount_x,recvbuf_x, Phi, N); - UnpackValues(dvcRecvList_y, recvCount_y,recvbuf_y, Phi, N); - UnpackValues(dvcRecvList_z, recvCount_z,recvbuf_z, Phi, N); - UnpackValues(dvcRecvList_X, recvCount_X,recvbuf_X, Phi, N); - UnpackValues(dvcRecvList_Y, recvCount_Y,recvbuf_Y, Phi, N); - UnpackValues(dvcRecvList_Z, recvCount_Z,recvbuf_Z, Phi, N); - UnpackValues(dvcRecvList_xy, recvCount_xy,recvbuf_xy, Phi, N); - UnpackValues(dvcRecvList_xY, recvCount_xY,recvbuf_xY, Phi, N); - UnpackValues(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, Phi, N); - UnpackValues(dvcRecvList_XY, recvCount_XY,recvbuf_XY, Phi, N); - UnpackValues(dvcRecvList_xz, recvCount_xz,recvbuf_xz, Phi, N); - UnpackValues(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, Phi, N); - UnpackValues(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, Phi, N); - UnpackValues(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, Phi, N); - UnpackValues(dvcRecvList_yz, recvCount_yz,recvbuf_yz, Phi, N); - UnpackValues(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, Phi, N); - UnpackValues(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N); - UnpackValues(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Y, recvCount_Y,recvbuf_Y, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xy, recvCount_xy,recvbuf_xy, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xY, recvCount_xY,recvbuf_xY, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_XY, recvCount_XY,recvbuf_XY, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, Phi, N); + ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, Phi, N); //................................................................................... if (pBC && kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (pBC && kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } //................................................................................... @@ -1921,8 +1921,8 @@ int main(int argc, char **argv) if (timestep%1000 == 995){ //........................................................................... // Copy the phase indicator field for the earlier timestep - DeviceBarrier(); - CopyToHost(Phase_tplus.data,Phi,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Phase_tplus.data,Phi,N*sizeof(double)); //........................................................................... } if (timestep%1000 == 0){ @@ -1931,20 +1931,20 @@ int main(int argc, char **argv) //........................................................................... // Copy the phase from the GPU -> CPU //........................................................................... - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - CopyToHost(Phase.data,Phi,N*sizeof(double)); - CopyToHost(Press.data,Pressure,N*sizeof(double)); - CopyToHost(Vel_x.data,&Velocity[0],N*sizeof(double)); - CopyToHost(Vel_y.data,&Velocity[N],N*sizeof(double)); - CopyToHost(Vel_z.data,&Velocity[2*N],N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_CopyToHost(Phase.data,Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Press.data,Pressure,N*sizeof(double)); + ScaLBL_CopyToHost(Vel_x.data,&Velocity[0],N*sizeof(double)); + ScaLBL_CopyToHost(Vel_y.data,&Velocity[N],N*sizeof(double)); + ScaLBL_CopyToHost(Vel_z.data,&Velocity[2*N],N*sizeof(double)); MPI_Barrier(comm); } if (timestep%1000 == 5){ //........................................................................... // Copy the phase indicator field for the later timestep - DeviceBarrier(); - CopyToHost(Phase_tminus.data,Phi,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Phase_tminus.data,Phi,N*sizeof(double)); //........................................................................... // Calculate the time derivative of the phase indicator field for (n=0; n CPU //........................................................................... - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - CopyToHost(Averages.Phase.data(),Phi,N*sizeof(double)); - CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); - CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); - CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); - CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_CopyToHost(Averages.Phase.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); //........................................................................... if (rank==0) printf("********************************************************\n"); @@ -694,7 +694,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... @@ -743,11 +743,11 @@ int main(int argc, char **argv) //************************************************************************* // Fused Color Gradient and Collision //************************************************************************* - ColorCollideOpt( ID,f_even,f_odd,Phi,ColorGrad, + ScaLBL_D3Q19_ColorCollide( ID,f_even,f_odd,Phi,ColorGrad, Velocity,Nx,Ny,Nz,rlxA,rlxB,alpha,beta,Fx,Fy,Fz); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //************************************************************************* // Pack and send the D3Q19 distributions ScaLBL_Comm.SendD3Q19(f_even, f_odd); @@ -756,36 +756,36 @@ int main(int argc, char **argv) //************************************************************************* // Carry out the density streaming step for mass transport //************************************************************************* - MassColorCollideD3Q7(ID, A_even, A_odd, B_even, B_odd, Den, Phi, + ScaLBL_D3Q7_ColorCollideMass(ID, A_even, A_odd, B_even, B_odd, Den, Phi, ColorGrad, Velocity, beta, N, pBC); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************* // Swap the distributions for momentum transport //************************************************************************* - SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_D3Q19_Swap(ID, f_even, f_odd, Nx, Ny, Nz); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************* // Wait for communications to complete and unpack the distributions ScaLBL_Comm.RecvD3Q19(f_even, f_odd); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //************************************************************************* // Pack and send the D3Q7 distributions ScaLBL_Comm.BiSendD3Q7(A_even, A_odd, B_even, B_odd); //************************************************************************* - DeviceBarrier(); - SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz); - SwapD3Q7(ID, B_even, B_odd, Nx, Ny, Nz); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz); + ScaLBL_D3Q7_Swap(ID, B_even, B_odd, Nx, Ny, Nz); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************* @@ -793,44 +793,44 @@ int main(int argc, char **argv) ScaLBL_Comm.BiRecvD3Q7(A_even, A_odd, B_even, B_odd); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //.................................................................................. - ComputeDensityD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); - ComputeDensityD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); + ScaLBL_D3Q7_Density(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); + ScaLBL_D3Q7_Density(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); //************************************************************************* // Compute the phase indicator field //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ComputePhi(ID, Phi, Den, N); + ScaLBL_ComputePhaseField(ID, Phi, Den, N); //************************************************************************* ScaLBL_Comm.SendHalo(Phi); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); ScaLBL_Comm.RecvHalo(Phi); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); // Pressure boundary conditions if (BoundaryCondition==1 && Mask.kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (BoundaryCondition==1 && Mask.kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } // Velocity boundary conditions if (BoundaryCondition==2 && Mask.kproc == 0) { ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - //ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + //ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); SetPhiSlice_z(Phi,1.0,Nx,Ny,Nz,0); } if (BoundaryCondition==2 && Mask.kproc == nprocz-1){ ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - //ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + //ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); SetPhiSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); } @@ -841,12 +841,12 @@ int main(int argc, char **argv) dout = 1.0-0.5*dp; // set the initial boundary conditions if (Mask.kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (Mask.kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } } @@ -864,8 +864,8 @@ int main(int argc, char **argv) if (timestep%1000 == 995){ //........................................................................... // Copy the phase indicator field for the earlier timestep - DeviceBarrier(); - CopyToHost(Averages.Phase_tplus.data(),Phi,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Averages.Phase_tplus.data(),Phi,N*sizeof(double)); // Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus); //........................................................................... } @@ -875,20 +875,20 @@ int main(int argc, char **argv) //........................................................................... // Copy the phase from the GPU -> CPU //........................................................................... - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - CopyToHost(Averages.Phase.data(),Phi,N*sizeof(double)); - CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); - CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); - CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); - CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_CopyToHost(Averages.Phase.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); MPI_Barrier(MPI_COMM_WORLD); } if (timestep%1000 == 5){ //........................................................................... // Copy the phase indicator field for the later timestep - DeviceBarrier(); - CopyToHost(Averages.Phase_tminus.data(),Phi,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Averages.Phase_tminus.data(),Phi,N*sizeof(double)); // Averages.ColorToSignedDistance(beta,Averages.Phase_tminus,Averages.Phase_tminus); //.................................................................... Averages.Initialize(); @@ -917,9 +917,9 @@ int main(int argc, char **argv) double *cDistEven = new double[10*N]; double *cDistOdd = new double[9*N]; - CopyToHost(cDistEven,f_even,10*N*sizeof(double)); - CopyToHost(cDistOdd,f_odd,9*N*sizeof(double)); - CopyToHost(cDen,Den,2*N*sizeof(double)); + ScaLBL_CopyToHost(cDistEven,f_even,10*N*sizeof(double)); + ScaLBL_CopyToHost(cDistOdd,f_odd,9*N*sizeof(double)); + ScaLBL_CopyToHost(cDen,Den,2*N*sizeof(double)); // Read in the restart file to CPU buffers WriteCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N); @@ -936,7 +936,7 @@ int main(int argc, char **argv) } PROFILE_STOP("Loop"); //************************************************************************ - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); @@ -962,8 +962,8 @@ int main(int argc, char **argv) int NumberComponents_NWP = ComputeGlobalPhaseComponent(Mask.Nx-2,Mask.Ny-2,Mask.Nz-2,Mask.rank_info,Averages.PhaseID,1,Averages.Label_NWP); printf("Number of non-wetting phase components: %i \n ",NumberComponents_NWP); - DeviceBarrier(); - CopyToHost(Averages.Phase.data(),Phi,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Averages.Phase.data(),Phi,N*sizeof(double)); */ /* Averages.WriteSurfaces(0); @@ -981,10 +981,10 @@ int main(int argc, char **argv) fwrite(Averages.Press.data(),8,N,PRESS); fclose(PRESS); - CopyToHost(Averages.Phase.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Phase.data(),Phi,N*sizeof(double)); double * Grad; Grad = new double [3*N]; - CopyToHost(Grad,ColorGrad,3*N*sizeof(double)); + ScaLBL_CopyToHost(Grad,ColorGrad,3*N*sizeof(double)); sprintf(LocalRankFilename,"%s%s","ColorGrad.",LocalRankString); FILE *GRAD; GRAD = fopen(LocalRankFilename,"wb"); diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index b3ec661c..44d21ece 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -511,7 +511,7 @@ int main(int argc, char **argv) //...........device phase ID................................................. if (rank==0) printf ("Copy phase ID to device \n"); char *ID; - AllocateDeviceMemory((void **) &ID, N); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Don't compute in the halo for (k=0;kSDs.data(), dist_mem_size); + ScaLBL_CopyToDevice(dvcSignDist, Averages->SDs.data(), dist_mem_size); //........................................................................... int logcount = 0; // number of surface write-outs @@ -566,10 +566,10 @@ int main(int argc, char **argv) //........................................................................... if (rank==0) printf("Setting the distributions, size = %i\n", N); //........................................................................... - DeviceBarrier(); - InitD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); - InitDenColor(ID, Den, Phi, das, dbs, Nx, Ny, Nz); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Init(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_Color_Init(ID, Den, Phi, das, dbs, Nx, Ny, Nz); + ScaLBL_DeviceBarrier(); //...................................................................... if (Restart == true){ @@ -595,10 +595,10 @@ int main(int argc, char **argv) double *cDistOdd = new double[9*N]; ReadCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N); // Copy the restart data to the GPU - CopyToDevice(f_even,cDistEven,10*N*sizeof(double)); - CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double)); - CopyToDevice(Den,cDen,2*N*sizeof(double)); - DeviceBarrier(); + ScaLBL_CopyToDevice(f_even,cDistEven,10*N*sizeof(double)); + ScaLBL_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double)); + ScaLBL_CopyToDevice(Den,cDen,2*N*sizeof(double)); + ScaLBL_DeviceBarrier(); delete [] cDen; delete [] cDistEven; delete [] cDistOdd; @@ -606,9 +606,9 @@ int main(int argc, char **argv) } //...................................................................... - InitD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); - InitD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); - DeviceBarrier(); + ScaLBL_D3Q7_Init(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); + ScaLBL_D3Q7_Init(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //....................................................................... // Once phase has been initialized, map solid to account for 'smeared' interface @@ -624,12 +624,12 @@ int main(int argc, char **argv) //************************************************************************* // Compute the phase indicator field and reset Copy, Den //************************************************************************* - ComputePhi(ID, Phi, Den, N); + ScaLBL_ComputePhaseField(ID, Phi, Den, N); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); ScaLBL_Comm.SendHalo(Phi); ScaLBL_Comm.RecvHalo(Phi); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************* @@ -638,13 +638,13 @@ int main(int argc, char **argv) printf("Setting outlet pressure = %f \n", dout); } if (BoundaryCondition==1 && Mask.kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (BoundaryCondition==1 && Mask.kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (rank==0 && BoundaryCondition==2){ @@ -653,13 +653,13 @@ int main(int argc, char **argv) } if (BoundaryCondition==2 && Mask.kproc == 0) { ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - //ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + //ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); SetPhiSlice_z(Phi,1.0,Nx,Ny,Nz,0); } if (BoundaryCondition==2 && Mask.kproc == nprocz-1){ ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - //ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + //ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); SetPhiSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); } @@ -675,17 +675,17 @@ int main(int argc, char **argv) dout = 1.0-0.5*dp; // set the initial boundary conditions if (Mask.kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (Mask.kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } } - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - ComputeVelocityD3Q19(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); /* if (BoundaryCondition==1 && Mask.kproc == 0){ for (n=Nx*Ny; n<2*Nx*Ny; n++){ @@ -701,21 +701,21 @@ int main(int argc, char **argv) */ //........................................................................... // Copy the phase indicator field for the earlier timestep - DeviceBarrier(); - CopyToHost(Averages->Phase_tplus.data(),Phi,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Averages->Phase_tplus.data(),Phi,N*sizeof(double)); //........................................................................... //........................................................................... // Copy the data for for the analysis timestep //........................................................................... // Copy the phase from the GPU -> CPU //........................................................................... - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); - CopyToHost(Averages->Press.data(),Pressure,N*sizeof(double)); - CopyToHost(Averages->Vel_x.data(),&Velocity[0],N*sizeof(double)); - CopyToHost(Averages->Vel_y.data(),&Velocity[N],N*sizeof(double)); - CopyToHost(Averages->Vel_z.data(),&Velocity[2*N],N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Averages->Press.data(),Pressure,N*sizeof(double)); + ScaLBL_CopyToHost(Averages->Vel_x.data(),&Velocity[0],N*sizeof(double)); + ScaLBL_CopyToHost(Averages->Vel_y.data(),&Velocity[N],N*sizeof(double)); + ScaLBL_CopyToHost(Averages->Vel_z.data(),&Velocity[2*N],N*sizeof(double)); //........................................................................... if (rank==0) printf("********************************************************\n"); @@ -723,7 +723,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... @@ -788,11 +788,11 @@ int main(int argc, char **argv) //************************************************************************* // Fused Color Gradient and Collision //************************************************************************* - ColorCollideOpt( ID,f_even,f_odd,Phi,ColorGrad, + ScaLBL_D3Q19_ColorCollide( ID,f_even,f_odd,Phi,ColorGrad, Velocity,Nx,Ny,Nz,rlxA,rlxB,alpha,beta,Fx,Fy,Fz); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //************************************************************************* // Pack and send the D3Q19 distributions ScaLBL_Comm.SendD3Q19(f_even, f_odd); @@ -801,36 +801,36 @@ int main(int argc, char **argv) //************************************************************************* // Carry out the density streaming step for mass transport //************************************************************************* - MassColorCollideD3Q7(ID, A_even, A_odd, B_even, B_odd, Den, Phi, + ScaLBL_D3Q7_ColorCollideMass(ID, A_even, A_odd, B_even, B_odd, Den, Phi, ColorGrad, Velocity, beta, N, pBC); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************* // Swap the distributions for momentum transport //************************************************************************* - SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_D3Q19_Swap(ID, f_even, f_odd, Nx, Ny, Nz); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************* // Wait for communications to complete and unpack the distributions ScaLBL_Comm.RecvD3Q19(f_even, f_odd); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //************************************************************************* // Pack and send the D3Q7 distributions ScaLBL_Comm.BiSendD3Q7(A_even, A_odd, B_even, B_odd); //************************************************************************* - DeviceBarrier(); - SwapD3Q7(ID, A_even, A_odd, Nx, Ny, Nz); - SwapD3Q7(ID, B_even, B_odd, Nx, Ny, Nz); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz); + ScaLBL_D3Q7_Swap(ID, B_even, B_odd, Nx, Ny, Nz); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************* @@ -838,44 +838,44 @@ int main(int argc, char **argv) ScaLBL_Comm.BiRecvD3Q7(A_even, A_odd, B_even, B_odd); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); //.................................................................................. - ComputeDensityD3Q7(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); - ComputeDensityD3Q7(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); + ScaLBL_D3Q7_Density(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); + ScaLBL_D3Q7_Density(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); //************************************************************************* // Compute the phase indicator field //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ComputePhi(ID, Phi, Den, N); + ScaLBL_ComputePhaseField(ID, Phi, Den, N); //************************************************************************* ScaLBL_Comm.SendHalo(Phi); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); ScaLBL_Comm.RecvHalo(Phi); //************************************************************************* - DeviceBarrier(); + ScaLBL_DeviceBarrier(); // Pressure boundary conditions if (BoundaryCondition==1 && Mask.kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (BoundaryCondition==1 && Mask.kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } // Velocity boundary conditions if (BoundaryCondition==2 && Mask.kproc == 0) { ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - //ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + //ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); SetPhiSlice_z(Phi,1.0,Nx,Ny,Nz,0); } if (BoundaryCondition==2 && Mask.kproc == nprocz-1){ ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - //ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + //ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); SetPhiSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); } @@ -886,12 +886,12 @@ int main(int argc, char **argv) dout = 1.0-0.5*dp; // set the initial boundary conditions if (Mask.kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); - ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } if (Mask.kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); } } @@ -923,7 +923,7 @@ int main(int argc, char **argv) tpool.wait_pool_finished(); PROFILE_STOP("Loop"); //************************************************************************ - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); @@ -949,8 +949,8 @@ int main(int argc, char **argv) int NumberComponents_NWP = ComputeGlobalPhaseComponent(Mask.Nx-2,Mask.Ny-2,Mask.Nz-2,Mask.rank_info,Averages->PhaseID,1,Averages->Label_NWP); printf("Number of non-wetting phase components: %i \n ",NumberComponents_NWP); - DeviceBarrier(); - CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); */ /* Averages->WriteSurfaces(0); @@ -968,10 +968,10 @@ int main(int argc, char **argv) fwrite(Averages->Press.data(),8,N,PRESS); fclose(PRESS); - CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); double * Grad; Grad = new double [3*N]; - CopyToHost(Grad,ColorGrad,3*N*sizeof(double)); + ScaLBL_CopyToHost(Grad,ColorGrad,3*N*sizeof(double)); sprintf(LocalRankFilename,"%s%s","ColorGrad.",LocalRankString); FILE *GRAD; GRAD = fopen(LocalRankFilename,"wb"); diff --git a/tests/lbpm_permeability_simulator.cpp b/tests/lbpm_permeability_simulator.cpp index b54eb4bd..04bff006 100644 --- a/tests/lbpm_permeability_simulator.cpp +++ b/tests/lbpm_permeability_simulator.cpp @@ -379,9 +379,9 @@ int main(int argc, char **argv) //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; - AllocateDeviceMemory((void **) &ID, N); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device - CopyToDevice(ID, id, N); + ScaLBL_CopyToDevice(ID, id, N); //........................................................................... //........................................................................... @@ -392,18 +392,18 @@ int main(int argc, char **argv) //......................device distributions................................. double *f_even,*f_odd; //........................................................................... - AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory - AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory //........................................................................... double *Velocity, *Pressure, *dvcSignDist; //........................................................................... - AllocateDeviceMemory((void **) &Pressure, dist_mem_size); - AllocateDeviceMemory((void **) &dvcSignDist, dist_mem_size); - AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &dvcSignDist, dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); //........................................................................... // Copy signed distance for device initialization - CopyToDevice(dvcSignDist, Averages.SDs.data(), dist_mem_size); + ScaLBL_CopyToDevice(dvcSignDist, Averages.SDs.data(), dist_mem_size); //........................................................................... int logcount = 0; // number of surface write-outs @@ -414,7 +414,7 @@ int main(int argc, char **argv) //........................................................................... if (rank==0) printf("Setting the distributions, size = %i\n", N); //........................................................................... - InitD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_D3Q19_Init(ID, f_even, f_odd, Nx, Ny, Nz); //...................................................................... //....................................................................... @@ -446,11 +446,11 @@ int main(int argc, char **argv) printf("Setting outlet pressure = %f \n", dout); } if (pBC && kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); } if (pBC && kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); } int timestep = 0; @@ -472,7 +472,7 @@ int main(int argc, char **argv) //************************************************************************* // Fused Color Gradient and Collision //************************************************************************* - MRT( ID,f_even,f_odd,rlxA,rlxB,Fx,Fy,Fz,Nx,Ny,Nz); + ScaLBL_D3Q19_MRT( ID,f_even,f_odd,rlxA,rlxB,Fx,Fy,Fz,Nx,Ny,Nz); //************************************************************************* //************************************************************************* @@ -481,21 +481,21 @@ int main(int argc, char **argv) //************************************************************************* // Swap the distributions for momentum transport //************************************************************************* - SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_D3Q19_Swap(ID, f_even, f_odd, Nx, Ny, Nz); //************************************************************************* // Wait for communications to complete and unpack the distributions ScaLBL_Comm.RecvD3Q19(f_even, f_odd); //************************************************************************* if (pBC && kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); } if (pBC && kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); } //................................................................................... - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // Timestep completed! @@ -507,13 +507,13 @@ int main(int argc, char **argv) //........................................................................... // Copy the phase from the GPU -> CPU //........................................................................... - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - ComputeVelocityD3Q19(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); - CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); - CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); - CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); + ScaLBL_CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); // Way more work than necessary -- this is just to get the solid interfacial area!! Averages.Initialize(); @@ -554,7 +554,7 @@ int main(int argc, char **argv) } } //************************************************************************/ - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); From 0815951952ff1893137c82596f639385426a3769 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 23 Nov 2016 17:03:12 -0500 Subject: [PATCH 4/7] refactoring scaLBL names for cpu/*cpp and gpu/*cu --- cpu/Color.cpp | 40 +- cpu/D3Q19.cpp | 12 +- cpu/D3Q7.cpp | 14 +- cpu/Extras.cpp | 8 +- cpu/MRT.cpp | 2 +- cpu/Multiphase.cpp | 1481 ++++++++++++++++++++++++++++++++++++++++++++ gpu/Color.cu | 62 +- gpu/CudaExtras.cu | 6 +- gpu/D3Q19.cu | 40 +- gpu/D3Q7.cu | 42 +- gpu/Extras.cu | 8 +- gpu/MRT.cu | 8 +- gpu/Multiphase.cu | 1468 +++++++++++++++++++++++++++++++++++++++++++ 13 files changed, 3070 insertions(+), 121 deletions(-) create mode 100644 cpu/Multiphase.cpp create mode 100644 gpu/Multiphase.cu diff --git a/cpu/Color.cpp b/cpu/Color.cpp index 3d6e020b..c1f73fb6 100644 --- a/cpu/Color.cpp +++ b/cpu/Color.cpp @@ -2,7 +2,7 @@ #define STOKES -extern "C" void InitDenColor(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) +extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) { int n,N; @@ -27,7 +27,7 @@ extern "C" void InitDenColor(char *ID, double *Den, double *Phi, double das, dou } } } -extern "C" void InitDenColorDistancePacked(char *ID, double *Den, double *Phi, double *Distance, +extern "C" void ScaLBL_Color_InitDistancePacked(char *ID, double *Den, double *Phi, double *Distance, double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz) { int i,j,k,n,N; @@ -70,7 +70,7 @@ extern "C" void InitDenColorDistancePacked(char *ID, double *Den, double *Phi, d } } -extern "C" void InitDenColorDistance(char *ID, double *Den, double *Phi, double *Distance, +extern "C" void ScaLBL_Color_InitDistance(char *ID, double *Den, double *Phi, double *Distance, double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz) { int n,N; @@ -103,7 +103,7 @@ extern "C" void InitDenColorDistance(char *ID, double *Den, double *Phi, double //************************************************************************* -extern "C" void ColorBC_inlet(double *Phi, double *Den, double *A_even, double *A_odd, +extern "C" void ScaLBL_Color_BC_z(double *Phi, double *Den, double *A_even, double *A_odd, double *B_even, double *B_odd, int Nx, int Ny, int Nz) { int i,j,k,n,N; @@ -146,7 +146,7 @@ extern "C" void ColorBC_inlet(double *Phi, double *Den, double *A_even, double * } } //************************************************************************* -extern "C" void ColorBC_outlet(double *Phi, double *Den, double *A_even, double *A_odd, +extern "C" void ScaLBL_Color_BC_Z(double *Phi, double *Den, double *A_even, double *A_odd, double *B_even, double *B_odd, int Nx, int Ny, int Nz) { int i,j,k,n,N; @@ -190,7 +190,7 @@ extern "C" void ColorBC_outlet(double *Phi, double *Den, double *A_even, double } } //************************************************************************* -extern "C" void PressureBC_inlet(double *disteven, double *distodd, double din, +extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din, int Nx, int Ny, int Nz) { int n,N; @@ -259,7 +259,7 @@ extern "C" void PressureBC_inlet(double *disteven, double *distodd, double din, } } -extern "C" void PressureBC_outlet(double *disteven, double *distodd, double dout, +extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout, int Nx, int Ny, int Nz, int outlet) { int n,N; @@ -327,7 +327,7 @@ extern "C" void PressureBC_outlet(double *disteven, double *distodd, double dout } //************************************************************************* -extern "C" void ComputeColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz) +extern "C" void ScaLBL_D3Q19_ColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz) { int n,N,i,j,k,nn; // distributions @@ -688,7 +688,7 @@ extern "C" void ColorCollide( char *ID, double *disteven, double *distodd, doubl } // loop over n } -extern "C" void ColorCollideOpt( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, +extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, double *Velocity, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB, double alpha, double beta, double Fx, double Fy, double Fz) { @@ -1090,7 +1090,7 @@ extern "C" void ColorCollideOpt( char *ID, double *disteven, double *distodd, do } // loop over n } -extern "C" void MassColorCollideD3Q7(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, +extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC) { char id; @@ -1396,7 +1396,7 @@ extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *P } } -extern "C" void ComputePhi(char *ID, double *Phi, double *Den, int N) +extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N) { int n; double Na,Nb; @@ -1423,16 +1423,16 @@ extern "C" void SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, /* // ************************************************************************* -extern "C" void InitDenColor( int nblocks, int nthreads, int S, +extern "C" void ScaLBL_Color_Init( int nblocks, int nthreads, int S, char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) { - InitDenColor <<>> (ID, Den, Phi, das, dbs, Nx, Ny, Nz, S); + ScaLBL_Color_Init <<>> (ID, Den, Phi, das, dbs, Nx, Ny, Nz, S); } // ************************************************************************* -extern "C" void ComputeColorGradient(int nBlocks, int nthreads, int S, +extern "C" void ScaLBL_D3Q19_ColorGradient(int nBlocks, int nthreads, int S, char *ID, double *Phi, double *ColorGrad, int Nx, int Ny, int Nz) { - ComputeColorGradient<<>>(ID, Phi, ColorGrad, Nx, Ny, Nz, S); + ScaLBL_D3Q19_ColorGradient<<>>(ID, Phi, ColorGrad, Nx, Ny, Nz, S); } // ************************************************************************* extern "C" void ColorCollide(int nBlocks, int nthreads, int S, @@ -1444,12 +1444,12 @@ extern "C" void ColorCollide(int nBlocks, int nthreads, int S, rlxA, rlxB, alpha, beta, Fx, Fy, Fz, pBC); } // ************************************************************************* -extern "C" void ColorCollideOpt(int nBlocks, int nthreads, int S, +extern "C" void ScaLBL_D3Q19_ColorCollide(int nBlocks, int nthreads, int S, char *ID, double *f_even, double *f_odd, double *Phi, double *ColorGrad, double *Velocity, int Nx, int Ny, int Nz,double rlxA, double rlxB, double alpha, double beta, double Fx, double Fy, double Fz) { - ColorCollideOpt<<>>(ID, f_even, f_odd, Phi, ColorGrad, Velocity, Nx, Ny, Nz, S, + ScaLBL_D3Q19_ColorCollide<<>>(ID, f_even, f_odd, Phi, ColorGrad, Velocity, Nx, Ny, Nz, S, rlxA, rlxB, alpha, beta, Fx, Fy, Fz); // bool pBC = false; // ColorCollide<<>>(ID, f_even, f_odd, ColorGrad, Velocity, Nx, Ny, Nz, S, @@ -1464,10 +1464,10 @@ extern "C" void DensityStreamD3Q7(int nBlocks, int nthreads, int S, DensityStreamD3Q7<<>>(ID,Den,Copy,Phi,ColorGrad,Velocity,beta,Nx,Ny,Nz,pBC,S); } // ************************************************************************* -extern "C" void ComputePhi(int nBlocks, int nthreads, int S, +extern "C" void ScaLBL_ComputePhaseField(int nBlocks, int nthreads, int S, char *ID, double *Phi, double *Copy, double *Den, int N) { - ComputePhi<<>>(ID,Phi,Copy,Den,N,S); + ScaLBL_ComputePhaseField<<>>(ID,Phi,Copy,Den,N,S); } // ************************************************************************* extern "C" void ComputePressure(int nBlocks, int nthreads, int S, @@ -1475,7 +1475,7 @@ extern "C" void ComputePressure(int nBlocks, int nthreads, int S, double *Pressure, int Nx, int Ny, int Nz) { - ComputePressureD3Q19<<>>(ID,disteven,distodd,Pressure,Nx,Ny,Nz,S); + ScaLBL_D3Q19_Pressure<<>>(ID,disteven,distodd,Pressure,Nx,Ny,Nz,S); } */ diff --git a/cpu/D3Q19.cpp b/cpu/D3Q19.cpp index 0c64a430..78e1e8e1 100644 --- a/cpu/D3Q19.cpp +++ b/cpu/D3Q19.cpp @@ -1,4 +1,4 @@ -extern "C" void PackDist(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){ +extern "C" void ScaLBL_D3Q19_Pack(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){ //.................................................................................... // Pack distribution q into the send buffer for the listed lattice sites // dist may be even or odd distributions stored by stream layout @@ -10,7 +10,7 @@ extern "C" void PackDist(int q, int *list, int start, int count, double *sendbuf } } -extern "C" void UnpackDist(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, +extern "C" void ScaLBL_D3Q19_Unpack(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, double *recvbuf, double *dist, int Nx, int Ny, int Nz){ //.................................................................................... // Unack distribution from the recv buffer @@ -54,7 +54,7 @@ extern "C" void UnpackDist(int q, int Cqx, int Cqy, int Cqz, int *list, int sta } } -extern "C" void InitD3Q19(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz) +extern "C" void ScaLBL_D3Q19_Init(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz) { int n,N; N = Nx*Ny*Nz; @@ -93,7 +93,7 @@ extern "C" void InitD3Q19(char *ID, double *f_even, double *f_odd, int Nx, int N } //************************************************************************* -extern "C" void SwapD3Q19(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz) +extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz) { int i,j,k,n,nn,N; // distributions @@ -340,7 +340,7 @@ extern "C" void ScaLBL_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, do } } -extern "C" void ComputeVelocityD3Q19(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz) +extern "C" void ScaLBL_D3Q19_Velocity(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz) { int n,N; // distributions @@ -395,7 +395,7 @@ extern "C" void ComputeVelocityD3Q19(char *ID, double *disteven, double *distodd } } -extern "C" void ComputePressureD3Q19(const char *ID, const double *disteven, const double *distodd, +extern "C" void ScaLBL_D3Q19_Pressure(const char *ID, const double *disteven, const double *distodd, double *Pressure, int Nx, int Ny, int Nz) { int n,N; diff --git a/cpu/D3Q7.cpp b/cpu/D3Q7.cpp index 6bf92af6..6a31aef1 100644 --- a/cpu/D3Q7.cpp +++ b/cpu/D3Q7.cpp @@ -1,6 +1,6 @@ // CPU Functions for D3Q7 Lattice Boltzmann Methods -extern "C" void PackValues(int *list, int count, double *sendbuf, double *Data, int N){ +extern "C" void ScaLBL_D3Q19_Unpack(int *list, int count, double *sendbuf, double *Data, int N){ //.................................................................................... // Pack distribution q into the send buffer for the listed lattice sites // dist may be even or odd distributions stored by stream layout @@ -11,7 +11,7 @@ extern "C" void PackValues(int *list, int count, double *sendbuf, double *Data, sendbuf[idx] = Data[n]; } } -extern "C" void UnpackValues(int *list, int count, double *recvbuf, double *Data, int N){ +extern "C" void ScaLBL_Scalar_Unpack(int *list, int count, double *recvbuf, double *Data, int N){ //.................................................................................... // Pack distribution q into the send buffer for the listed lattice sites // dist may be even or odd distributions stored by stream layout @@ -23,7 +23,7 @@ extern "C" void UnpackValues(int *list, int count, double *recvbuf, double *Data } } -extern "C" void PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){ +extern "C" void ScaLBL_PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){ //.................................................................................... // Pack distribution into the send buffer for the listed lattice sites //.................................................................................... @@ -38,7 +38,7 @@ extern "C" void PackDenD3Q7(int *list, int count, double *sendbuf, int number, d } -extern "C" void UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){ +extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){ //.................................................................................... // Unack distribution from the recv buffer // Sum to the existing density value @@ -52,7 +52,7 @@ extern "C" void UnpackDenD3Q7(int *list, int count, double *recvbuf, int number } } -extern "C" void InitD3Q7(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz) +extern "C" void ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz) { int n,N; N = Nx*Ny*Nz; @@ -81,7 +81,7 @@ extern "C" void InitD3Q7(char *ID, double *f_even, double *f_odd, double *Den, i } //************************************************************************* -extern "C" void SwapD3Q7(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz) +extern "C" void ScaLBL_D3Q7_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz) { int i,j,k,n,nn,N; // distributions @@ -141,7 +141,7 @@ extern "C" void SwapD3Q7(char *ID, double *disteven, double *distodd, int Nx, in } //************************************************************************* -extern "C" void ComputeDensityD3Q7(char *ID, double *disteven, double *distodd, double *Den, +extern "C" void ScaLBL_D3Q7_Density(char *ID, double *disteven, double *distodd, double *Den, int Nx, int Ny, int Nz) { char id; diff --git a/cpu/Extras.cpp b/cpu/Extras.cpp index 2cdedc39..d3abfc8f 100644 --- a/cpu/Extras.cpp +++ b/cpu/Extras.cpp @@ -3,7 +3,7 @@ #include #include -extern "C" void AllocateDeviceMemory(void** address, size_t size){ +extern "C" void ScaLBL_ScaLBL_ScaLBL_AllocateDeviceMemory(void** address, size_t size){ //cudaMalloc(address,size); (*address) = malloc(size); memset(*address,0,size); @@ -13,17 +13,17 @@ extern "C" void AllocateDeviceMemory(void** address, size_t size){ } } -extern "C" void CopyToDevice(void* dest, const void* source, size_t size){ +extern "C" void ScaLBL_ScaLBL_CopyToDevice(void* dest, const void* source, size_t size){ // cudaMemcpy(dest,source,size,cudaMemcpyHostToDevice); memcpy(dest, source, size); } -extern "C" void CopyToHost(void* dest, const void* source, size_t size){ +extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size){ // cudaMemcpy(dest,source,size,cudaMemcpyDeviceToHost); memcpy(dest, source, size); } -extern "C" void DeviceBarrier(){ +extern "C" void ScaLBL_DeviceBarrier(){ // cudaDeviceSynchronize(); } diff --git a/cpu/MRT.cpp b/cpu/MRT.cpp index 1fa2b762..b3ddb47d 100644 --- a/cpu/MRT.cpp +++ b/cpu/MRT.cpp @@ -88,7 +88,7 @@ extern "C" void Compute_VELOCITY(char *ID, double *disteven, double *distodd, do //************************************************************************* -extern "C" void MRT(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz, +extern "C" void ScaLBL_D3Q19_MRT(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz) { diff --git a/cpu/Multiphase.cpp b/cpu/Multiphase.cpp new file mode 100644 index 00000000..c1f73fb6 --- /dev/null +++ b/cpu/Multiphase.cpp @@ -0,0 +1,1481 @@ +#include + +#define STOKES + +extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) +{ + int n,N; + + N = Nx*Ny*Nz; + + for (n=0; n 0){ + + // Retrieve the color gradient + nx = ColorGrad[n]; + ny = ColorGrad[N+n]; + nz = ColorGrad[2*N+n]; + //...........Normalize the Color Gradient................................. + C = sqrt(nx*nx+ny*ny+nz*nz); + if (C==0.0) C=1.0; + nx = nx/C; + ny = ny/C; + nz = nz/C; + //......No color gradient at z-boundary if pressure BC are set............. + // if (pBC && k==0) nx = ny = nz = 0.f; + // if (pBC && k==Nz-1) nx = ny = nz = 0.f; + //........................................................................ + // READ THE DISTRIBUTIONS + // (read from opposite array due to previous swap operation) + //........................................................................ + f2 = distodd[n]; + f4 = distodd[N+n]; + f6 = distodd[2*N+n]; + f8 = distodd[3*N+n]; + f10 = distodd[4*N+n]; + f12 = distodd[5*N+n]; + f14 = distodd[6*N+n]; + f16 = distodd[7*N+n]; + f18 = distodd[8*N+n]; + //........................................................................ + f0 = disteven[n]; + f1 = disteven[N+n]; + f3 = disteven[2*N+n]; + f5 = disteven[3*N+n]; + f7 = disteven[4*N+n]; + f9 = disteven[5*N+n]; + f11 = disteven[6*N+n]; + f13 = disteven[7*N+n]; + f15 = disteven[8*N+n]; + f17 = disteven[9*N+n]; + //........................................................................ + // PERFORM RELAXATION PROCESS + //........................................................................ + //....................compute the moments............................................... + rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + m1 = -30*f0-11*(f2+f1+f4+f3+f6+f5)+8*(f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18 +f17); + m2 = 12*f0-4*(f2+f1 +f4+f3+f6 +f5)+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + jx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; + m4 = 4*(-f1+f2)+f7-f8+f9-f10+f11-f12+f13-f14; + jy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; + m6 = -4*(f3-f4)+f7-f8-f9+f10+f15-f16+f17-f18; + jz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18; + m8 = -4*(f5-f6)+f11-f12-f13+f14+f15-f16-f17+f18; + m9 = 2*(f1+f2)-f3-f4-f5-f6+f7+f8+f9+f10+f11+f12+f13+f14-2*(f15+f16+f17+f18); + m10 = -4*(f1+f2)+2*(f4+f3+f6+f5)+f8+f7+f10+f9+f12+f11+f14+f13-2*(f16+f15+f18+f17); + m11 = f4+f3-f6-f5+f8+f7+f10+f9-f12-f11-f14-f13; + m12 = -2*(f4+f3-f6-f5)+f8+f7+f10+f9-f12-f11-f14-f13; + m13 = f8+f7-f10-f9; + m14 = f16+f15-f18-f17; + m15 = f12+f11-f14-f13; + m16 = f7-f8+f9-f10-f11+f12-f13+f14; + m17 = -f7+f8+f9-f10+f15-f16+f17-f18; + m18 = f11-f12-f13+f14-f15+f16+f17-f18; + //..........Toelke, Fruediger et. al. 2006............... + if (C == 0.0) nx = ny = nz = 1.0; +#ifdef STOKES + m1 = m1 + rlx_setA*(- 11*rho -alpha*C - m1); + m2 = m2 + rlx_setA*(3*rho - m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8); + m9 = m9 + rlx_setA*( 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); + m10 = m10 + rlx_setA*( - m10); + m11 = m11 + rlx_setA*( 0.5*alpha*C*(ny*ny-nz*nz)- m11); + m12 = m12 + rlx_setA*( - m12); + m13 = m13 + rlx_setA*( 0.5*alpha*C*nx*ny - m13); + m14 = m14 + rlx_setA*( 0.5*alpha*C*ny*nz - m14); + m15 = m15 + rlx_setA*( 0.5*alpha*C*nx*nz - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); +#else + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) -alpha*C - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho)- m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); + m10 = m10 + rlx_setA*( - m10); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) + 0.5*alpha*C*(ny*ny-nz*nz)- m11); + m12 = m12 + rlx_setA*( - m12); + m13 = m13 + rlx_setA*( (jx*jy/rho) + 0.5*alpha*C*nx*ny - m13); + m14 = m14 + rlx_setA*( (jy*jz/rho) + 0.5*alpha*C*ny*nz - m14); + m15 = m15 + rlx_setA*( (jx*jz/rho) + 0.5*alpha*C*nx*nz - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); +#endif + //.................inverse transformation...................................................... + f0 = 0.05263157894736842*rho-0.012531328320802*m1+0.04761904761904762*m2; + f1 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(jx-m4)+0.0555555555555555555555555*(m9-m10); + f2 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(m4-jx)+0.0555555555555555555555555*(m9-m10); + f3 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(jy-m6)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12); + f4 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(m6-jy)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12); + f5 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(jz-m8)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11); + f6 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(m8-jz)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11); + f7 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx+jy)+0.025*(m4+m6) + +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 + +0.04166666666666666*m12+0.25*m13+0.125*(m16-m17); + f8 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2-0.1*(jx+jy)-0.025*(m4+m6) + +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 + +0.04166666666666666*m12+0.25*m13+0.125*(m17-m16); + f9 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx-jy)+0.025*(m4-m6) + +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 + +0.04166666666666666*m12-0.25*m13+0.125*(m16+m17); + f10 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jy-jx)+0.025*(m6-m4) + +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 + +0.04166666666666666*m12-0.25*m13-0.125*(m16+m17); + f11 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jx+jz)+0.025*(m4+m8) + +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 + -0.04166666666666666*m12+0.25*m15+0.125*(m18-m16); + f12 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2-0.1*(jx+jz)-0.025*(m4+m8) + +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 + -0.04166666666666666*m12+0.25*m15+0.125*(m16-m18); + f13 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jx-jz)+0.025*(m4-m8) + +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 + -0.04166666666666666*m12-0.25*m15-0.125*(m16+m18); + f14 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jz-jx)+0.025*(m8-m4) + +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 + -0.04166666666666666*m12-0.25*m15+0.125*(m16+m18); + f15 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jy+jz)+0.025*(m6+m8) + -0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m17-m18); + f16 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2-0.1*(jy+jz)-0.025*(m6+m8) + -0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m18-m17); + f17 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jy-jz)+0.025*(m6-m8) + -0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14+0.125*(m17+m18); + f18 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jz-jy)+0.025*(m8-m6) + -0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14-0.125*(m17+m18); + //....................................................................................................... + // incorporate external force + f1 += 0.16666666*Fx; + f2 -= 0.16666666*Fx; + f3 += 0.16666666*Fy; + f4 -= 0.16666666*Fy; + f5 += 0.16666666*Fz; + f6 -= 0.16666666*Fz; + f7 += 0.08333333333*(Fx+Fy); + f8 -= 0.08333333333*(Fx+Fy); + f9 += 0.08333333333*(Fx-Fy); + f10 -= 0.08333333333*(Fx-Fy); + f11 += 0.08333333333*(Fx+Fz); + f12 -= 0.08333333333*(Fx+Fz); + f13 += 0.08333333333*(Fx-Fz); + f14 -= 0.08333333333*(Fx-Fz); + f15 += 0.08333333333*(Fy+Fz); + f16 -= 0.08333333333*(Fy+Fz); + f17 += 0.08333333333*(Fy-Fz); + f18 -= 0.08333333333*(Fy-Fz); + //*********** WRITE UPDATED VALUES TO MEMORY ****************** + // Write the updated distributions + //....EVEN..................................... + disteven[n] = f0; + disteven[N+n] = f2; + disteven[2*N+n] = f4; + disteven[3*N+n] = f6; + disteven[4*N+n] = f8; + disteven[5*N+n] = f10; + disteven[6*N+n] = f12; + disteven[7*N+n] = f14; + disteven[8*N+n] = f16; + disteven[9*N+n] = f18; + //....ODD...................................... + distodd[n] = f1; + distodd[N+n] = f3; + distodd[2*N+n] = f5; + distodd[3*N+n] = f7; + distodd[4*N+n] = f9; + distodd[5*N+n] = f11; + distodd[6*N+n] = f13; + distodd[7*N+n] = f15; + distodd[8*N+n] = f17; + + //...Store the Velocity.......................... + Velocity[n] = jx; + Velocity[N+n] = jy; + Velocity[2*N+n] = jz; + /* Velocity[3*n] = jx; + Velocity[3*n+1] = jy; + Velocity[3*n+2] = jz; + */ //...Store the Color Gradient.................... + // ColorGrad[3*n] = nx*C; + // ColorGrad[3*n+1] = ny*C; + // ColorGrad[3*n+2] = nz*C; + //............................................... + //*************************************************************** + } // check if n is in the solid + } // loop over n +} + +extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, + double *Velocity, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB, + double alpha, double beta, double Fx, double Fy, double Fz) +{ + + int i,j,k,n,nn,N; + // distributions + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + // additional variables needed for computations + double rho,jx,jy,jz,C,nx,ny,nz; + + N = Nx*Ny*Nz; + char id; + + for (n=0; n 0){ + + //.......Back out the 3-D indices for node n.............. + k = n/(Nx*Ny); + j = (n-Nx*Ny*k)/Nx; + i = n-Nx*Ny*k-Nx*j; + //........................................................................ + //........Get 1-D index for this thread.................... + // n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x; + //........................................................................ + // COMPUTE THE COLOR GRADIENT + //........................................................................ + //.................Read Phase Indicator Values............................ + //........................................................................ + nn = n-1; // neighbor index (get convention) + if (i-1<0) nn += Nx; // periodic BC along the x-boundary + f1 = phi[nn]; // get neighbor for phi - 1 + //........................................................................ + nn = n+1; // neighbor index (get convention) + if (!(i+10)) delta=0; + a1 = na*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nb*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = na*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nb*(0.1111111111111111*(1-4.5*ux))+delta; + + A_odd[n] = a1; + A_even[N+n] = a2; + B_odd[n] = b1; + B_even[N+n] = b2; + //............................................... + // q = 2 + // Cq = {0,1,0} + delta = beta*na*nb*nab*0.1111111111111111*ny; + if (!(na*nb*nab>0)) delta=0; + a1 = na*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nb*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = na*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nb*(0.1111111111111111*(1-4.5*uy))+delta; + + A_odd[N+n] = a1; + A_even[2*N+n] = a2; + B_odd[N+n] = b1; + B_even[2*N+n] = b2; + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*na*nb*nab*0.1111111111111111*nz; + if (!(na*nb*nab>0)) delta=0; + a1 = na*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nb*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = na*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nb*(0.1111111111111111*(1-4.5*uz))+delta; + + A_odd[2*N+n] = a1; + A_even[3*N+n] = a2; + B_odd[2*N+n] = b1; + B_even[3*N+n] = b2; + //............................................... + + /* // Construction and streaming for the components + for (idx=0; idx<3; idx++){ + //............................................... + // Distribution index + q = 2*idx; + // Associated discrete velocity + Cqx = D3Q7[idx][0]; + Cqy = D3Q7[idx][1]; + Cqz = D3Q7[idx][2]; + // Generate the Equilibrium Distribution + a1 = na*feq[q]; + b1 = nb*feq[q]; + a2 = na*feq[q+1]; + b2 = nb*feq[q+1]; + // Recolor the distributions + if (C > 0.0){ + sp = nx*double(Cqx)+ny*double(Cqy)+nz*double(Cqz); + //if (idx > 2) sp = 0.7071067811865475*sp; + //delta = sp*min( min(a1,a2), min(b1,b2) ); + delta = na*nb/(na+nb)*0.1111111111111111*sp; + //if (a1>0 && b1>0){ + a1 += beta*delta; + a2 -= beta*delta; + b1 -= beta*delta; + b2 += beta*delta; + } + // Save the re-colored distributions + A_odd[N*idx+n] = a1; + A_even[N*(idx+1)+n] = a2; + B_odd[N*idx+n] = b1; + B_even[N*(idx+1)+n] = b2; + //............................................... + } + */ + } + } +} + +//************************************************************************* +extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, + double beta, int Nx, int Ny, int Nz, bool pBC, int S) +{ + char id; + + int idx; + int in,jn,kn,n,nn,N; + int q,Cqx,Cqy,Cqz; + // int sendLoc; + + double na,nb; // density values + double ux,uy,uz; // flow velocity + double nx,ny,nz,C; // color gradient components + double a1,a2,b1,b2; + double sp,delta; + double feq[6]; // equilibrium distributions + // Set of Discrete velocities for the D3Q19 Model + int D3Q7[3][3]={{1,0,0},{0,1,0},{0,0,1}}; + N = Nx*Ny*Nz; + + for (n=0; n 0 && na+nb > 0.0){ + //.......Back out the 3-D indices for node n.............. + int k = n/(Nx*Ny); + int j = (n-Nx*Ny*k)/Nx; + int i = n-Nx*Ny*k-Nx*j; + //.....Load the Color gradient......... + nx = ColorGrad[n]; + ny = ColorGrad[N+n]; + nz = ColorGrad[2*N+n]; + C = sqrt(nx*nx+ny*ny+nz*nz); + nx = nx/C; + ny = ny/C; + nz = nz/C; + //....Load the flow velocity........... + ux = Velocity[n]; + uy = Velocity[N+n]; + uz = Velocity[2*N+n]; + //....Instantiate the density distributions + // Generate Equilibrium Distributions and stream + // Stationary value - distribution 0 + // Den[2*n] += 0.3333333333333333*na; + // Den[2*n+1] += 0.3333333333333333*nb; + Den[2*n] += 0.3333333333333333*na; + Den[2*n+1] += 0.3333333333333333*nb; + // Non-Stationary equilibrium distributions + feq[0] = 0.1111111111111111*(1+3*ux); + feq[1] = 0.1111111111111111*(1-3*ux); + feq[2] = 0.1111111111111111*(1+3*uy); + feq[3] = 0.1111111111111111*(1-3*uy); + feq[4] = 0.1111111111111111*(1+3*uz); + feq[5] = 0.1111111111111111*(1-3*uz); + // Construction and streaming for the components + for (idx=0; idx<3; idx++){ + // Distribution index + q = 2*idx; + // Associated discrete velocity + Cqx = D3Q7[idx][0]; + Cqy = D3Q7[idx][1]; + Cqz = D3Q7[idx][2]; + // Generate the Equilibrium Distribution + a1 = na*feq[q]; + b1 = nb*feq[q]; + a2 = na*feq[q+1]; + b2 = nb*feq[q+1]; + // Recolor the distributions + if (C > 0.0){ + sp = nx*double(Cqx)+ny*double(Cqy)+nz*double(Cqz); + //if (idx > 2) sp = 0.7071067811865475*sp; + //delta = sp*min( min(a1,a2), min(b1,b2) ); + delta = na*nb/(na+nb)*0.1111111111111111*sp; + //if (a1>0 && b1>0){ + a1 += beta*delta; + a2 -= beta*delta; + b1 -= beta*delta; + b2 += beta*delta; + } + + // .......Get the neighbor node.............. + //nn = n + Stride[idx]; + in = i+Cqx; + jn = j+Cqy; + kn = k+Cqz; + + // Adjust for periodic BC, if necessary + // if (in<0) in+= Nx; + // if (jn<0) jn+= Ny; + // if (kn<0) kn+= Nz; + // if (!(in 0 ){ + // Get the density value (Streaming already performed) + Na = Den[n]; + Nb = Den[N+n]; + Phi[n] = (Na-Nb)/(Na+Nb); + } + } + //................................................................... +} + +extern "C" void SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){ + int n; + for (n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){ + Phi[n] = value; + } +} + +/* +// ************************************************************************* +extern "C" void ScaLBL_Color_Init( int nblocks, int nthreads, int S, + char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) +{ + ScaLBL_Color_Init <<>> (ID, Den, Phi, das, dbs, Nx, Ny, Nz, S); +} +// ************************************************************************* +extern "C" void ScaLBL_D3Q19_ColorGradient(int nBlocks, int nthreads, int S, + char *ID, double *Phi, double *ColorGrad, int Nx, int Ny, int Nz) +{ + ScaLBL_D3Q19_ColorGradient<<>>(ID, Phi, ColorGrad, Nx, Ny, Nz, S); +} +// ************************************************************************* +extern "C" void ColorCollide(int nBlocks, int nthreads, int S, + char *ID, double *f_even, double *f_odd, double *ColorGrad, double *Velocity, + double rlxA, double rlxB,double alpha, double beta, double Fx, double Fy, double Fz, + int Nx, int Ny, int Nz, bool pBC) +{ + ColorCollide<<>>(ID, f_even, f_odd, ColorGrad, Velocity, Nx, Ny, Nz, S, + rlxA, rlxB, alpha, beta, Fx, Fy, Fz, pBC); +} +// ************************************************************************* +extern "C" void ScaLBL_D3Q19_ColorCollide(int nBlocks, int nthreads, int S, + char *ID, double *f_even, double *f_odd, double *Phi, double *ColorGrad, + double *Velocity, int Nx, int Ny, int Nz,double rlxA, double rlxB, + double alpha, double beta, double Fx, double Fy, double Fz) +{ + ScaLBL_D3Q19_ColorCollide<<>>(ID, f_even, f_odd, Phi, ColorGrad, Velocity, Nx, Ny, Nz, S, + rlxA, rlxB, alpha, beta, Fx, Fy, Fz); +// bool pBC = false; +// ColorCollide<<>>(ID, f_even, f_odd, ColorGrad, Velocity, Nx, Ny, Nz, S, +// rlxA, rlxB, alpha, beta, Fx, Fy, Fz, pBC); +} + +// ************************************************************************* +extern "C" void DensityStreamD3Q7(int nBlocks, int nthreads, int S, + char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, + double beta, int Nx, int Ny, int Nz, bool pBC) +{ + DensityStreamD3Q7<<>>(ID,Den,Copy,Phi,ColorGrad,Velocity,beta,Nx,Ny,Nz,pBC,S); +} +// ************************************************************************* +extern "C" void ScaLBL_ComputePhaseField(int nBlocks, int nthreads, int S, + char *ID, double *Phi, double *Copy, double *Den, int N) +{ + ScaLBL_ComputePhaseField<<>>(ID,Phi,Copy,Den,N,S); +} +// ************************************************************************* +extern "C" void ComputePressure(int nBlocks, int nthreads, int S, + char *ID, double *disteven, double *distodd, + double *Pressure, int Nx, int Ny, int Nz) +{ + + ScaLBL_D3Q19_Pressure<<>>(ID,disteven,distodd,Pressure,Nx,Ny,Nz,S); + +} +*/ diff --git a/gpu/Color.cu b/gpu/Color.cu index a250876c..8d91c8c5 100644 --- a/gpu/Color.cu +++ b/gpu/Color.cu @@ -3,7 +3,7 @@ #define NBLOCKS 32 #define NTHREADS 128 -__global__ void dvc_InitDenColor(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) +__global__ void dvc_ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) { //int i,j,k; int n,N; @@ -40,7 +40,7 @@ __global__ void dvc_InitDenColor(char *ID, double *Den, double *Phi, double das } } } -__global__ void dvc_InitDenColorDistancePacked(char *ID, double *Den, double *Phi, double *Distance, +__global__ void dvc_ScaLBL_Color_InitDistancePacked(char *ID, double *Den, double *Phi, double *Distance, double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz) { int i,j,k,n,N; @@ -88,7 +88,7 @@ __global__ void dvc_InitDenColorDistancePacked(char *ID, double *Den, double *P } } -__global__ void dvc_InitDenColorDistance(char *ID, double *Den, double *Phi, double *Distance, +__global__ void dvc_ScaLBL_Color_InitDistance(char *ID, double *Den, double *Phi, double *Distance, double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz) { int n,N; @@ -122,7 +122,7 @@ __global__ void dvc_InitDenColorDistance(char *ID, double *Den, double *Phi, do } } //************************************************************************* -__global__ void dvc_ColorBC_inlet(double *Phi, double *Den, double *A_even, double *A_odd, +__global__ void dvc_ScaLBL_Color_BC_z(double *Phi, double *Den, double *A_even, double *A_odd, double *B_even, double *B_odd, int Nx, int Ny, int Nz) { int ij,k,n,N; @@ -161,7 +161,7 @@ __global__ void dvc_ColorBC_inlet(double *Phi, double *Den, double *A_even, dou } } //************************************************************************* -__global__ void dvc_ColorBC_outlet(double *Phi, double *Den, double *A_even, double *A_odd, +__global__ void dvc_ScaLBL_Color_BC_Z(double *Phi, double *Den, double *A_even, double *A_odd, double *B_even, double *B_odd, int Nx, int Ny, int Nz) { int ij,k,n,N; @@ -199,7 +199,7 @@ __global__ void dvc_ColorBC_outlet(double *Phi, double *Den, double *A_even, do } } //************************************************************************* -__global__ void dvc_PressureBC_inlet(double *disteven, double *distodd, double din, +__global__ void dvc_ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din, int Nx, int Ny, int Nz) { int n,N; @@ -269,7 +269,7 @@ __global__ void dvc_PressureBC_inlet(double *disteven, double *distodd, double } } -__global__ void dvc_PressureBC_outlet(double *disteven, double *distodd, double dout, +__global__ void dvc_ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout, int Nx, int Ny, int Nz, int outlet) { int n,N; @@ -337,7 +337,7 @@ __global__ void dvc_PressureBC_outlet(double *disteven, double *distodd, double } } //************************************************************************* -__global__ void dvc_ComputeColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz) +__global__ void dvc_ScaLBL_D3Q19_ColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz) { int n,N,i,j,k,nn; // distributions @@ -683,7 +683,7 @@ __global__ void dvc_ColorCollide( char *ID, double *disteven, double *distodd, } // loop over n } -__global__ void dvc_ColorCollideOpt( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, +__global__ void dvc_ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, double *Velocity, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB, double alpha, double beta, double Fx, double Fy, double Fz) { @@ -1089,7 +1089,7 @@ __global__ void dvc_ColorCollideOpt( char *ID, double *disteven, double *distod } // loop over n } -__global__ void dvc_MassColorCollideD3Q7(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, +__global__ void dvc_ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC) { int n; @@ -1364,7 +1364,7 @@ __global__ void dvc_DensityStreamD3Q7(char *ID, double *Den, double *Copy, doub } } -__global__ void dvc_ComputePhi(char *ID, double *Phi, double *Den, int N) +__global__ void dvc_ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N) { int n; double Na,Nb; @@ -1401,18 +1401,18 @@ extern "C" void SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, dvc_SetPhiSlice_z<<>>(Phi,value,Nx,Ny,Nz,Slice); } -extern "C" void InitDenColor(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz){ - dvc_InitDenColor<<>>(ID, Den, Phi, das, dbs, Nx, Ny, Nz); +extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz){ + dvc_ScaLBL_Color_Init<<>>(ID, Den, Phi, das, dbs, Nx, Ny, Nz); } -extern "C" void InitDenColorDistance(char *ID, double *Den, double *Phi, double *Distance, +extern "C" void ScaLBL_Color_InitDistance(char *ID, double *Den, double *Phi, double *Distance, double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz){ - dvc_InitDenColorDistance<<>>(ID, Den, Phi, Distance, das, dbs, beta, xp, Nx, Ny, Nz); + dvc_ScaLBL_Color_InitDistance<<>>(ID, Den, Phi, Distance, das, dbs, beta, xp, Nx, Ny, Nz); } -extern "C" void ComputeColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz){ - dvc_ComputeColorGradient<<>>(ID, phi, ColorGrad, Nx, Ny, Nz); +extern "C" void ScaLBL_D3Q19_ColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz){ + dvc_ScaLBL_D3Q19_ColorGradient<<>>(ID, phi, ColorGrad, Nx, Ny, Nz); } extern "C" void ColorCollide( char *ID, double *disteven, double *distodd, double *ColorGrad, @@ -1423,10 +1423,10 @@ extern "C" void ColorCollide( char *ID, double *disteven, double *distodd, doubl } -extern "C" void ColorCollideOpt( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, +extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB, double alpha, double beta, double Fx, double Fy, double Fz){ - dvc_ColorCollideOpt<<>>(ID, disteven, distodd, phi, ColorGrad, Velocity, Nx, Ny, Nz, rlx_setA, rlx_setB, + dvc_ScaLBL_D3Q19_ColorCollide<<>>(ID, disteven, distodd, phi, ColorGrad, Velocity, Nx, Ny, Nz, rlx_setA, rlx_setB, alpha, beta, Fx, Fy, Fz); } @@ -1437,32 +1437,32 @@ extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *P dvc_DensityStreamD3Q7<<>>(ID, Den, Copy, Phi, ColorGrad, Velocity, beta, Nx, Ny, Nz, pBC); } -extern "C" void ComputePhi(char *ID, double *Phi, double *Den, int N){ - dvc_ComputePhi<<>>(ID, Phi, Den, N); +extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N){ + dvc_ScaLBL_ComputePhaseField<<>>(ID, Phi, Den, N); } -extern "C" void MassColorCollideD3Q7(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, +extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC){ - dvc_MassColorCollideD3Q7<<>>(ID, A_even, A_odd, B_even, B_odd, Den, Phi, ColorGrad, Velocity, beta, N, pBC); + dvc_ScaLBL_D3Q7_ColorCollideMass<<>>(ID, A_even, A_odd, B_even, B_odd, Den, Phi, ColorGrad, Velocity, beta, N, pBC); } // Pressure Boundary Conditions Functions -extern "C" void ColorBC_inlet(double *Phi, double *Den, double *A_even, double *A_odd, +extern "C" void ScaLBL_Color_BC_z(double *Phi, double *Den, double *A_even, double *A_odd, double *B_even, double *B_odd, int Nx, int Ny, int Nz){ int GRID = Nx*Ny / 512 + 1; - dvc_ColorBC_inlet<<< GRID,512 >>>(Phi, Den, A_even, A_odd, B_even, B_odd, Nx, Ny, Nz); + dvc_ScaLBL_Color_BC_z<<< GRID,512 >>>(Phi, Den, A_even, A_odd, B_even, B_odd, Nx, Ny, Nz); } -extern "C" void ColorBC_outlet(double *Phi, double *Den, double *A_even, double *A_odd, +extern "C" void ScaLBL_Color_BC_Z(double *Phi, double *Den, double *A_even, double *A_odd, double *B_even, double *B_odd, int Nx, int Ny, int Nz){ int GRID = Nx*Ny / 512 + 1; - dvc_ColorBC_outlet<<< GRID,512 >>>(Phi, Den, A_even, A_odd, B_even, B_odd, Nx, Ny, Nz); + dvc_ScaLBL_Color_BC_Z<<< GRID,512 >>>(Phi, Den, A_even, A_odd, B_even, B_odd, Nx, Ny, Nz); } -extern "C" void PressureBC_inlet(double *disteven, double *distodd, double din, int Nx, int Ny, int Nz){ +extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din, int Nx, int Ny, int Nz){ int GRID = Nx*Ny / 512 + 1; - dvc_PressureBC_inlet<<>>(disteven, distodd, din, Nx, Ny, Nz); + dvc_ScaLBL_D3Q19_Pressure_BC_z<<>>(disteven, distodd, din, Nx, Ny, Nz); } -extern "C" void PressureBC_outlet(double *disteven, double *distodd, double dout, +extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout, int Nx, int Ny, int Nz, int outlet){ int GRID = Nx*Ny / 512 + 1; - dvc_PressureBC_outlet<<>>(disteven, distodd, dout, Nx, Ny, Nz, outlet); + dvc_ScaLBL_D3Q19_Pressure_BC_Z<<>>(disteven, distodd, dout, Nx, Ny, Nz, outlet); } diff --git a/gpu/CudaExtras.cu b/gpu/CudaExtras.cu index c08b46f5..4a31a466 100644 --- a/gpu/CudaExtras.cu +++ b/gpu/CudaExtras.cu @@ -1,17 +1,17 @@ // Basic cuda functions callable from C/C++ code #include -extern "C" void dvc_AllocateDeviceMemory(void** address, size_t size){ +extern "C" void dvc_ScaLBL_ScaLBL_ScaLBL_AllocateDeviceMemory(void** address, size_t size){ cudaMalloc(address,size); cudaMemset(*address,0,size); } -extern "C" void dvc_CopyToDevice(void* dest, void* source, size_t size){ +extern "C" void dvc_ScaLBL_ScaLBL_CopyToDevice(void* dest, void* source, size_t size){ cudaMemcpy(dest,source,size,cudaMemcpyHostToDevice); } -extern "C" void dvc_CopyToHost(void* dest, void* source, size_t size){ +extern "C" void dvc_ScaLBL_CopyToHost(void* dest, void* source, size_t size){ cudaMemcpy(dest,source,size,cudaMemcpyDeviceToHost); } diff --git a/gpu/D3Q19.cu b/gpu/D3Q19.cu index 0643fbb5..5770963e 100644 --- a/gpu/D3Q19.cu +++ b/gpu/D3Q19.cu @@ -2,7 +2,7 @@ #define NBLOCKS 32 #define NTHREADS 128 -__global__ void dvc_PackDist(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){ +__global__ void dvc_ScaLBL_D3Q19_Pack(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){ //.................................................................................... // Pack distribution q into the send buffer for the listed lattice sites // dist may be even or odd distributions stored by stream layout @@ -15,7 +15,7 @@ __global__ void dvc_PackDist(int q, int *list, int start, int count, double *se } } -__global__ void dvc_UnpackDist(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, +__global__ void dvc_ScaLBL_D3Q19_Unpack(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, double *recvbuf, double *dist, int Nx, int Ny, int Nz){ //.................................................................................... // Unack distribution from the recv buffer @@ -59,7 +59,7 @@ __global__ void dvc_UnpackDist(int q, int Cqx, int Cqy, int Cqz, int *list, int } } -__global__ void dvc_InitD3Q19(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz) +__global__ void dvc_ScaLBL_D3Q19_Init(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz) { int n,N; N = Nx*Ny*Nz; @@ -103,7 +103,7 @@ __global__ void dvc_InitD3Q19(char *ID, double *f_even, double *f_odd, int Nx, i } //************************************************************************* -__global__ void dvc_SwapD3Q19(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz) +__global__ void dvc_ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz) { int i,j,k,n,nn,N; // distributions @@ -244,7 +244,7 @@ __global__ void dvc_SwapD3Q19(char *ID, double *disteven, double *distodd, int } -__global__ void dvc_ComputeVelocityD3Q19(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz) +__global__ void dvc_ScaLBL_D3Q19_Velocity(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz) { int n,N; // distributions @@ -305,7 +305,7 @@ __global__ void dvc_ComputeVelocityD3Q19(char *ID, double *disteven, double *di } } -__global__ void dvc_ComputePressureD3Q19(const char *ID, const double *disteven, const double *distodd, +__global__ void dvc_ScaLBL_D3Q19_Pressure(const char *ID, const double *disteven, const double *distodd, double *Pressure, int Nx, int Ny, int Nz) { int n,N; @@ -466,39 +466,39 @@ __global__ void dvc_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, doubl } } -extern "C" void PackDist(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){ +extern "C" void ScaLBL_D3Q19_Pack(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){ int GRID = count / 512 + 1; - dvc_PackDist <<>>(q, list, start, count, sendbuf, dist, N); + dvc_ScaLBL_D3Q19_Pack <<>>(q, list, start, count, sendbuf, dist, N); } -extern "C" void UnpackDist(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, +extern "C" void ScaLBL_D3Q19_Unpack(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, double *recvbuf, double *dist, int Nx, int Ny, int Nz){ int GRID = count / 512 + 1; - dvc_UnpackDist <<>>(q, Cqx, Cqy, Cqz, list, start, count, recvbuf, dist, Nx, Ny, Nz); + dvc_ScaLBL_D3Q19_Unpack <<>>(q, Cqx, Cqy, Cqz, list, start, count, recvbuf, dist, Nx, Ny, Nz); } //************************************************************************* -extern "C" void InitD3Q19(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz){ - dvc_InitD3Q19<<>>(ID, f_even, f_odd, Nx, Ny, Nz); +extern "C" void ScaLBL_D3Q19_Init(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz){ + dvc_ScaLBL_D3Q19_Init<<>>(ID, f_even, f_odd, Nx, Ny, Nz); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in InitD3Q19: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err)); } } -extern "C" void SwapD3Q19(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){ - dvc_SwapD3Q19<<>>(ID, disteven, distodd, Nx, Ny, Nz); +extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){ + dvc_ScaLBL_D3Q19_Swap<<>>(ID, disteven, distodd, Nx, Ny, Nz); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in SwapD3Q19: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_Swap: %s \n",cudaGetErrorString(err)); } } -extern "C" void ComputeVelocityD3Q19(char *ID, double *disteven, double *distodd, double *vel, int Nx, int \ +extern "C" void ScaLBL_D3Q19_Velocity(char *ID, double *disteven, double *distodd, double *vel, int Nx, int \ Ny, int Nz){ - dvc_ComputeVelocityD3Q19<<>>(ID, disteven, distodd, vel, Nx, Ny, Nz); + dvc_ScaLBL_D3Q19_Velocity<<>>(ID, disteven, distodd, vel, Nx, Ny, Nz); } -extern "C" void ComputePressureD3Q19(char *ID, double *disteven, double *distodd, double *Pressure, +extern "C" void ScaLBL_D3Q19_Pressure(char *ID, double *disteven, double *distodd, double *Pressure, int Nx, int Ny, int Nz){ - dvc_ComputePressureD3Q19<<< NBLOCKS,NTHREADS >>>(ID, disteven, distodd, Pressure, Nx, Ny, Nz); + dvc_ScaLBL_D3Q19_Pressure<<< NBLOCKS,NTHREADS >>>(ID, disteven, distodd, Pressure, Nx, Ny, Nz); } extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz,int Nx, int Ny, int Nz){ diff --git a/gpu/D3Q7.cu b/gpu/D3Q7.cu index 72cb70d0..613a302e 100644 --- a/gpu/D3Q7.cu +++ b/gpu/D3Q7.cu @@ -3,7 +3,7 @@ #define NBLOCKS 32 #define NTHREADS 128 -__global__ void dvc_PackValues(int *list, int count, double *sendbuf, double *Data, int N){ +__global__ void dvc_ScaLBL_D3Q19_Unpack(int *list, int count, double *sendbuf, double *Data, int N){ //.................................................................................... // Pack distribution q into the send buffer for the listed lattice sites // dist may be even or odd distributions stored by stream layout @@ -15,7 +15,7 @@ __global__ void dvc_PackValues(int *list, int count, double *sendbuf, double *D sendbuf[idx] = Data[n]; } } -__global__ void dvc_UnpackValues(int *list, int count, double *recvbuf, double *Data, int N){ +__global__ void dvc_ScaLBL_Scalar_Unpack(int *list, int count, double *recvbuf, double *Data, int N){ //.................................................................................... // Pack distribution q into the send buffer for the listed lattice sites // dist may be even or odd distributions stored by stream layout @@ -28,7 +28,7 @@ __global__ void dvc_UnpackValues(int *list, int count, double *recvbuf, double } } -__global__ void dvc_PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){ +__global__ void dvc_ScaLBL_PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){ //.................................................................................... // Pack distribution into the send buffer for the listed lattice sites //.................................................................................... @@ -44,7 +44,7 @@ __global__ void dvc_PackDenD3Q7(int *list, int count, double *sendbuf, int numb } -__global__ void dvc_UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){ +__global__ void dvc_ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){ //.................................................................................... // Unack distribution from the recv buffer // Sum to the existing density value @@ -59,7 +59,7 @@ __global__ void dvc_UnpackDenD3Q7(int *list, int count, double *recvbuf, int num } } -__global__ void dvc_InitD3Q7(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz) +__global__ void dvc_ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz) { int n,N; N = Nx*Ny*Nz; @@ -93,7 +93,7 @@ __global__ void dvc_InitD3Q7(char *ID, double *f_even, double *f_odd, double *De } //************************************************************************* -__global__ void dvc_SwapD3Q7(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz) +__global__ void dvc_ScaLBL_D3Q7_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz) { int i,j,k,n,nn,N; // distributions @@ -157,7 +157,7 @@ __global__ void dvc_SwapD3Q7(char *ID, double *disteven, double *distodd, int N } //************************************************************************* -__global__ void dvc_ComputeDensityD3Q7(char *ID, double *disteven, double *distodd, double *Den, +__global__ void dvc_ScaLBL_D3Q7_Density(char *ID, double *disteven, double *distodd, double *Den, int Nx, int Ny, int Nz) { char id; @@ -187,35 +187,35 @@ __global__ void dvc_ComputeDensityD3Q7(char *ID, double *disteven, double *dist } } -extern "C" void PackValues(int *list, int count, double *sendbuf, double *Data, int N){ +extern "C" void ScaLBL_D3Q19_Unpack(int *list, int count, double *sendbuf, double *Data, int N){ int GRID = count / 512 + 1; - dvc_PackValues <<>>(list, count, sendbuf, Data, N); + dvc_ScaLBL_D3Q19_Unpack <<>>(list, count, sendbuf, Data, N); } -extern "C" void UnpackValues(int *list, int count, double *recvbuf, double *Data, int N){ +extern "C" void ScaLBL_Scalar_Unpack(int *list, int count, double *recvbuf, double *Data, int N){ int GRID = count / 512 + 1; - dvc_UnpackValues <<>>(list, count, recvbuf, Data, N); + dvc_ScaLBL_Scalar_Unpack <<>>(list, count, recvbuf, Data, N); } -extern "C" void PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){ +extern "C" void ScaLBL_PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){ int GRID = count / 512 + 1; - dvc_PackDenD3Q7 <<>>(list, count, sendbuf, number, Data, N); + dvc_ScaLBL_PackDenD3Q7 <<>>(list, count, sendbuf, number, Data, N); } -extern "C" void UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){ +extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){ int GRID = count / 512 + 1; - dvc_UnpackDenD3Q7 <<>>(list, count, recvbuf, number, Data, N); + dvc_ScaLBL_UnpackDenD3Q7 <<>>(list, count, recvbuf, number, Data, N); } -extern "C" void InitD3Q7(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz){ - dvc_InitD3Q7 <<>>(ID, f_even, f_odd, Den, Nx, Ny, Nz); +extern "C" void ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz){ + dvc_ScaLBL_D3Q7_Init <<>>(ID, f_even, f_odd, Den, Nx, Ny, Nz); } -extern "C" void SwapD3Q7(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){ - dvc_SwapD3Q7 <<>>(ID, disteven, distodd, Nx, Ny, Nz); +extern "C" void ScaLBL_D3Q7_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){ + dvc_ScaLBL_D3Q7_Swap <<>>(ID, disteven, distodd, Nx, Ny, Nz); } -extern "C" void ComputeDensityD3Q7(char *ID, double *disteven, double *distodd, double *Den, +extern "C" void ScaLBL_D3Q7_Density(char *ID, double *disteven, double *distodd, double *Den, int Nx, int Ny, int Nz){ - dvc_ComputeDensityD3Q7 <<>>(ID, disteven, distodd, Den, Nx, Ny, Nz); + dvc_ScaLBL_D3Q7_Density <<>>(ID, disteven, distodd, Den, Nx, Ny, Nz); } diff --git a/gpu/Extras.cu b/gpu/Extras.cu index 21ff53a0..dffd3704 100644 --- a/gpu/Extras.cu +++ b/gpu/Extras.cu @@ -2,7 +2,7 @@ #include #include -extern "C" void AllocateDeviceMemory(void** address, size_t size){ +extern "C" void ScaLBL_ScaLBL_ScaLBL_AllocateDeviceMemory(void** address, size_t size){ cudaMalloc(address,size); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -10,7 +10,7 @@ extern "C" void AllocateDeviceMemory(void** address, size_t size){ } } -extern "C" void CopyToDevice(void* dest, const void* source, size_t size){ +extern "C" void ScaLBL_ScaLBL_CopyToDevice(void* dest, const void* source, size_t size){ cudaMemcpy(dest,source,size,cudaMemcpyHostToDevice); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -19,7 +19,7 @@ extern "C" void CopyToDevice(void* dest, const void* source, size_t size){ } -extern "C" void CopyToHost(void* dest, const void* source, size_t size){ +extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size){ cudaMemcpy(dest,source,size,cudaMemcpyDeviceToHost); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -27,6 +27,6 @@ extern "C" void CopyToHost(void* dest, const void* source, size_t size){ } } -extern "C" void DeviceBarrier(){ +extern "C" void ScaLBL_DeviceBarrier(){ cudaDeviceSynchronize(); } diff --git a/gpu/MRT.cu b/gpu/MRT.cu index 25bf661d..1cc556ef 100644 --- a/gpu/MRT.cu +++ b/gpu/MRT.cu @@ -1,5 +1,5 @@ //************************************************************************* -// CUDA kernels for single-phase MRT code +// CUDA kernels for single-phase ScaLBL_D3Q19_MRT code // James McClure //************************************************************************* #include @@ -106,7 +106,7 @@ __global__ void Compute_VELOCITY(char *ID, double *disteven, double *distodd, do } //************************************************************************* -__global__ void MRT(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz, +__global__ void D3Q19_MRT(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz) { @@ -300,9 +300,9 @@ __global__ void MRT(char *ID, double *disteven, double *distodd, int Nx, int Ny, } } -extern "C" void MRT(char *ID, double *f_even, double *f_odd, double rlxA, double rlxB, double Fx, double Fy, double Fz,int Nx, int Ny, int Nz) +extern "C" void ScaLBL_D3Q19_MRT(char *ID, double *f_even, double *f_odd, double rlxA, double rlxB, double Fx, double Fy, double Fz,int Nx, int Ny, int Nz) { - MRT <<< NBLOCKS,NTHREADS>>> (ID, f_even, f_odd, Nx, Ny, Nz, rlxA, rlxB, Fx, Fy, Fz); + D3Q19_MRT <<< NBLOCKS,NTHREADS>>> (ID, f_even, f_odd, Nx, Ny, Nz, rlxA, rlxB, Fx, Fy, Fz); } diff --git a/gpu/Multiphase.cu b/gpu/Multiphase.cu new file mode 100644 index 00000000..8d91c8c5 --- /dev/null +++ b/gpu/Multiphase.cu @@ -0,0 +1,1468 @@ +#include + +#define NBLOCKS 32 +#define NTHREADS 128 + +__global__ void dvc_ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) +{ + //int i,j,k; + int n,N; + char id; + N = Nx*Ny*Nz; + + int S = N/NBLOCKS/NTHREADS + 1; + for (int s=0; s 0){ + + // Retrieve the color gradient + nx = ColorGrad[n]; + ny = ColorGrad[N+n]; + nz = ColorGrad[2*N+n]; + //...........Normalize the Color Gradient................................. + C = sqrt(nx*nx+ny*ny+nz*nz); + nx = nx/C; + ny = ny/C; + nz = nz/C; + //......No color gradient at z-boundary if pressure BC are set............. + // if (pBC && k==0) nx = ny = nz = 0.f; + // if (pBC && k==Nz-1) nx = ny = nz = 0.f; + //........................................................................ + // READ THE DISTRIBUTIONS + // (read from opposite array due to previous swap operation) + //........................................................................ + f2 = distodd[n]; + f4 = distodd[N+n]; + f6 = distodd[2*N+n]; + f8 = distodd[3*N+n]; + f10 = distodd[4*N+n]; + f12 = distodd[5*N+n]; + f14 = distodd[6*N+n]; + f16 = distodd[7*N+n]; + f18 = distodd[8*N+n]; + //........................................................................ + f0 = disteven[n]; + f1 = disteven[N+n]; + f3 = disteven[2*N+n]; + f5 = disteven[3*N+n]; + f7 = disteven[4*N+n]; + f9 = disteven[5*N+n]; + f11 = disteven[6*N+n]; + f13 = disteven[7*N+n]; + f15 = disteven[8*N+n]; + f17 = disteven[9*N+n]; + //........................................................................ + // PERFORM RELAXATION PROCESS + //........................................................................ + //....................compute the moments............................................... + rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + m1 = -30*f0-11*(f2+f1+f4+f3+f6+f5)+8*(f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18 +f17); + m2 = 12*f0-4*(f2+f1 +f4+f3+f6 +f5)+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + jx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; + m4 = 4*(-f1+f2)+f7-f8+f9-f10+f11-f12+f13-f14; + jy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; + m6 = -4*(f3-f4)+f7-f8-f9+f10+f15-f16+f17-f18; + jz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18; + m8 = -4*(f5-f6)+f11-f12-f13+f14+f15-f16-f17+f18; + m9 = 2*(f1+f2)-f3-f4-f5-f6+f7+f8+f9+f10+f11+f12+f13+f14-2*(f15+f16+f17+f18); + m10 = -4*(f1+f2)+2*(f4+f3+f6+f5)+f8+f7+f10+f9+f12+f11+f14+f13-2*(f16+f15+f18+f17); + m11 = f4+f3-f6-f5+f8+f7+f10+f9-f12-f11-f14-f13; + m12 = -2*(f4+f3-f6-f5)+f8+f7+f10+f9-f12-f11-f14-f13; + m13 = f8+f7-f10-f9; + m14 = f16+f15-f18-f17; + m15 = f12+f11-f14-f13; + m16 = f7-f8+f9-f10-f11+f12-f13+f14; + m17 = -f7+f8+f9-f10+f15-f16+f17-f18; + m18 = f11-f12-f13+f14-f15+f16+f17-f18; + //..........Toelke, Fruediger et. al. 2006............... + if (C == 0.0) nx = ny = nz = 1.0; + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) -alpha*C - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho)- m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); + m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) + 0.5*alpha*C*(ny*ny-nz*nz)- m11); + m12 = m12 + rlx_setA*( -0.5*((jy*jy-jz*jz)/rho) - m12); + m13 = m13 + rlx_setA*( (jx*jy/rho) + 0.5*alpha*C*nx*ny - m13); + m14 = m14 + rlx_setA*( (jy*jz/rho) + 0.5*alpha*C*ny*nz - m14); + m15 = m15 + rlx_setA*( (jx*jz/rho) + 0.5*alpha*C*nx*nz - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //.................inverse transformation...................................................... + f0 = 0.05263157894736842*rho-0.012531328320802*m1+0.04761904761904762*m2; + f1 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(jx-m4)+0.0555555555555555555555555*(m9-m10); + f2 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(m4-jx)+0.0555555555555555555555555*(m9-m10); + f3 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(jy-m6)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12); + f4 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(m6-jy)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12); + f5 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(jz-m8)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11); + f6 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 + +0.1*(m8-jz)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11); + f7 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx+jy)+0.025*(m4+m6) + +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 + +0.04166666666666666*m12+0.25*m13+0.125*(m16-m17); + f8 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2-0.1*(jx+jy)-0.025*(m4+m6) + +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 + +0.04166666666666666*m12+0.25*m13+0.125*(m17-m16); + f9 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx-jy)+0.025*(m4-m6) + +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 + +0.04166666666666666*m12-0.25*m13+0.125*(m16+m17); + f10 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jy-jx)+0.025*(m6-m4) + +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 + +0.04166666666666666*m12-0.25*m13-0.125*(m16+m17); + f11 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jx+jz)+0.025*(m4+m8) + +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 + -0.04166666666666666*m12+0.25*m15+0.125*(m18-m16); + f12 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2-0.1*(jx+jz)-0.025*(m4+m8) + +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 + -0.04166666666666666*m12+0.25*m15+0.125*(m16-m18); + f13 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jx-jz)+0.025*(m4-m8) + +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 + -0.04166666666666666*m12-0.25*m15-0.125*(m16+m18); + f14 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jz-jx)+0.025*(m8-m4) + +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 + -0.04166666666666666*m12-0.25*m15+0.125*(m16+m18); + f15 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jy+jz)+0.025*(m6+m8) + -0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m17-m18); + f16 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2-0.1*(jy+jz)-0.025*(m6+m8) + -0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m18-m17); + f17 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jy-jz)+0.025*(m6-m8) + -0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14+0.125*(m17+m18); + f18 = 0.05263157894736842*rho+0.003341687552213868*m1 + +0.003968253968253968*m2+0.1*(jz-jy)+0.025*(m8-m6) + -0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14-0.125*(m17+m18); + //....................................................................................................... + // incorporate external force + f1 += 0.16666666*Fx; + f2 -= 0.16666666*Fx; + f3 += 0.16666666*Fy; + f4 -= 0.16666666*Fy; + f5 += 0.16666666*Fz; + f6 -= 0.16666666*Fz; + f7 += 0.08333333333*(Fx+Fy); + f8 -= 0.08333333333*(Fx+Fy); + f9 += 0.08333333333*(Fx-Fy); + f10 -= 0.08333333333*(Fx-Fy); + f11 += 0.08333333333*(Fx+Fz); + f12 -= 0.08333333333*(Fx+Fz); + f13 += 0.08333333333*(Fx-Fz); + f14 -= 0.08333333333*(Fx-Fz); + f15 += 0.08333333333*(Fy+Fz); + f16 -= 0.08333333333*(Fy+Fz); + f17 += 0.08333333333*(Fy-Fz); + f18 -= 0.08333333333*(Fy-Fz); + //*********** WRITE UPDATED VALUES TO MEMORY ****************** + // Write the updated distributions + //....EVEN..................................... + disteven[n] = f0; + disteven[N+n] = f2; + disteven[2*N+n] = f4; + disteven[3*N+n] = f6; + disteven[4*N+n] = f8; + disteven[5*N+n] = f10; + disteven[6*N+n] = f12; + disteven[7*N+n] = f14; + disteven[8*N+n] = f16; + disteven[9*N+n] = f18; + //....ODD...................................... + distodd[n] = f1; + distodd[N+n] = f3; + distodd[2*N+n] = f5; + distodd[3*N+n] = f7; + distodd[4*N+n] = f9; + distodd[5*N+n] = f11; + distodd[6*N+n] = f13; + distodd[7*N+n] = f15; + distodd[8*N+n] = f17; + + //...Store the Velocity.......................... + Velocity[n] = jx; + Velocity[N+n] = jy; + Velocity[2*N+n] = jz; + /* Velocity[3*n] = jx; + Velocity[3*n+1] = jy; + Velocity[3*n+2] = jz; + */ //...Store the Color Gradient.................... + // ColorGrad[3*n] = nx*C; + // ColorGrad[3*n+1] = ny*C; + // ColorGrad[3*n+2] = nz*C; + //............................................... + //*************************************************************** + } // check if n is in the solid + } // loop over n +} + +__global__ void dvc_ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, + double *Velocity, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB, + double alpha, double beta, double Fx, double Fy, double Fz) +{ + + int i,j,k,n,nn,N; + // distributions + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + // additional variables needed for computations + double rho,jx,jy,jz,C,nx,ny,nz; + char id; + + N = Nx*Ny*Nz; + + int S = N/NBLOCKS/NTHREADS + 1; + for (int s=0; s 0){ + + //.......Back out the 3-D indices for node n.............. + k = n/(Nx*Ny); + j = (n-Nx*Ny*k)/Nx; + i = n-Nx*Ny*k-Nx*j; + //........................................................................ + //........Get 1-D index for this thread.................... + // n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x; + //........................................................................ + // COMPUTE THE COLOR GRADIENT + //........................................................................ + //.................Read Phase Indicator Values............................ + //........................................................................ + nn = n-1; // neighbor index (get convention) + if (i-1<0) nn += Nx; // periodic BC along the x-boundary + f1 = phi[nn]; // get neighbor for phi - 1 + //........................................................................ + nn = n+1; // neighbor index (get convention) + if (!(i+10)) delta=0; + a1 = na*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nb*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = na*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nb*(0.1111111111111111*(1-4.5*ux))+delta; + + A_odd[n] = a1; + A_even[N+n] = a2; + B_odd[n] = b1; + B_even[N+n] = b2; + //............................................... + // q = 2 + // Cq = {0,1,0} + delta = beta*na*nb*nab*0.1111111111111111*ny; + if (!(na*nb*nab>0)) delta=0; + a1 = na*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nb*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = na*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nb*(0.1111111111111111*(1-4.5*uy))+delta; + + A_odd[N+n] = a1; + A_even[2*N+n] = a2; + B_odd[N+n] = b1; + B_even[2*N+n] = b2; + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*na*nb*nab*0.1111111111111111*nz; + if (!(na*nb*nab>0)) delta=0; + a1 = na*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nb*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = na*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nb*(0.1111111111111111*(1-4.5*uz))+delta; + + A_odd[2*N+n] = a1; + A_even[3*N+n] = a2; + B_odd[2*N+n] = b1; + B_even[3*N+n] = b2; + + } + } +} + +//************************************************************************* +__global__ void dvc_DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, + double beta, int Nx, int Ny, int Nz, bool pBC) +{ + char id; + + int idx; + int in,jn,kn,n,nn,N; + int q,Cqx,Cqy,Cqz; + // int sendLoc; + + double na,nb; // density values + double ux,uy,uz; // flow velocity + double nx,ny,nz,C; // color gradient components + double a1,a2,b1,b2; + double sp,delta; + double feq[6]; // equilibrium distributions + // Set of Discrete velocities for the D3Q19 Model + int D3Q7[3][3]={{1,0,0},{0,1,0},{0,0,1}}; + N = Nx*Ny*Nz; + + int S = N/NBLOCKS/NTHREADS + 1; + for (int s=0; s 0 && na+nb > 0.0){ + //.......Back out the 3-D indices for node n.............. + int k = n/(Nx*Ny); + int j = (n-Nx*Ny*k)/Nx; + int i = n-Nx*Ny*k-Nx*j; + //.....Load the Color gradient......... + nx = ColorGrad[n]; + ny = ColorGrad[N+n]; + nz = ColorGrad[2*N+n]; + C = sqrt(nx*nx+ny*ny+nz*nz); + if (C == 0.0) C=1.0; + nx = nx/C; + ny = ny/C; + nz = nz/C; + //....Load the flow velocity........... + ux = Velocity[n]; + uy = Velocity[N+n]; + uz = Velocity[2*N+n]; + //....Instantiate the density distributions + // Generate Equilibrium Distributions and stream + // Stationary value - distribution 0 + // Den[2*n] += 0.3333333333333333*na; + // Den[2*n+1] += 0.3333333333333333*nb; + Den[2*n] += 0.3333333333333333*na; + Den[2*n+1] += 0.3333333333333333*nb; + // Non-Stationary equilibrium distributions + feq[0] = 0.1111111111111111*(1+3*ux); + feq[1] = 0.1111111111111111*(1-3*ux); + feq[2] = 0.1111111111111111*(1+3*uy); + feq[3] = 0.1111111111111111*(1-3*uy); + feq[4] = 0.1111111111111111*(1+3*uz); + feq[5] = 0.1111111111111111*(1-3*uz); + // Construction and streaming for the components + for (idx=0; idx<3; idx++){ + // Distribution index + q = 2*idx; + // Associated discrete velocity + Cqx = D3Q7[idx][0]; + Cqy = D3Q7[idx][1]; + Cqz = D3Q7[idx][2]; + // Generate the Equilibrium Distribution + a1 = na*feq[q]; + b1 = nb*feq[q]; + a2 = na*feq[q+1]; + b2 = nb*feq[q+1]; + // Recolor the distributions + if (C > 0.0){ + sp = nx*double(Cqx)+ny*double(Cqy)+nz*double(Cqz); + //if (idx > 2) sp = 0.7071067811865475*sp; + //delta = sp*min( min(a1,a2), min(b1,b2) ); + delta = na*nb/(na+nb)*0.1111111111111111*sp; + //if (a1>0 && b1>0){ + a1 += beta*delta; + a2 -= beta*delta; + b1 -= beta*delta; + b2 += beta*delta; + } + + // .......Get the neighbor node.............. + //nn = n + Stride[idx]; + in = i+Cqx; + jn = j+Cqy; + kn = k+Cqz; + + // Adjust for periodic BC, if necessary + // if (in<0) in+= Nx; + // if (jn<0) jn+= Ny; + // if (kn<0) kn+= Nz; + // if (!(in 0){ + // Get the density value (Streaming already performed) + Na = Den[n]; + Nb = Den[N+n]; + Phi[n] = (Na-Nb)/(Na+Nb); + } + } + } + //................................................................... +} + +__global__ void dvc_SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice) +{ + int n = Slice*Nx*Ny + blockIdx.x*blockDim.x + threadIdx.x; + if (n < (Slice+1)*Nx*Ny){ + Phi[n] = value; + } +} + +extern "C" void SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){ + int GRID = Nx*Ny / 512 + 1; + dvc_SetPhiSlice_z<<>>(Phi,value,Nx,Ny,Nz,Slice); +} + +extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz){ + dvc_ScaLBL_Color_Init<<>>(ID, Den, Phi, das, dbs, Nx, Ny, Nz); +} + +extern "C" void ScaLBL_Color_InitDistance(char *ID, double *Den, double *Phi, double *Distance, + double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz){ + + dvc_ScaLBL_Color_InitDistance<<>>(ID, Den, Phi, Distance, das, dbs, beta, xp, Nx, Ny, Nz); +} + +extern "C" void ScaLBL_D3Q19_ColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz){ + dvc_ScaLBL_D3Q19_ColorGradient<<>>(ID, phi, ColorGrad, Nx, Ny, Nz); +} + +extern "C" void ColorCollide( char *ID, double *disteven, double *distodd, double *ColorGrad, + double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB, + double alpha, double beta, double Fx, double Fy, double Fz, bool pBC){ + dvc_ColorCollide<<>>( ID, disteven, distodd, ColorGrad,Velocity, Nx, Ny, Nz,rlx_setA, rlx_setB, + alpha, beta, Fx, Fy, Fz, pBC); + +} + +extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, + double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB, + double alpha, double beta, double Fx, double Fy, double Fz){ + dvc_ScaLBL_D3Q19_ColorCollide<<>>(ID, disteven, distodd, phi, ColorGrad, Velocity, Nx, Ny, Nz, rlx_setA, rlx_setB, + alpha, beta, Fx, Fy, Fz); + +} + +extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, + double beta, int Nx, int Ny, int Nz, bool pBC){ + + dvc_DensityStreamD3Q7<<>>(ID, Den, Copy, Phi, ColorGrad, Velocity, beta, Nx, Ny, Nz, pBC); +} + +extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N){ + dvc_ScaLBL_ComputePhaseField<<>>(ID, Phi, Den, N); +} +extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, + double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC){ + dvc_ScaLBL_D3Q7_ColorCollideMass<<>>(ID, A_even, A_odd, B_even, B_odd, Den, Phi, ColorGrad, Velocity, beta, N, pBC); +} +// Pressure Boundary Conditions Functions +extern "C" void ScaLBL_Color_BC_z(double *Phi, double *Den, double *A_even, double *A_odd, + double *B_even, double *B_odd, int Nx, int Ny, int Nz){ + int GRID = Nx*Ny / 512 + 1; + dvc_ScaLBL_Color_BC_z<<< GRID,512 >>>(Phi, Den, A_even, A_odd, B_even, B_odd, Nx, Ny, Nz); +} +extern "C" void ScaLBL_Color_BC_Z(double *Phi, double *Den, double *A_even, double *A_odd, + double *B_even, double *B_odd, int Nx, int Ny, int Nz){ + + int GRID = Nx*Ny / 512 + 1; + dvc_ScaLBL_Color_BC_Z<<< GRID,512 >>>(Phi, Den, A_even, A_odd, B_even, B_odd, Nx, Ny, Nz); +} +extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din, int Nx, int Ny, int Nz){ + int GRID = Nx*Ny / 512 + 1; + dvc_ScaLBL_D3Q19_Pressure_BC_z<<>>(disteven, distodd, din, Nx, Ny, Nz); +} +extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout, + int Nx, int Ny, int Nz, int outlet){ + int GRID = Nx*Ny / 512 + 1; + dvc_ScaLBL_D3Q19_Pressure_BC_Z<<>>(disteven, distodd, dout, Nx, Ny, Nz, outlet); +} + From 51e06f1df86050a5bb51503fdee0460e31851578 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 24 Nov 2016 11:11:06 -0500 Subject: [PATCH 5/7] remove lbpm_color_macro_simulator from project --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2e8f8d21..afc51b9a 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,7 +2,7 @@ ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_nondarcy_simulator ) ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) +#ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_sphere_pp ) ADD_LBPM_EXECUTABLE( lbpm_random_pp ) ADD_LBPM_EXECUTABLE( lbpm_morphdrain_pp ) From 9890e2a71b183b4c8b38c2a83489f48261761a43 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 24 Nov 2016 11:19:56 -0500 Subject: [PATCH 6/7] refactored lbpm_nondarcy_simulator to match ScaLBL conventions --- tests/lbpm_nondarcy_simulator.cpp | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/lbpm_nondarcy_simulator.cpp b/tests/lbpm_nondarcy_simulator.cpp index a4c21a11..254058a9 100644 --- a/tests/lbpm_nondarcy_simulator.cpp +++ b/tests/lbpm_nondarcy_simulator.cpp @@ -34,7 +34,7 @@ int main(int argc, char **argv) printf("Pore-scale lattice Boltzmann simulator for non-Darcy flow in porous media \n \n"); printf("Simulate non-Darcy flow in porous media \n"); printf(" MPI-based lattice Boltzmann simulator \n"); - printf(" Multi-relaxation time (MRT) D3Q19 \n \n"); + printf(" Multi-relaxation time (ScaLBL_D3Q19_MRT) D3Q19 \n \n"); printf(" Launch with MPI (e.g.) \n \n"); printf(" mpirun -np $NUMPROCS $LBPM_WIA_DIR/lbpm_nondarcy_simulator \n \n"); printf("**********CITATION********** \n"); @@ -360,9 +360,9 @@ int main(int argc, char **argv) //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; - AllocateDeviceMemory((void **) &ID, N); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device - CopyToDevice(ID, id, N); + ScaLBL_CopyToDevice(ID, id, N); //........................................................................... //........................................................................... @@ -373,18 +373,18 @@ int main(int argc, char **argv) //......................device distributions................................. double *f_even,*f_odd; //........................................................................... - AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory - AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory //........................................................................... double *Velocity, *Pressure, *dvcSignDist; //........................................................................... - AllocateDeviceMemory((void **) &Pressure, dist_mem_size); - AllocateDeviceMemory((void **) &dvcSignDist, dist_mem_size); - AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &dvcSignDist, dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); //........................................................................... // Copy signed distance for device initialization - CopyToDevice(dvcSignDist, Averages.SDs.data(), dist_mem_size); + ScaLBL_CopyToDevice(dvcSignDist, Averages.SDs.data(), dist_mem_size); //........................................................................... int logcount = 0; // number of surface write-outs @@ -395,7 +395,7 @@ int main(int argc, char **argv) //........................................................................... if (rank==0) printf("Setting the distributions, size = %i\n", N); //........................................................................... - InitD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_D3Q19_Init(ID, f_even, f_odd, Nx, Ny, Nz); //...................................................................... //....................................................................... @@ -427,11 +427,11 @@ int main(int argc, char **argv) printf("Setting outlet pressure = %f \n", dout); } if (pBC && kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); } if (pBC && kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); } int timestep = 0; @@ -465,7 +465,7 @@ int main(int argc, char **argv) //************************************************************************* // Fused Color Gradient and Collision //************************************************************************* - MRT( ID,f_even,f_odd,rlxA,rlxB,Fx,Fy,Fz,Nx,Ny,Nz); + ScaLBL_D3Q19_MRT( ID,f_even,f_odd,rlxA,rlxB,Fx,Fy,Fz,Nx,Ny,Nz); //************************************************************************* //************************************************************************* @@ -474,21 +474,21 @@ int main(int argc, char **argv) //************************************************************************* // Swap the distributions for momentum transport //************************************************************************* - SwapD3Q19(ID, f_even, f_odd, Nx, Ny, Nz); + ScaLBL_D3Q19_Swap(ID, f_even, f_odd, Nx, Ny, Nz); //************************************************************************* // Wait for communications to complete and unpack the distributions ScaLBL_Comm.RecvD3Q19(f_even, f_odd); //************************************************************************* if (pBC && kproc == 0) { - PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); + ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); } if (pBC && kproc == nprocz-1){ - PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); } //................................................................................... - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // Timestep completed! @@ -506,13 +506,13 @@ int main(int argc, char **argv) //........................................................................... // Copy the phase from the GPU -> CPU //........................................................................... - DeviceBarrier(); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - ComputeVelocityD3Q19(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); - CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); - CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); - CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); + ScaLBL_CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); // Way more work than necessary -- this is just to get the solid interfacial area!! Averages.Initialize(); @@ -565,7 +565,7 @@ int main(int argc, char **argv) } //************************************************************************/ fclose(NONDARCY); - DeviceBarrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); From bba9e52911130be7d5d151ced0df5dc8673e5955 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 24 Nov 2016 11:26:51 -0500 Subject: [PATCH 7/7] Refactored LBPM-WIA to make calls consistent with ScaLBL library conventions --- cpu/Color.cpp | 2 +- cpu/D3Q7.cpp | 2 +- cpu/Extras.cpp | 4 +- cpu/Multiphase.cpp | 1481 ---------------------------- gpu/Color.cu | 2 +- gpu/D3Q7.cu | 2 +- gpu/Extras.cu | 4 +- gpu/Multiphase.cu | 1468 --------------------------- tests/TestBubble.cpp | 174 ++-- tests/TestCommD3Q19.cpp | 6 +- tests/TestMassConservationD3Q7.cpp | 18 +- tests/lbpm_color_simulator.cpp | 8 +- tests/lbpm_color_simulator.h | 22 +- 13 files changed, 122 insertions(+), 3071 deletions(-) delete mode 100644 cpu/Multiphase.cpp delete mode 100644 gpu/Multiphase.cu diff --git a/cpu/Color.cpp b/cpu/Color.cpp index c1f73fb6..405ea133 100644 --- a/cpu/Color.cpp +++ b/cpu/Color.cpp @@ -1414,7 +1414,7 @@ extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int //................................................................... } -extern "C" void SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){ +extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){ int n; for (n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){ Phi[n] = value; diff --git a/cpu/D3Q7.cpp b/cpu/D3Q7.cpp index 6a31aef1..7acef4be 100644 --- a/cpu/D3Q7.cpp +++ b/cpu/D3Q7.cpp @@ -1,6 +1,6 @@ // CPU Functions for D3Q7 Lattice Boltzmann Methods -extern "C" void ScaLBL_D3Q19_Unpack(int *list, int count, double *sendbuf, double *Data, int N){ +extern "C" void ScaLBL_Scalar_Pack(int *list, int count, double *sendbuf, double *Data, int N){ //.................................................................................... // Pack distribution q into the send buffer for the listed lattice sites // dist may be even or odd distributions stored by stream layout diff --git a/cpu/Extras.cpp b/cpu/Extras.cpp index d3abfc8f..c2b17cac 100644 --- a/cpu/Extras.cpp +++ b/cpu/Extras.cpp @@ -3,7 +3,7 @@ #include #include -extern "C" void ScaLBL_ScaLBL_ScaLBL_AllocateDeviceMemory(void** address, size_t size){ +extern "C" void ScaLBL_AllocateDeviceMemory(void** address, size_t size){ //cudaMalloc(address,size); (*address) = malloc(size); memset(*address,0,size); @@ -13,7 +13,7 @@ extern "C" void ScaLBL_ScaLBL_ScaLBL_AllocateDeviceMemory(void** address, size_t } } -extern "C" void ScaLBL_ScaLBL_CopyToDevice(void* dest, const void* source, size_t size){ +extern "C" void ScaLBL_CopyToDevice(void* dest, const void* source, size_t size){ // cudaMemcpy(dest,source,size,cudaMemcpyHostToDevice); memcpy(dest, source, size); } diff --git a/cpu/Multiphase.cpp b/cpu/Multiphase.cpp deleted file mode 100644 index c1f73fb6..00000000 --- a/cpu/Multiphase.cpp +++ /dev/null @@ -1,1481 +0,0 @@ -#include - -#define STOKES - -extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) -{ - int n,N; - - N = Nx*Ny*Nz; - - for (n=0; n 0){ - - // Retrieve the color gradient - nx = ColorGrad[n]; - ny = ColorGrad[N+n]; - nz = ColorGrad[2*N+n]; - //...........Normalize the Color Gradient................................. - C = sqrt(nx*nx+ny*ny+nz*nz); - if (C==0.0) C=1.0; - nx = nx/C; - ny = ny/C; - nz = nz/C; - //......No color gradient at z-boundary if pressure BC are set............. - // if (pBC && k==0) nx = ny = nz = 0.f; - // if (pBC && k==Nz-1) nx = ny = nz = 0.f; - //........................................................................ - // READ THE DISTRIBUTIONS - // (read from opposite array due to previous swap operation) - //........................................................................ - f2 = distodd[n]; - f4 = distodd[N+n]; - f6 = distodd[2*N+n]; - f8 = distodd[3*N+n]; - f10 = distodd[4*N+n]; - f12 = distodd[5*N+n]; - f14 = distodd[6*N+n]; - f16 = distodd[7*N+n]; - f18 = distodd[8*N+n]; - //........................................................................ - f0 = disteven[n]; - f1 = disteven[N+n]; - f3 = disteven[2*N+n]; - f5 = disteven[3*N+n]; - f7 = disteven[4*N+n]; - f9 = disteven[5*N+n]; - f11 = disteven[6*N+n]; - f13 = disteven[7*N+n]; - f15 = disteven[8*N+n]; - f17 = disteven[9*N+n]; - //........................................................................ - // PERFORM RELAXATION PROCESS - //........................................................................ - //....................compute the moments............................................... - rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; - m1 = -30*f0-11*(f2+f1+f4+f3+f6+f5)+8*(f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18 +f17); - m2 = 12*f0-4*(f2+f1 +f4+f3+f6 +f5)+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; - jx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; - m4 = 4*(-f1+f2)+f7-f8+f9-f10+f11-f12+f13-f14; - jy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; - m6 = -4*(f3-f4)+f7-f8-f9+f10+f15-f16+f17-f18; - jz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18; - m8 = -4*(f5-f6)+f11-f12-f13+f14+f15-f16-f17+f18; - m9 = 2*(f1+f2)-f3-f4-f5-f6+f7+f8+f9+f10+f11+f12+f13+f14-2*(f15+f16+f17+f18); - m10 = -4*(f1+f2)+2*(f4+f3+f6+f5)+f8+f7+f10+f9+f12+f11+f14+f13-2*(f16+f15+f18+f17); - m11 = f4+f3-f6-f5+f8+f7+f10+f9-f12-f11-f14-f13; - m12 = -2*(f4+f3-f6-f5)+f8+f7+f10+f9-f12-f11-f14-f13; - m13 = f8+f7-f10-f9; - m14 = f16+f15-f18-f17; - m15 = f12+f11-f14-f13; - m16 = f7-f8+f9-f10-f11+f12-f13+f14; - m17 = -f7+f8+f9-f10+f15-f16+f17-f18; - m18 = f11-f12-f13+f14-f15+f16+f17-f18; - //..........Toelke, Fruediger et. al. 2006............... - if (C == 0.0) nx = ny = nz = 1.0; -#ifdef STOKES - m1 = m1 + rlx_setA*(- 11*rho -alpha*C - m1); - m2 = m2 + rlx_setA*(3*rho - m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8); - m9 = m9 + rlx_setA*( 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); - m10 = m10 + rlx_setA*( - m10); - m11 = m11 + rlx_setA*( 0.5*alpha*C*(ny*ny-nz*nz)- m11); - m12 = m12 + rlx_setA*( - m12); - m13 = m13 + rlx_setA*( 0.5*alpha*C*nx*ny - m13); - m14 = m14 + rlx_setA*( 0.5*alpha*C*ny*nz - m14); - m15 = m15 + rlx_setA*( 0.5*alpha*C*nx*nz - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); -#else - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) -alpha*C - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho)- m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8); - m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); - m10 = m10 + rlx_setA*( - m10); - m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) + 0.5*alpha*C*(ny*ny-nz*nz)- m11); - m12 = m12 + rlx_setA*( - m12); - m13 = m13 + rlx_setA*( (jx*jy/rho) + 0.5*alpha*C*nx*ny - m13); - m14 = m14 + rlx_setA*( (jy*jz/rho) + 0.5*alpha*C*ny*nz - m14); - m15 = m15 + rlx_setA*( (jx*jz/rho) + 0.5*alpha*C*nx*nz - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); -#endif - //.................inverse transformation...................................................... - f0 = 0.05263157894736842*rho-0.012531328320802*m1+0.04761904761904762*m2; - f1 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(jx-m4)+0.0555555555555555555555555*(m9-m10); - f2 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(m4-jx)+0.0555555555555555555555555*(m9-m10); - f3 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(jy-m6)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12); - f4 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(m6-jy)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12); - f5 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(jz-m8)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11); - f6 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(m8-jz)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11); - f7 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx+jy)+0.025*(m4+m6) - +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 - +0.04166666666666666*m12+0.25*m13+0.125*(m16-m17); - f8 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2-0.1*(jx+jy)-0.025*(m4+m6) - +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 - +0.04166666666666666*m12+0.25*m13+0.125*(m17-m16); - f9 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx-jy)+0.025*(m4-m6) - +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 - +0.04166666666666666*m12-0.25*m13+0.125*(m16+m17); - f10 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jy-jx)+0.025*(m6-m4) - +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 - +0.04166666666666666*m12-0.25*m13-0.125*(m16+m17); - f11 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jx+jz)+0.025*(m4+m8) - +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 - -0.04166666666666666*m12+0.25*m15+0.125*(m18-m16); - f12 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2-0.1*(jx+jz)-0.025*(m4+m8) - +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 - -0.04166666666666666*m12+0.25*m15+0.125*(m16-m18); - f13 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jx-jz)+0.025*(m4-m8) - +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 - -0.04166666666666666*m12-0.25*m15-0.125*(m16+m18); - f14 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jz-jx)+0.025*(m8-m4) - +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 - -0.04166666666666666*m12-0.25*m15+0.125*(m16+m18); - f15 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jy+jz)+0.025*(m6+m8) - -0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m17-m18); - f16 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2-0.1*(jy+jz)-0.025*(m6+m8) - -0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m18-m17); - f17 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jy-jz)+0.025*(m6-m8) - -0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14+0.125*(m17+m18); - f18 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jz-jy)+0.025*(m8-m6) - -0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14-0.125*(m17+m18); - //....................................................................................................... - // incorporate external force - f1 += 0.16666666*Fx; - f2 -= 0.16666666*Fx; - f3 += 0.16666666*Fy; - f4 -= 0.16666666*Fy; - f5 += 0.16666666*Fz; - f6 -= 0.16666666*Fz; - f7 += 0.08333333333*(Fx+Fy); - f8 -= 0.08333333333*(Fx+Fy); - f9 += 0.08333333333*(Fx-Fy); - f10 -= 0.08333333333*(Fx-Fy); - f11 += 0.08333333333*(Fx+Fz); - f12 -= 0.08333333333*(Fx+Fz); - f13 += 0.08333333333*(Fx-Fz); - f14 -= 0.08333333333*(Fx-Fz); - f15 += 0.08333333333*(Fy+Fz); - f16 -= 0.08333333333*(Fy+Fz); - f17 += 0.08333333333*(Fy-Fz); - f18 -= 0.08333333333*(Fy-Fz); - //*********** WRITE UPDATED VALUES TO MEMORY ****************** - // Write the updated distributions - //....EVEN..................................... - disteven[n] = f0; - disteven[N+n] = f2; - disteven[2*N+n] = f4; - disteven[3*N+n] = f6; - disteven[4*N+n] = f8; - disteven[5*N+n] = f10; - disteven[6*N+n] = f12; - disteven[7*N+n] = f14; - disteven[8*N+n] = f16; - disteven[9*N+n] = f18; - //....ODD...................................... - distodd[n] = f1; - distodd[N+n] = f3; - distodd[2*N+n] = f5; - distodd[3*N+n] = f7; - distodd[4*N+n] = f9; - distodd[5*N+n] = f11; - distodd[6*N+n] = f13; - distodd[7*N+n] = f15; - distodd[8*N+n] = f17; - - //...Store the Velocity.......................... - Velocity[n] = jx; - Velocity[N+n] = jy; - Velocity[2*N+n] = jz; - /* Velocity[3*n] = jx; - Velocity[3*n+1] = jy; - Velocity[3*n+2] = jz; - */ //...Store the Color Gradient.................... - // ColorGrad[3*n] = nx*C; - // ColorGrad[3*n+1] = ny*C; - // ColorGrad[3*n+2] = nz*C; - //............................................... - //*************************************************************** - } // check if n is in the solid - } // loop over n -} - -extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, - double *Velocity, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB, - double alpha, double beta, double Fx, double Fy, double Fz) -{ - - int i,j,k,n,nn,N; - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - // additional variables needed for computations - double rho,jx,jy,jz,C,nx,ny,nz; - - N = Nx*Ny*Nz; - char id; - - for (n=0; n 0){ - - //.......Back out the 3-D indices for node n.............. - k = n/(Nx*Ny); - j = (n-Nx*Ny*k)/Nx; - i = n-Nx*Ny*k-Nx*j; - //........................................................................ - //........Get 1-D index for this thread.................... - // n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x; - //........................................................................ - // COMPUTE THE COLOR GRADIENT - //........................................................................ - //.................Read Phase Indicator Values............................ - //........................................................................ - nn = n-1; // neighbor index (get convention) - if (i-1<0) nn += Nx; // periodic BC along the x-boundary - f1 = phi[nn]; // get neighbor for phi - 1 - //........................................................................ - nn = n+1; // neighbor index (get convention) - if (!(i+10)) delta=0; - a1 = na*(0.1111111111111111*(1+4.5*ux))+delta; - b1 = nb*(0.1111111111111111*(1+4.5*ux))-delta; - a2 = na*(0.1111111111111111*(1-4.5*ux))-delta; - b2 = nb*(0.1111111111111111*(1-4.5*ux))+delta; - - A_odd[n] = a1; - A_even[N+n] = a2; - B_odd[n] = b1; - B_even[N+n] = b2; - //............................................... - // q = 2 - // Cq = {0,1,0} - delta = beta*na*nb*nab*0.1111111111111111*ny; - if (!(na*nb*nab>0)) delta=0; - a1 = na*(0.1111111111111111*(1+4.5*uy))+delta; - b1 = nb*(0.1111111111111111*(1+4.5*uy))-delta; - a2 = na*(0.1111111111111111*(1-4.5*uy))-delta; - b2 = nb*(0.1111111111111111*(1-4.5*uy))+delta; - - A_odd[N+n] = a1; - A_even[2*N+n] = a2; - B_odd[N+n] = b1; - B_even[2*N+n] = b2; - //............................................... - // q = 4 - // Cq = {0,0,1} - delta = beta*na*nb*nab*0.1111111111111111*nz; - if (!(na*nb*nab>0)) delta=0; - a1 = na*(0.1111111111111111*(1+4.5*uz))+delta; - b1 = nb*(0.1111111111111111*(1+4.5*uz))-delta; - a2 = na*(0.1111111111111111*(1-4.5*uz))-delta; - b2 = nb*(0.1111111111111111*(1-4.5*uz))+delta; - - A_odd[2*N+n] = a1; - A_even[3*N+n] = a2; - B_odd[2*N+n] = b1; - B_even[3*N+n] = b2; - //............................................... - - /* // Construction and streaming for the components - for (idx=0; idx<3; idx++){ - //............................................... - // Distribution index - q = 2*idx; - // Associated discrete velocity - Cqx = D3Q7[idx][0]; - Cqy = D3Q7[idx][1]; - Cqz = D3Q7[idx][2]; - // Generate the Equilibrium Distribution - a1 = na*feq[q]; - b1 = nb*feq[q]; - a2 = na*feq[q+1]; - b2 = nb*feq[q+1]; - // Recolor the distributions - if (C > 0.0){ - sp = nx*double(Cqx)+ny*double(Cqy)+nz*double(Cqz); - //if (idx > 2) sp = 0.7071067811865475*sp; - //delta = sp*min( min(a1,a2), min(b1,b2) ); - delta = na*nb/(na+nb)*0.1111111111111111*sp; - //if (a1>0 && b1>0){ - a1 += beta*delta; - a2 -= beta*delta; - b1 -= beta*delta; - b2 += beta*delta; - } - // Save the re-colored distributions - A_odd[N*idx+n] = a1; - A_even[N*(idx+1)+n] = a2; - B_odd[N*idx+n] = b1; - B_even[N*(idx+1)+n] = b2; - //............................................... - } - */ - } - } -} - -//************************************************************************* -extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, - double beta, int Nx, int Ny, int Nz, bool pBC, int S) -{ - char id; - - int idx; - int in,jn,kn,n,nn,N; - int q,Cqx,Cqy,Cqz; - // int sendLoc; - - double na,nb; // density values - double ux,uy,uz; // flow velocity - double nx,ny,nz,C; // color gradient components - double a1,a2,b1,b2; - double sp,delta; - double feq[6]; // equilibrium distributions - // Set of Discrete velocities for the D3Q19 Model - int D3Q7[3][3]={{1,0,0},{0,1,0},{0,0,1}}; - N = Nx*Ny*Nz; - - for (n=0; n 0 && na+nb > 0.0){ - //.......Back out the 3-D indices for node n.............. - int k = n/(Nx*Ny); - int j = (n-Nx*Ny*k)/Nx; - int i = n-Nx*Ny*k-Nx*j; - //.....Load the Color gradient......... - nx = ColorGrad[n]; - ny = ColorGrad[N+n]; - nz = ColorGrad[2*N+n]; - C = sqrt(nx*nx+ny*ny+nz*nz); - nx = nx/C; - ny = ny/C; - nz = nz/C; - //....Load the flow velocity........... - ux = Velocity[n]; - uy = Velocity[N+n]; - uz = Velocity[2*N+n]; - //....Instantiate the density distributions - // Generate Equilibrium Distributions and stream - // Stationary value - distribution 0 - // Den[2*n] += 0.3333333333333333*na; - // Den[2*n+1] += 0.3333333333333333*nb; - Den[2*n] += 0.3333333333333333*na; - Den[2*n+1] += 0.3333333333333333*nb; - // Non-Stationary equilibrium distributions - feq[0] = 0.1111111111111111*(1+3*ux); - feq[1] = 0.1111111111111111*(1-3*ux); - feq[2] = 0.1111111111111111*(1+3*uy); - feq[3] = 0.1111111111111111*(1-3*uy); - feq[4] = 0.1111111111111111*(1+3*uz); - feq[5] = 0.1111111111111111*(1-3*uz); - // Construction and streaming for the components - for (idx=0; idx<3; idx++){ - // Distribution index - q = 2*idx; - // Associated discrete velocity - Cqx = D3Q7[idx][0]; - Cqy = D3Q7[idx][1]; - Cqz = D3Q7[idx][2]; - // Generate the Equilibrium Distribution - a1 = na*feq[q]; - b1 = nb*feq[q]; - a2 = na*feq[q+1]; - b2 = nb*feq[q+1]; - // Recolor the distributions - if (C > 0.0){ - sp = nx*double(Cqx)+ny*double(Cqy)+nz*double(Cqz); - //if (idx > 2) sp = 0.7071067811865475*sp; - //delta = sp*min( min(a1,a2), min(b1,b2) ); - delta = na*nb/(na+nb)*0.1111111111111111*sp; - //if (a1>0 && b1>0){ - a1 += beta*delta; - a2 -= beta*delta; - b1 -= beta*delta; - b2 += beta*delta; - } - - // .......Get the neighbor node.............. - //nn = n + Stride[idx]; - in = i+Cqx; - jn = j+Cqy; - kn = k+Cqz; - - // Adjust for periodic BC, if necessary - // if (in<0) in+= Nx; - // if (jn<0) jn+= Ny; - // if (kn<0) kn+= Nz; - // if (!(in 0 ){ - // Get the density value (Streaming already performed) - Na = Den[n]; - Nb = Den[N+n]; - Phi[n] = (Na-Nb)/(Na+Nb); - } - } - //................................................................... -} - -extern "C" void SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){ - int n; - for (n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){ - Phi[n] = value; - } -} - -/* -// ************************************************************************* -extern "C" void ScaLBL_Color_Init( int nblocks, int nthreads, int S, - char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) -{ - ScaLBL_Color_Init <<>> (ID, Den, Phi, das, dbs, Nx, Ny, Nz, S); -} -// ************************************************************************* -extern "C" void ScaLBL_D3Q19_ColorGradient(int nBlocks, int nthreads, int S, - char *ID, double *Phi, double *ColorGrad, int Nx, int Ny, int Nz) -{ - ScaLBL_D3Q19_ColorGradient<<>>(ID, Phi, ColorGrad, Nx, Ny, Nz, S); -} -// ************************************************************************* -extern "C" void ColorCollide(int nBlocks, int nthreads, int S, - char *ID, double *f_even, double *f_odd, double *ColorGrad, double *Velocity, - double rlxA, double rlxB,double alpha, double beta, double Fx, double Fy, double Fz, - int Nx, int Ny, int Nz, bool pBC) -{ - ColorCollide<<>>(ID, f_even, f_odd, ColorGrad, Velocity, Nx, Ny, Nz, S, - rlxA, rlxB, alpha, beta, Fx, Fy, Fz, pBC); -} -// ************************************************************************* -extern "C" void ScaLBL_D3Q19_ColorCollide(int nBlocks, int nthreads, int S, - char *ID, double *f_even, double *f_odd, double *Phi, double *ColorGrad, - double *Velocity, int Nx, int Ny, int Nz,double rlxA, double rlxB, - double alpha, double beta, double Fx, double Fy, double Fz) -{ - ScaLBL_D3Q19_ColorCollide<<>>(ID, f_even, f_odd, Phi, ColorGrad, Velocity, Nx, Ny, Nz, S, - rlxA, rlxB, alpha, beta, Fx, Fy, Fz); -// bool pBC = false; -// ColorCollide<<>>(ID, f_even, f_odd, ColorGrad, Velocity, Nx, Ny, Nz, S, -// rlxA, rlxB, alpha, beta, Fx, Fy, Fz, pBC); -} - -// ************************************************************************* -extern "C" void DensityStreamD3Q7(int nBlocks, int nthreads, int S, - char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, - double beta, int Nx, int Ny, int Nz, bool pBC) -{ - DensityStreamD3Q7<<>>(ID,Den,Copy,Phi,ColorGrad,Velocity,beta,Nx,Ny,Nz,pBC,S); -} -// ************************************************************************* -extern "C" void ScaLBL_ComputePhaseField(int nBlocks, int nthreads, int S, - char *ID, double *Phi, double *Copy, double *Den, int N) -{ - ScaLBL_ComputePhaseField<<>>(ID,Phi,Copy,Den,N,S); -} -// ************************************************************************* -extern "C" void ComputePressure(int nBlocks, int nthreads, int S, - char *ID, double *disteven, double *distodd, - double *Pressure, int Nx, int Ny, int Nz) -{ - - ScaLBL_D3Q19_Pressure<<>>(ID,disteven,distodd,Pressure,Nx,Ny,Nz,S); - -} -*/ diff --git a/gpu/Color.cu b/gpu/Color.cu index 8d91c8c5..9ac48199 100644 --- a/gpu/Color.cu +++ b/gpu/Color.cu @@ -1388,7 +1388,7 @@ __global__ void dvc_ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den //................................................................... } -__global__ void dvc_SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice) +__global__ void dvc_ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice) { int n = Slice*Nx*Ny + blockIdx.x*blockDim.x + threadIdx.x; if (n < (Slice+1)*Nx*Ny){ diff --git a/gpu/D3Q7.cu b/gpu/D3Q7.cu index 613a302e..19b0fcea 100644 --- a/gpu/D3Q7.cu +++ b/gpu/D3Q7.cu @@ -3,7 +3,7 @@ #define NBLOCKS 32 #define NTHREADS 128 -__global__ void dvc_ScaLBL_D3Q19_Unpack(int *list, int count, double *sendbuf, double *Data, int N){ +__global__ void dvc_ScaLBL_Scalar_Pack(int *list, int count, double *sendbuf, double *Data, int N){ //.................................................................................... // Pack distribution q into the send buffer for the listed lattice sites // dist may be even or odd distributions stored by stream layout diff --git a/gpu/Extras.cu b/gpu/Extras.cu index dffd3704..08429d0c 100644 --- a/gpu/Extras.cu +++ b/gpu/Extras.cu @@ -2,7 +2,7 @@ #include #include -extern "C" void ScaLBL_ScaLBL_ScaLBL_AllocateDeviceMemory(void** address, size_t size){ +extern "C" void ScaLBL_AllocateDeviceMemory(void** address, size_t size){ cudaMalloc(address,size); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -10,7 +10,7 @@ extern "C" void ScaLBL_ScaLBL_ScaLBL_AllocateDeviceMemory(void** address, size_t } } -extern "C" void ScaLBL_ScaLBL_CopyToDevice(void* dest, const void* source, size_t size){ +extern "C" void ScaLBL_CopyToDevice(void* dest, const void* source, size_t size){ cudaMemcpy(dest,source,size,cudaMemcpyHostToDevice); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ diff --git a/gpu/Multiphase.cu b/gpu/Multiphase.cu deleted file mode 100644 index 8d91c8c5..00000000 --- a/gpu/Multiphase.cu +++ /dev/null @@ -1,1468 +0,0 @@ -#include - -#define NBLOCKS 32 -#define NTHREADS 128 - -__global__ void dvc_ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz) -{ - //int i,j,k; - int n,N; - char id; - N = Nx*Ny*Nz; - - int S = N/NBLOCKS/NTHREADS + 1; - for (int s=0; s 0){ - - // Retrieve the color gradient - nx = ColorGrad[n]; - ny = ColorGrad[N+n]; - nz = ColorGrad[2*N+n]; - //...........Normalize the Color Gradient................................. - C = sqrt(nx*nx+ny*ny+nz*nz); - nx = nx/C; - ny = ny/C; - nz = nz/C; - //......No color gradient at z-boundary if pressure BC are set............. - // if (pBC && k==0) nx = ny = nz = 0.f; - // if (pBC && k==Nz-1) nx = ny = nz = 0.f; - //........................................................................ - // READ THE DISTRIBUTIONS - // (read from opposite array due to previous swap operation) - //........................................................................ - f2 = distodd[n]; - f4 = distodd[N+n]; - f6 = distodd[2*N+n]; - f8 = distodd[3*N+n]; - f10 = distodd[4*N+n]; - f12 = distodd[5*N+n]; - f14 = distodd[6*N+n]; - f16 = distodd[7*N+n]; - f18 = distodd[8*N+n]; - //........................................................................ - f0 = disteven[n]; - f1 = disteven[N+n]; - f3 = disteven[2*N+n]; - f5 = disteven[3*N+n]; - f7 = disteven[4*N+n]; - f9 = disteven[5*N+n]; - f11 = disteven[6*N+n]; - f13 = disteven[7*N+n]; - f15 = disteven[8*N+n]; - f17 = disteven[9*N+n]; - //........................................................................ - // PERFORM RELAXATION PROCESS - //........................................................................ - //....................compute the moments............................................... - rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; - m1 = -30*f0-11*(f2+f1+f4+f3+f6+f5)+8*(f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18 +f17); - m2 = 12*f0-4*(f2+f1 +f4+f3+f6 +f5)+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; - jx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; - m4 = 4*(-f1+f2)+f7-f8+f9-f10+f11-f12+f13-f14; - jy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; - m6 = -4*(f3-f4)+f7-f8-f9+f10+f15-f16+f17-f18; - jz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18; - m8 = -4*(f5-f6)+f11-f12-f13+f14+f15-f16-f17+f18; - m9 = 2*(f1+f2)-f3-f4-f5-f6+f7+f8+f9+f10+f11+f12+f13+f14-2*(f15+f16+f17+f18); - m10 = -4*(f1+f2)+2*(f4+f3+f6+f5)+f8+f7+f10+f9+f12+f11+f14+f13-2*(f16+f15+f18+f17); - m11 = f4+f3-f6-f5+f8+f7+f10+f9-f12-f11-f14-f13; - m12 = -2*(f4+f3-f6-f5)+f8+f7+f10+f9-f12-f11-f14-f13; - m13 = f8+f7-f10-f9; - m14 = f16+f15-f18-f17; - m15 = f12+f11-f14-f13; - m16 = f7-f8+f9-f10-f11+f12-f13+f14; - m17 = -f7+f8+f9-f10+f15-f16+f17-f18; - m18 = f11-f12-f13+f14-f15+f16+f17-f18; - //..........Toelke, Fruediger et. al. 2006............... - if (C == 0.0) nx = ny = nz = 1.0; - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) -alpha*C - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho)- m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8); - m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); - m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); - m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) + 0.5*alpha*C*(ny*ny-nz*nz)- m11); - m12 = m12 + rlx_setA*( -0.5*((jy*jy-jz*jz)/rho) - m12); - m13 = m13 + rlx_setA*( (jx*jy/rho) + 0.5*alpha*C*nx*ny - m13); - m14 = m14 + rlx_setA*( (jy*jz/rho) + 0.5*alpha*C*ny*nz - m14); - m15 = m15 + rlx_setA*( (jx*jz/rho) + 0.5*alpha*C*nx*nz - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //.................inverse transformation...................................................... - f0 = 0.05263157894736842*rho-0.012531328320802*m1+0.04761904761904762*m2; - f1 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(jx-m4)+0.0555555555555555555555555*(m9-m10); - f2 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(m4-jx)+0.0555555555555555555555555*(m9-m10); - f3 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(jy-m6)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12); - f4 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(m6-jy)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12); - f5 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(jz-m8)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11); - f6 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2 - +0.1*(m8-jz)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11); - f7 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx+jy)+0.025*(m4+m6) - +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 - +0.04166666666666666*m12+0.25*m13+0.125*(m16-m17); - f8 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2-0.1*(jx+jy)-0.025*(m4+m6) - +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 - +0.04166666666666666*m12+0.25*m13+0.125*(m17-m16); - f9 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx-jy)+0.025*(m4-m6) - +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 - +0.04166666666666666*m12-0.25*m13+0.125*(m16+m17); - f10 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jy-jx)+0.025*(m6-m4) - +0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11 - +0.04166666666666666*m12-0.25*m13-0.125*(m16+m17); - f11 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jx+jz)+0.025*(m4+m8) - +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 - -0.04166666666666666*m12+0.25*m15+0.125*(m18-m16); - f12 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2-0.1*(jx+jz)-0.025*(m4+m8) - +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 - -0.04166666666666666*m12+0.25*m15+0.125*(m16-m18); - f13 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jx-jz)+0.025*(m4-m8) - +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 - -0.04166666666666666*m12-0.25*m15-0.125*(m16+m18); - f14 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jz-jx)+0.025*(m8-m4) - +0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11 - -0.04166666666666666*m12-0.25*m15+0.125*(m16+m18); - f15 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jy+jz)+0.025*(m6+m8) - -0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m17-m18); - f16 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2-0.1*(jy+jz)-0.025*(m6+m8) - -0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m18-m17); - f17 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jy-jz)+0.025*(m6-m8) - -0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14+0.125*(m17+m18); - f18 = 0.05263157894736842*rho+0.003341687552213868*m1 - +0.003968253968253968*m2+0.1*(jz-jy)+0.025*(m8-m6) - -0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14-0.125*(m17+m18); - //....................................................................................................... - // incorporate external force - f1 += 0.16666666*Fx; - f2 -= 0.16666666*Fx; - f3 += 0.16666666*Fy; - f4 -= 0.16666666*Fy; - f5 += 0.16666666*Fz; - f6 -= 0.16666666*Fz; - f7 += 0.08333333333*(Fx+Fy); - f8 -= 0.08333333333*(Fx+Fy); - f9 += 0.08333333333*(Fx-Fy); - f10 -= 0.08333333333*(Fx-Fy); - f11 += 0.08333333333*(Fx+Fz); - f12 -= 0.08333333333*(Fx+Fz); - f13 += 0.08333333333*(Fx-Fz); - f14 -= 0.08333333333*(Fx-Fz); - f15 += 0.08333333333*(Fy+Fz); - f16 -= 0.08333333333*(Fy+Fz); - f17 += 0.08333333333*(Fy-Fz); - f18 -= 0.08333333333*(Fy-Fz); - //*********** WRITE UPDATED VALUES TO MEMORY ****************** - // Write the updated distributions - //....EVEN..................................... - disteven[n] = f0; - disteven[N+n] = f2; - disteven[2*N+n] = f4; - disteven[3*N+n] = f6; - disteven[4*N+n] = f8; - disteven[5*N+n] = f10; - disteven[6*N+n] = f12; - disteven[7*N+n] = f14; - disteven[8*N+n] = f16; - disteven[9*N+n] = f18; - //....ODD...................................... - distodd[n] = f1; - distodd[N+n] = f3; - distodd[2*N+n] = f5; - distodd[3*N+n] = f7; - distodd[4*N+n] = f9; - distodd[5*N+n] = f11; - distodd[6*N+n] = f13; - distodd[7*N+n] = f15; - distodd[8*N+n] = f17; - - //...Store the Velocity.......................... - Velocity[n] = jx; - Velocity[N+n] = jy; - Velocity[2*N+n] = jz; - /* Velocity[3*n] = jx; - Velocity[3*n+1] = jy; - Velocity[3*n+2] = jz; - */ //...Store the Color Gradient.................... - // ColorGrad[3*n] = nx*C; - // ColorGrad[3*n+1] = ny*C; - // ColorGrad[3*n+2] = nz*C; - //............................................... - //*************************************************************** - } // check if n is in the solid - } // loop over n -} - -__global__ void dvc_ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, - double *Velocity, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB, - double alpha, double beta, double Fx, double Fy, double Fz) -{ - - int i,j,k,n,nn,N; - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - // additional variables needed for computations - double rho,jx,jy,jz,C,nx,ny,nz; - char id; - - N = Nx*Ny*Nz; - - int S = N/NBLOCKS/NTHREADS + 1; - for (int s=0; s 0){ - - //.......Back out the 3-D indices for node n.............. - k = n/(Nx*Ny); - j = (n-Nx*Ny*k)/Nx; - i = n-Nx*Ny*k-Nx*j; - //........................................................................ - //........Get 1-D index for this thread.................... - // n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x; - //........................................................................ - // COMPUTE THE COLOR GRADIENT - //........................................................................ - //.................Read Phase Indicator Values............................ - //........................................................................ - nn = n-1; // neighbor index (get convention) - if (i-1<0) nn += Nx; // periodic BC along the x-boundary - f1 = phi[nn]; // get neighbor for phi - 1 - //........................................................................ - nn = n+1; // neighbor index (get convention) - if (!(i+10)) delta=0; - a1 = na*(0.1111111111111111*(1+4.5*ux))+delta; - b1 = nb*(0.1111111111111111*(1+4.5*ux))-delta; - a2 = na*(0.1111111111111111*(1-4.5*ux))-delta; - b2 = nb*(0.1111111111111111*(1-4.5*ux))+delta; - - A_odd[n] = a1; - A_even[N+n] = a2; - B_odd[n] = b1; - B_even[N+n] = b2; - //............................................... - // q = 2 - // Cq = {0,1,0} - delta = beta*na*nb*nab*0.1111111111111111*ny; - if (!(na*nb*nab>0)) delta=0; - a1 = na*(0.1111111111111111*(1+4.5*uy))+delta; - b1 = nb*(0.1111111111111111*(1+4.5*uy))-delta; - a2 = na*(0.1111111111111111*(1-4.5*uy))-delta; - b2 = nb*(0.1111111111111111*(1-4.5*uy))+delta; - - A_odd[N+n] = a1; - A_even[2*N+n] = a2; - B_odd[N+n] = b1; - B_even[2*N+n] = b2; - //............................................... - // q = 4 - // Cq = {0,0,1} - delta = beta*na*nb*nab*0.1111111111111111*nz; - if (!(na*nb*nab>0)) delta=0; - a1 = na*(0.1111111111111111*(1+4.5*uz))+delta; - b1 = nb*(0.1111111111111111*(1+4.5*uz))-delta; - a2 = na*(0.1111111111111111*(1-4.5*uz))-delta; - b2 = nb*(0.1111111111111111*(1-4.5*uz))+delta; - - A_odd[2*N+n] = a1; - A_even[3*N+n] = a2; - B_odd[2*N+n] = b1; - B_even[3*N+n] = b2; - - } - } -} - -//************************************************************************* -__global__ void dvc_DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, - double beta, int Nx, int Ny, int Nz, bool pBC) -{ - char id; - - int idx; - int in,jn,kn,n,nn,N; - int q,Cqx,Cqy,Cqz; - // int sendLoc; - - double na,nb; // density values - double ux,uy,uz; // flow velocity - double nx,ny,nz,C; // color gradient components - double a1,a2,b1,b2; - double sp,delta; - double feq[6]; // equilibrium distributions - // Set of Discrete velocities for the D3Q19 Model - int D3Q7[3][3]={{1,0,0},{0,1,0},{0,0,1}}; - N = Nx*Ny*Nz; - - int S = N/NBLOCKS/NTHREADS + 1; - for (int s=0; s 0 && na+nb > 0.0){ - //.......Back out the 3-D indices for node n.............. - int k = n/(Nx*Ny); - int j = (n-Nx*Ny*k)/Nx; - int i = n-Nx*Ny*k-Nx*j; - //.....Load the Color gradient......... - nx = ColorGrad[n]; - ny = ColorGrad[N+n]; - nz = ColorGrad[2*N+n]; - C = sqrt(nx*nx+ny*ny+nz*nz); - if (C == 0.0) C=1.0; - nx = nx/C; - ny = ny/C; - nz = nz/C; - //....Load the flow velocity........... - ux = Velocity[n]; - uy = Velocity[N+n]; - uz = Velocity[2*N+n]; - //....Instantiate the density distributions - // Generate Equilibrium Distributions and stream - // Stationary value - distribution 0 - // Den[2*n] += 0.3333333333333333*na; - // Den[2*n+1] += 0.3333333333333333*nb; - Den[2*n] += 0.3333333333333333*na; - Den[2*n+1] += 0.3333333333333333*nb; - // Non-Stationary equilibrium distributions - feq[0] = 0.1111111111111111*(1+3*ux); - feq[1] = 0.1111111111111111*(1-3*ux); - feq[2] = 0.1111111111111111*(1+3*uy); - feq[3] = 0.1111111111111111*(1-3*uy); - feq[4] = 0.1111111111111111*(1+3*uz); - feq[5] = 0.1111111111111111*(1-3*uz); - // Construction and streaming for the components - for (idx=0; idx<3; idx++){ - // Distribution index - q = 2*idx; - // Associated discrete velocity - Cqx = D3Q7[idx][0]; - Cqy = D3Q7[idx][1]; - Cqz = D3Q7[idx][2]; - // Generate the Equilibrium Distribution - a1 = na*feq[q]; - b1 = nb*feq[q]; - a2 = na*feq[q+1]; - b2 = nb*feq[q+1]; - // Recolor the distributions - if (C > 0.0){ - sp = nx*double(Cqx)+ny*double(Cqy)+nz*double(Cqz); - //if (idx > 2) sp = 0.7071067811865475*sp; - //delta = sp*min( min(a1,a2), min(b1,b2) ); - delta = na*nb/(na+nb)*0.1111111111111111*sp; - //if (a1>0 && b1>0){ - a1 += beta*delta; - a2 -= beta*delta; - b1 -= beta*delta; - b2 += beta*delta; - } - - // .......Get the neighbor node.............. - //nn = n + Stride[idx]; - in = i+Cqx; - jn = j+Cqy; - kn = k+Cqz; - - // Adjust for periodic BC, if necessary - // if (in<0) in+= Nx; - // if (jn<0) jn+= Ny; - // if (kn<0) kn+= Nz; - // if (!(in 0){ - // Get the density value (Streaming already performed) - Na = Den[n]; - Nb = Den[N+n]; - Phi[n] = (Na-Nb)/(Na+Nb); - } - } - } - //................................................................... -} - -__global__ void dvc_SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice) -{ - int n = Slice*Nx*Ny + blockIdx.x*blockDim.x + threadIdx.x; - if (n < (Slice+1)*Nx*Ny){ - Phi[n] = value; - } -} - -extern "C" void SetPhiSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){ - int GRID = Nx*Ny / 512 + 1; - dvc_SetPhiSlice_z<<>>(Phi,value,Nx,Ny,Nz,Slice); -} - -extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz){ - dvc_ScaLBL_Color_Init<<>>(ID, Den, Phi, das, dbs, Nx, Ny, Nz); -} - -extern "C" void ScaLBL_Color_InitDistance(char *ID, double *Den, double *Phi, double *Distance, - double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz){ - - dvc_ScaLBL_Color_InitDistance<<>>(ID, Den, Phi, Distance, das, dbs, beta, xp, Nx, Ny, Nz); -} - -extern "C" void ScaLBL_D3Q19_ColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz){ - dvc_ScaLBL_D3Q19_ColorGradient<<>>(ID, phi, ColorGrad, Nx, Ny, Nz); -} - -extern "C" void ColorCollide( char *ID, double *disteven, double *distodd, double *ColorGrad, - double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB, - double alpha, double beta, double Fx, double Fy, double Fz, bool pBC){ - dvc_ColorCollide<<>>( ID, disteven, distodd, ColorGrad,Velocity, Nx, Ny, Nz,rlx_setA, rlx_setB, - alpha, beta, Fx, Fy, Fz, pBC); - -} - -extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad, - double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB, - double alpha, double beta, double Fx, double Fy, double Fz){ - dvc_ScaLBL_D3Q19_ColorCollide<<>>(ID, disteven, distodd, phi, ColorGrad, Velocity, Nx, Ny, Nz, rlx_setA, rlx_setB, - alpha, beta, Fx, Fy, Fz); - -} - -extern "C" void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity, - double beta, int Nx, int Ny, int Nz, bool pBC){ - - dvc_DensityStreamD3Q7<<>>(ID, Den, Copy, Phi, ColorGrad, Velocity, beta, Nx, Ny, Nz, pBC); -} - -extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N){ - dvc_ScaLBL_ComputePhaseField<<>>(ID, Phi, Den, N); -} -extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, - double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC){ - dvc_ScaLBL_D3Q7_ColorCollideMass<<>>(ID, A_even, A_odd, B_even, B_odd, Den, Phi, ColorGrad, Velocity, beta, N, pBC); -} -// Pressure Boundary Conditions Functions -extern "C" void ScaLBL_Color_BC_z(double *Phi, double *Den, double *A_even, double *A_odd, - double *B_even, double *B_odd, int Nx, int Ny, int Nz){ - int GRID = Nx*Ny / 512 + 1; - dvc_ScaLBL_Color_BC_z<<< GRID,512 >>>(Phi, Den, A_even, A_odd, B_even, B_odd, Nx, Ny, Nz); -} -extern "C" void ScaLBL_Color_BC_Z(double *Phi, double *Den, double *A_even, double *A_odd, - double *B_even, double *B_odd, int Nx, int Ny, int Nz){ - - int GRID = Nx*Ny / 512 + 1; - dvc_ScaLBL_Color_BC_Z<<< GRID,512 >>>(Phi, Den, A_even, A_odd, B_even, B_odd, Nx, Ny, Nz); -} -extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din, int Nx, int Ny, int Nz){ - int GRID = Nx*Ny / 512 + 1; - dvc_ScaLBL_D3Q19_Pressure_BC_z<<>>(disteven, distodd, din, Nx, Ny, Nz); -} -extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout, - int Nx, int Ny, int Nz, int outlet){ - int GRID = Nx*Ny / 512 + 1; - dvc_ScaLBL_D3Q19_Pressure_BC_Z<<>>(disteven, distodd, dout, Nx, Ny, Nz, outlet); -} - diff --git a/tests/TestBubble.cpp b/tests/TestBubble.cpp index 72c6878a..6b27dabe 100644 --- a/tests/TestBubble.cpp +++ b/tests/TestBubble.cpp @@ -654,43 +654,43 @@ int main(int argc, char **argv) double *recvbuf_xy, *recvbuf_yz, *recvbuf_xz, *recvbuf_Xy, *recvbuf_Yz, *recvbuf_xZ; double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ; //...................................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_X, 5*sendCount_X*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_y, 5*sendCount_y*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_z, 5*sendCount_z*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double)); // Allocatevoid * memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_X, 5*sendCount_X*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_y, 5*sendCount_y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_z, 5*sendCount_z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double)); // Allocatevoid * memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory //...................................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_x, 5*recvCount_x*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_X, 5*recvCount_X*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_y, 5*recvCount_y*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_z, 5*recvCount_z*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_x, 5*recvCount_x*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_X, 5*recvCount_X*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_y, 5*recvCount_y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_z, 5*recvCount_z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory //...................................................................................... int *dvcSendList_x, *dvcSendList_y, *dvcSendList_z, *dvcSendList_X, *dvcSendList_Y, *dvcSendList_Z; int *dvcSendList_xy, *dvcSendList_yz, *dvcSendList_xz, *dvcSendList_Xy, *dvcSendList_Yz, *dvcSendList_xZ; @@ -700,43 +700,43 @@ int main(int argc, char **argv) int *dvcRecvList_xy, *dvcRecvList_yz, *dvcRecvList_xz, *dvcRecvList_Xy, *dvcRecvList_Yz, *dvcRecvList_xZ; int *dvcRecvList_xY, *dvcRecvList_yZ, *dvcRecvList_Xz, *dvcRecvList_XY, *dvcRecvList_YZ, *dvcRecvList_XZ; //...................................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_x, sendCount_x*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_X, sendCount_X*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_y, sendCount_y*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Y, sendCount_Y*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_z, sendCount_z*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Z, sendCount_Z*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xy, sendCount_xy*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xY, sendCount_xY*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Xy, sendCount_Xy*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_XY, sendCount_XY*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xz, sendCount_xz*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xZ, sendCount_xZ*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Xz, sendCount_Xz*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_XZ, sendCount_XZ*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_yz, sendCount_yz*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_yZ, sendCount_yZ*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Yz, sendCount_Yz*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_YZ, sendCount_YZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_x, sendCount_x*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_X, sendCount_X*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_y, sendCount_y*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Y, sendCount_Y*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_z, sendCount_z*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Z, sendCount_Z*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xy, sendCount_xy*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xY, sendCount_xY*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Xy, sendCount_Xy*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_XY, sendCount_XY*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xz, sendCount_xz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_xZ, sendCount_xZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Xz, sendCount_Xz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_XZ, sendCount_XZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_yz, sendCount_yz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_yZ, sendCount_yZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_Yz, sendCount_Yz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcSendList_YZ, sendCount_YZ*sizeof(int)); // Allocate device memory //...................................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_x, recvCount_x*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_X, recvCount_X*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_y, recvCount_y*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Y, recvCount_Y*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_z, recvCount_z*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Z, recvCount_Z*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xy, recvCount_xy*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xY, recvCount_xY*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Xy, recvCount_Xy*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_XY, recvCount_XY*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xz, recvCount_xz*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xZ, recvCount_xZ*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Xz, recvCount_Xz*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_XZ, recvCount_XZ*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_yz, recvCount_yz*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_yZ, recvCount_yZ*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_x, recvCount_x*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_X, recvCount_X*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_y, recvCount_y*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Y, recvCount_Y*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_z, recvCount_z*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Z, recvCount_Z*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xy, recvCount_xy*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xY, recvCount_xY*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Xy, recvCount_Xy*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_XY, recvCount_XY*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xz, recvCount_xz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_xZ, recvCount_xZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Xz, recvCount_Xz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_XZ, recvCount_XZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_yz, recvCount_yz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_yZ, recvCount_yZ*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_Yz, recvCount_Yz*sizeof(int)); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &dvcRecvList_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory //...................................................................................... MPI_Barrier(comm); if (rank==0) printf ("Prepare to copy send/recv Lists to device \n"); @@ -953,7 +953,7 @@ int main(int argc, char **argv) //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device ScaLBL_CopyToDevice(ID, id, N); //........................................................................... @@ -969,23 +969,23 @@ int main(int argc, char **argv) double *f_even,*f_odd; double *A_even,*A_odd,*B_even,*B_odd; //........................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &A_even, 4*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &A_odd, 3*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &B_even, 4*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &B_odd, 3*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &A_even, 4*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &A_odd, 3*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &B_even, 4*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &B_odd, 3*dist_mem_size); // Allocate device memory //........................................................................... double *Phi,*Den; // double *Copy; double *ColorGrad, *Velocity, *Pressure; //........................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &Phi, dist_mem_size); - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &Pressure, dist_mem_size); - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); -// ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &Copy, 2*dist_mem_size); - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Phi, dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); +// ScaLBL_AllocateDeviceMemory((void **) &Copy, 2*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*dist_mem_size); //........................................................................... //........................................................................... // Phase indicator (in array form as needed by PMMC algorithm) @@ -1198,8 +1198,8 @@ int main(int argc, char **argv) //........................................................................... ScaLBL_D3Q19_Init(ID, f_even, f_odd, Nx, Ny, Nz); //...................................................................... - // ScaLBL_Color_InitDistance(ID, Copy, Phi, SDs.data(), das, dbs, beta, xIntPos, Nx, Ny, Nz, S); - ScaLBL_Color_InitDistance(ID, Den, Phi, SDs.data(), das, dbs, beta, xIntPos, Nx, Ny, Nz); + //ScaLBL_Color_InitDistance(ID, Den, Phi, SDs.data(), das, dbs, beta, xIntPos, Nx, Ny, Nz); + ScaLBL_Color_Init(ID, Den, Phi, das, dbs, Nx, Ny, Nz); ScaLBL_D3Q7_Init(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); ScaLBL_D3Q7_Init(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); //...................................................................... diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index 232a366b..068e7343 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -365,7 +365,7 @@ int main(int argc, char **argv) //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device ScaLBL_CopyToDevice(ID, id, N); //........................................................................... @@ -378,8 +378,8 @@ int main(int argc, char **argv) //......................device distributions................................. double *f_even,*f_odd; //........................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory //........................................................................... double *f_even_host,*f_odd_host; f_even_host = new double [10*N]; diff --git a/tests/TestMassConservationD3Q7.cpp b/tests/TestMassConservationD3Q7.cpp index 2564d5fd..f21b2871 100644 --- a/tests/TestMassConservationD3Q7.cpp +++ b/tests/TestMassConservationD3Q7.cpp @@ -90,22 +90,22 @@ int main(int argc, char **argv) //......................device distributions................................. double *A_even,*A_odd,*B_even,*B_odd; //........................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &A_even, 4*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &A_odd, 3*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &B_even, 4*dist_mem_size); // Allocate device memory - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &B_odd, 3*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &A_even, 4*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &A_odd, 3*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &B_even, 4*dist_mem_size); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &B_odd, 3*dist_mem_size); // Allocate device memory //........................................................................... double *Phi,*Den; double *ColorGrad, *Velocity; //........................................................................... - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &Phi, dist_mem_size); - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*dist_mem_size); - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Phi, dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; - ScaLBL_ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory + ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device ScaLBL_CopyToDevice(ID, id, N); ScaLBL_CopyToDevice(Velocity, Vel, 3*N*sizeof(double)); diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index 44d21ece..eca42998 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -654,13 +654,13 @@ int main(int argc, char **argv) if (BoundaryCondition==2 && Mask.kproc == 0) { ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz); //ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - SetPhiSlice_z(Phi,1.0,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); } if (BoundaryCondition==2 && Mask.kproc == nprocz-1){ ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); //ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - SetPhiSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); } // Set dynamic pressure boundary conditions @@ -871,12 +871,12 @@ int main(int argc, char **argv) if (BoundaryCondition==2 && Mask.kproc == 0) { ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz); //ScaLBL_Color_BC_z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - SetPhiSlice_z(Phi,1.0,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); } if (BoundaryCondition==2 && Mask.kproc == nprocz-1){ ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); //ScaLBL_Color_BC_Z(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - SetPhiSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); } if (BoundaryCondition==3){ diff --git a/tests/lbpm_color_simulator.h b/tests/lbpm_color_simulator.h index b22032e3..1b85ae0d 100644 --- a/tests/lbpm_color_simulator.h +++ b/tests/lbpm_color_simulator.h @@ -287,14 +287,14 @@ void run_analysis( int timestep, int restart_interval, PROFILE_START("start_analysis"); // Copy the appropriate variables to the host (so we can spawn new threads) - DeviceBarrier(); + ScaLBL_DeviceBarrier(); PROFILE_START("Copy data to host",1); std::shared_ptr phase; if ( (type&CopyPhaseIndicator)!=0 || (type&ComputeAverages)!=0 || (type&CopySimState)!=0 || (type&IdentifyBlobs)!=0 ) { phase = std::shared_ptr(new DoubleArray(Nx,Ny,Nz)); - CopyToHost(phase->data(),Phi,N*sizeof(double)); + ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double)); } if ( (type&CopyPhaseIndicator)!=0 ) { memcpy(Averages.Phase_tplus.data(),phase->data(),N*sizeof(double)); @@ -308,8 +308,8 @@ void run_analysis( int timestep, int restart_interval, // Copy the members of Averages to the cpu (phase was copied above) // Wait PROFILE_START("Copy-Pressure",1); - ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - DeviceBarrier(); + ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); + ScaLBL_DeviceBarrier(); PROFILE_STOP("Copy-Pressure",1); PROFILE_START("Copy-Wait",1); tpool.wait(wait.analysis); @@ -317,10 +317,10 @@ void run_analysis( int timestep, int restart_interval, PROFILE_STOP("Copy-Wait",1); PROFILE_START("Copy-State",1); memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double)); - CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); - CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); - CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); - CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Press.data(),Pressure,N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_x.data(),&Velocity[0],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_y.data(),&Velocity[N],N*sizeof(double)); + ScaLBL_CopyToHost(Averages.Vel_z.data(),&Velocity[2*N],N*sizeof(double)); PROFILE_STOP("Copy-State",1); } std::shared_ptr cDen, cDistEven, cDistOdd; @@ -329,9 +329,9 @@ void run_analysis( int timestep, int restart_interval, cDen = std::shared_ptr(new double[2*N],DeleteArray); cDistEven = std::shared_ptr(new double[10*N],DeleteArray); cDistOdd = std::shared_ptr(new double[9*N],DeleteArray); - CopyToHost(cDistEven.get(),f_even,10*N*sizeof(double)); - CopyToHost(cDistOdd.get(),f_odd,9*N*sizeof(double)); - CopyToHost(cDen.get(),Den,2*N*sizeof(double)); + ScaLBL_CopyToHost(cDistEven.get(),f_even,10*N*sizeof(double)); + ScaLBL_CopyToHost(cDistOdd.get(),f_odd,9*N*sizeof(double)); + ScaLBL_CopyToHost(cDen.get(),Den,2*N*sizeof(double)); } PROFILE_STOP("Copy data to host",1);